path: root/subprojects/freeamo/src/search-pages

#!/usr/bin/env bash
#    This file is part of FreeAMO
#    Copyright (C) 2017 David Hedlund
#
#    FreeAMO is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    FreeAMO is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

cd "$run_path/build/json/search-pages" || exit

base_uri="https://addons.mozilla.org/api/v$api_version/addons/search/?platform=$platform&sort=$sort&type=$type&app=$app&appversion=$appversion&page_size=$page_size&page=";

function search-pages--download() {

    function wget_page {
        
        
        wget_uri="$base_uri$page";

        wget -q "$wget_uri" --no-verbose -O "$page.json"

        average_daily_users="$(jq ".results[].average_daily_users" "$page.json" | tail -n 1)"
        
    }

   
    page="1";
    echo "Downloading search page: 1 (to get the number of total pages)"
    wget_page

    page_count="$(jq ".page_count" "$page.json")";

    function pv__message {

        regular_message="Download $pv__size pages from: ${base_uri//page=/page=\$page}"

    }
    
    if [ "$debug" = true ]; then

        pv__size="2";
        pv__discard=false;
        pv__message
        echo "$regular_message"
        
    elif [ "$minimum_average_daily_users" -gt "0" ]; then
        
        pv__discard=true;
        echo "Cannot show a progress counter since minimum_average_daily_users is set to more than 0 in freeamo.conf."
        extra_text="max"
        
    else

        pv__size="$page_count"
        pv__discard=false;
        pv__message
        echo "$regular_message"

    fi

    source "$run_path/src/main_functions"
    
    while [ "$page" -lt "$page_count" ]; do
        
        ((++page))
        
        if ( [ "$debug" = true ] && [ "$page" -le "3" ] || [ "$debug" = false ] ) && ( [ "$average_daily_users" -ge "$minimum_average_daily_users" ] ); then

            source "$run_path/src/main_functions"
            echo "Downloading search page: $page of $extra_text $page_count"
            wget_page

        else

            exit 0
            
        fi


        if [ ! "$pv__discard" = true ]; then

            if ! $debug; then echo "foo"; fi # for pv

        fi

    done | pv__progress
    
}

function search-pages--filter() {

    pv__size=$(find . -not -name . | wc -l)
    output_file="$output_file_1";
    source "$run_path/src/main_functions"

    echo "Filter $pv__size search pages from EULA and native messaging."
    
    rm -f "$output_file_1"
    
    # Sort by file modification time stamp for the downloaded JSON files
    find . -type f  | sort -n | while read -r file; do
        
        filename="$(echo $(basename "$file"))";
        entry="0"; # Should not be 1

       
        # Should not be limited to 3 in --debug mode.
        for id in $(jq ".results[].average_daily_users" "$file"); do

            source "$run_path/src/main_functions"
            amo_variables
            
            essential_addon
            if [ "$essential_addon" = true ]; then
                
                echo -e "$filename\\t$entry\\t$slug" >> "$output_file_1"
                
            fi
            
            ((++entry))

        done

        if ! $debug; then echo "foo"; fi # for pv

    done | pv__progress

}