aboutsummaryrefslogtreecommitdiff
path: root/subprojects/freeamo/src/search-pages
blob: 3e894dc64d164c4a48da1307dd2d1f0a26eeb0a7 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
#!/usr/bin/env bash
#    This file is part of FreeAMO
#    Copyright (C) 2017 David Hedlund
#
#    FreeAMO is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    FreeAMO is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

cd "$run_path/build/json/search-pages" || exit

base_uri="https://addons.mozilla.org/api/v$api_version/addons/search/?platform=$platform&sort=$sort&type=$type&app=$app&appversion=$appversion&page_size=$page_size&page=";

echo "$base_uri"
function search-pages--download() {
    

    function wget_page {
        
        
        wget_uri="$base_uri$page";

        wget -q "$wget_uri" --no-verbose -O "$page.json"

        average_daily_users="$(jq ".results[].average_daily_users" "$page.json" | tail -n 1)"
        
    }
    
    page="1";
    wget_page

    page_count="$(jq ".page_count" "$page.json")";
    pv__size="$page_count"
    source "$run_path/src/main_functions"
    
    
    if [ "$debug" = true ]; then

        pv__size="2";

    elif [ "$minimum_average_daily_users" -gt "0" ]; then
        
        pv__discard=true;
        echo "Cannot show a progress counter since minimum_average_daily_users is set to more than 0 in freeamo.conf."
        
    else

        pv__size="$page_count"

    fi
    
    while [ "$page" -lt "$page_count" ]; do

        ((++page))
        
        if ( [ "$debug" = true ] && [ "$page" -le "3" ] || [ "$debug" = false ] ) && ( [ "$average_daily_users" -ge "$minimum_average_daily_users" ] ); then

            echo "$page"
            wget_page

        else

            exit 0
            
        fi
        
        echo "foo" # Must be here to work with pv
    done | pv__progress
    
}

function search-pages--filter() {

    pv__size=$(find . -not -name . | wc -l)
    output_file="$output_file_1";
    source "$run_path/src/main_functions"
    
    rm -f "$output_file_1"
    
    # Sort by file modification time stamp for the downloaded JSON files
    find . -type f  | sort -n | while read -r file; do

        
        filename="$(echo $(basename "$file"))";

        entry="0"; # Should not be 1

        # Should not be limited to 3 in --debug mode.
        for id in $(jq ".results[].average_daily_users" "$file"); do

            source "$run_path/src/main_functions"
            amo_variables
            
            essential_addon
            if [ "$essential_addon" = true ]; then
                
                echo -e "$filename\\t$entry\\t$slug" >> "$output_file_1"
                
            fi
            
            ((++entry))

        done

        echo "$filename"


        echo "foo" # Must be here to work with pv
    done | pv__progress

}