1 files changed, 136 insertions, 0 deletions
diff --git a/subprojects/freeamo/src/search-pages b/subprojects/freeamo/src/search-pages
new file mode 100644
index 0000000..a723601
--- /dev/null
+++ b/subprojects/freeamo/src/search-pages
@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+#    This file is part of FreeAMO
+#    Copyright (C) 2017 David Hedlund
+#
+#    FreeAMO is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    FreeAMO is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+cd "$run_path/build/json/search-pages" || exit
+
+base_uri="https://addons.mozilla.org/api/v3/addons/search/?platform=$platform&sort=$sort&type=$type&app=$app&appversion=$appversion&page_size=$page_size&page=";
+
+function search-pages--download() {
+    
+    echo -e "\\nDownloading search pages from $base_uri :" | sed "s|page=|page=\$page|"
+
+    echo "This file is used to describe the settings that were used to generate the files.
+
+build/json/search-pages/
+$wget_uri" > "$run_path/build/declared_settings.txt"
+    # indicate that the page page variable are the page files
+    sed -i "s|page=1|page=\$page|g" "$run_path/build/declared_settings.txt"
+
+
+
+    #####################################################
+
+    function wget_page {
+        
+        
+        wget_uri="$base_uri$page";
+
+        wget -q "$wget_uri" --no-verbose -O "$page.json"
+
+        average_daily_users="$(jq ".results[].average_daily_users" "$page.json" | tail -n 1)"
+        
+    }
+    
+    page="1";
+    wget_page
+
+    page_count="$(jq ".page_count" "$page.json")";
+    pv__size="$page_count"
+    source "$run_path/src/main_functions"
+    
+    
+    if [ "$debug" = true ]; then
+
+        pv__size="2";
+
+    else
+
+        pv__size="$page_count"
+
+    fi
+
+    
+    while [ "$page" -lt "$page_count" ]; do
+
+        ((++page))
+        
+        if ( [ "$debug" = true ] && [ "$page" -le "3" ] || [ "$debug" = false ] ) && ( [ "$average_daily_users" -ge "$minimum_average_daily_users" ] ); then
+
+            echo "$page"
+            wget_page
+
+        else
+
+            exit 0
+            
+        fi
+        
+        echo "foo" # Must be here to work with pv
+    done | progress
+    
+}
+
+function search-pages--filter() {
+
+ 
+    echo -e "\\nFilter search pages -- no EULA, no native messaging, and minimum average daily users ($minimum_average_daily_users):"
+
+    #    source "$run_path/src/main_functions"
+    #    amo_variables
+    echo "
+build/index-search-pages.txt
+has_eula=false
+nativeMessaging__pass=true
+minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/declared_settings.txt"
+
+    pv__size=$(ls . | wc -l)
+    output_file="$output_file_1";
+    source "$run_path/src/main_functions"
+    
+    rm -f $output_file_1
+    
+    # Sort by file modification time stamp for the downloaded JSON files
+    find . -type f  | sort -n | while read -r file; do
+
+        
+        filename="$(echo $(basename "$file"))";
+
+        entry="0"; # Should not be 1
+
+        for id in $(jq ".results[].average_daily_users" "$file"); do
+
+            source "$run_path/src/main_functions"
+            amo_variables
+            
+            essential_addon
+            if [ "$essential_addon" = true ]; then
+                
+                echo -e "$filename\\t$entry" >> "$output_file_1"
+                
+            fi
+            
+            ((++entry))
+
+        done
+
+        echo "$filename"
+
+
+        echo "foo" # Must be here to work with pv
+    done | progress
+
+}