diff options
Diffstat (limited to 'subprojects/freeamo/src/search-pages')
-rw-r--r-- | subprojects/freeamo/src/search-pages | 137 |
1 files changed, 0 insertions, 137 deletions
diff --git a/subprojects/freeamo/src/search-pages b/subprojects/freeamo/src/search-pages deleted file mode 100644 index 22f293a..0000000 --- a/subprojects/freeamo/src/search-pages +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env bash -# This file is part of FreeAMO -# Copyright (C) 2017 David Hedlund -# -# FreeAMO is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# FreeAMO is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -cd "$run_path/build/json/search-pages" || exit - -base_uri="https://addons.mozilla.org/api/v$api_version/addons/search/?platform=$platform&sort=$sort&type=$type&app=$app&appversion=$appversion&page_size=$page_size&page="; - -function search-pages--download() { - - function wget_page { - - - wget_uri="$base_uri$page"; - - wget -q "$wget_uri" --no-verbose -O "$page.json" - - average_daily_users="$(jq ".results[].average_daily_users" "$page.json" | tail -n 1)" - - } - - - page="1"; - echo "Downloading search page: 1 (to get the number of total pages)" - wget_page - - page_count="$(jq ".page_count" "$page.json")"; - - function pv__message { - - regular_message="Download $pv__size pages from: ${base_uri//page=/page=\$page}" - - } - - if [ "$debug" = true ]; then - - pv__size="2"; - pv__discard=false; - pv__message - echo "$regular_message" - - elif [ "$minimum_average_daily_users" -gt "0" ]; then - - pv__discard=true; - echo "Cannot show a progress counter since minimum_average_daily_users is set to more than 0 in freeamo.conf." - extra_text="max" - - else - - pv__size="$page_count" - pv__discard=false; - pv__message - echo "$regular_message" - - fi - - source "$run_path/src/main_functions" - - while [ "$page" -lt "$page_count" ]; do - - ((++page)) - - if ( [ "$debug" = true ] && [ "$page" -le "3" ] || [ "$debug" = false ] ) && ( [ "$average_daily_users" -ge "$minimum_average_daily_users" ] ); then - - source "$run_path/src/main_functions" - echo "Downloading search page: $page of $extra_text $page_count" - wget_page - - else - - exit 0 - - fi - - - if [ ! "$pv__discard" = true ]; then - - if ! $debug; then echo "foo"; fi # for pv - - fi - - done | pv__progress - -} - -function search-pages--filter() { - - pv__size=$(find . -not -name . | wc -l) - output_file="$output_file_1"; - source "$run_path/src/main_functions" - - echo "Filter $pv__size search pages from EULA and native messaging." - - rm -f "$output_file_1" - - # Sort by file modification time stamp for the downloaded JSON files - find . -type f | sort -n | while read -r file; do - - filename="$(echo $(basename "$file"))"; - entry="0"; # Should not be 1 - - - # Should not be limited to 3 in --debug mode. - for id in $(jq ".results[].average_daily_users" "$file"); do - - source "$run_path/src/main_functions" - amo_variables - - essential_addon - if [ "$essential_addon" = true ]; then - - echo -e "$filename\\t$entry\\t$slug" >> "$output_file_1" - - fi - - ((++entry)) - - done - - if ! $debug; then echo "foo"; fi # for pv - - done | pv__progress - -} |