From 85cea3c6634e9af141612a406ba15ebca37e20db Mon Sep 17 00:00:00 2001 From: david Date: Mon, 23 Jul 2018 19:45:17 +0200 Subject: FreeAMO: Improved terminal output --- subprojects/freeamo/src/get-licenses | 32 ++++++++++++++++++++++++++++---- subprojects/freeamo/src/make-wiki | 13 ++++++++++++- subprojects/freeamo/src/search-pages | 22 ++++++++++++++-------- 3 files changed, 54 insertions(+), 13 deletions(-) (limited to 'subprojects/freeamo') diff --git a/subprojects/freeamo/src/get-licenses b/subprojects/freeamo/src/get-licenses index 79b6da4..4298962 100644 --- a/subprojects/freeamo/src/get-licenses +++ b/subprojects/freeamo/src/get-licenses @@ -23,10 +23,23 @@ cd "build/json/search-pages" || exit function get-licenses--download { # API: Add "last_license" to the search pages - https://github.com/mozilla/addons-server/issues/8945 - + output_file="$output_file_1"; - pv__size="$(wc -l < "$output_file")" + + if [ "$debug" = true ]; then + + pv__size=$(cat "$output_file" | head -n 3 | wc -l) + + else + + pv__size="$(wc -l < "$output_file")" + + fi + source "$run_path/src/main_functions" + + echo "Download $pv__size current_versions files that contain license information." + while IFS= read -r table; do @@ -44,7 +57,7 @@ function get-licenses--download { # echo -e "#================== #Entry $line (of $total): $name\n" - # Do now download the same license files over and over + # Do not download the same license files over and over wget -nc -q "https://addons.mozilla.org/api/v3/addons/addon/$slug/versions/$current_version__id/" -O "$run_path/build/json/current_versions/$current_version__id.json" fi @@ -58,10 +71,21 @@ function get-licenses--download { function get-licenses--filter { output_file="$output_file_1"; - pv__size="$(wc -l < "$output_file")" + + if [ "$debug" = true ]; then + + pv__size=$(cat "$output_file" | head -n 3 | wc -l) + + else + + pv__size="$(wc -l < "$output_file")" + + fi + source "$run_path/src/main_functions" rm -f "$run_path/build/index-search-pages--free-licenses.txt" + echo "Filter free licenses for $pv__size entries." while IFS= read -r table; do diff --git a/subprojects/freeamo/src/make-wiki b/subprojects/freeamo/src/make-wiki index 118f257..531116d 100644 --- a/subprojects/freeamo/src/make-wiki +++ b/subprojects/freeamo/src/make-wiki @@ -31,10 +31,21 @@ if [ "$minimum_average_daily_users" == "100000" ]; then fi -pv__size=$(find . -not -name . | wc -l) +if [ "$debug" = true ]; then + + pv__size=$(find . -not -name . | head -n 3 | wc -l) + +else + + pv__size=$(find . -not -name . | wc -l) + +fi + output_file="$output_file_2"; source "$run_path/src/main_functions" +echo "Make $pv__size .wiki pages" + while IFS= read -r table; do ((++line)) diff --git a/subprojects/freeamo/src/search-pages b/subprojects/freeamo/src/search-pages index 3e894dc..bc4e14b 100644 --- a/subprojects/freeamo/src/search-pages +++ b/subprojects/freeamo/src/search-pages @@ -19,9 +19,7 @@ cd "$run_path/build/json/search-pages" || exit base_uri="https://addons.mozilla.org/api/v$api_version/addons/search/?platform=$platform&sort=$sort&type=$type&app=$app&appversion=$appversion&page_size=$page_size&page="; -echo "$base_uri" function search-pages--download() { - function wget_page { @@ -33,15 +31,14 @@ function search-pages--download() { average_daily_users="$(jq ".results[].average_daily_users" "$page.json" | tail -n 1)" } - + + page="1"; wget_page page_count="$(jq ".page_count" "$page.json")"; pv__size="$page_count" - source "$run_path/src/main_functions" - - + if [ "$debug" = true ]; then pv__size="2"; @@ -56,6 +53,14 @@ function search-pages--download() { pv__size="$page_count" fi + + + + source "$run_path/src/main_functions" + + + + echo "Download $pv__size pages from: ${base_uri//page=/page=\$page}" while [ "$page" -lt "$page_count" ]; do @@ -82,17 +87,18 @@ function search-pages--filter() { pv__size=$(find . -not -name . | wc -l) output_file="$output_file_1"; source "$run_path/src/main_functions" + + echo "Filter $pv__size search pages from EULA and native messaging." rm -f "$output_file_1" # Sort by file modification time stamp for the downloaded JSON files find . -type f | sort -n | while read -r file; do - filename="$(echo $(basename "$file"))"; - entry="0"; # Should not be 1 + # Should not be limited to 3 in --debug mode. for id in $(jq ".results[].average_daily_users" "$file"); do -- cgit v1.2.3