diff options
Diffstat (limited to 'subprojects/freeamo/src/search-pages')
| -rw-r--r-- | subprojects/freeamo/src/search-pages | 37 |
1 files changed, 25 insertions, 12 deletions
diff --git a/subprojects/freeamo/src/search-pages b/subprojects/freeamo/src/search-pages index bc4e14b..45ed716 100644 --- a/subprojects/freeamo/src/search-pages +++ b/subprojects/freeamo/src/search-pages @@ -34,15 +34,24 @@ function search-pages--download() { page="1"; + echo "Downloading search page: 1 (to get the number of total pages)" wget_page page_count="$(jq ".page_count" "$page.json")"; - pv__size="$page_count" + function pv__message { + + regular_message="Download $pv__size pages from: ${base_uri//page=/page=\$page}" + + } + if [ "$debug" = true ]; then pv__size="2"; - + pv__discard=false; + pv__message + echo "$regular_message" + elif [ "$minimum_average_daily_users" -gt "0" ]; then pv__discard=true; @@ -51,24 +60,22 @@ function search-pages--download() { else pv__size="$page_count" + pv__discard=false; + pv__message + echo "$regular_message" fi - - source "$run_path/src/main_functions" - - - echo "Download $pv__size pages from: ${base_uri//page=/page=\$page}" - while [ "$page" -lt "$page_count" ]; do - + ((++page)) if ( [ "$debug" = true ] && [ "$page" -le "3" ] || [ "$debug" = false ] ) && ( [ "$average_daily_users" -ge "$minimum_average_daily_users" ] ); then - echo "$page" + source "$run_path/src/main_functions" + echo "Downloading search page: $page of $page_count" wget_page else @@ -76,8 +83,14 @@ function search-pages--download() { exit 0 fi - - echo "foo" # Must be here to work with pv + + + if [ ! "$pv__discard" = true ]; then + + echo "foo" # Must be here to work with pv + + fi + done | pv__progress } |