diff options
author | david <public@beloved.name> | 2018-07-20 12:45:24 +0200 |
---|---|---|
committer | david <public@beloved.name> | 2018-07-20 12:45:24 +0200 |
commit | 29b7728d89b8c64220ffd06e268fdaa35dac7f45 (patch) | |
tree | 4284463133589f582fbd56759a5ab7ec088138f1 | |
parent | e82f2396032865c0e54a3fb58832633c5b532e74 (diff) |
FreeAMO. General bug fixes. Fixed the description fields for .wiki pages
-rw-r--r-- | subprojects/freeamo/default.conf | 20 | ||||
-rwxr-xr-x | subprojects/freeamo/freeamo | 77 | ||||
-rw-r--r-- | subprojects/freeamo/freeamo-org.conf | 19 | ||||
-rw-r--r-- | subprojects/freeamo/freeamo.conf | 17 | ||||
-rw-r--r-- | subprojects/freeamo/src/get-licenses | 21 | ||||
-rw-r--r-- | subprojects/freeamo/src/main_functions | 184 | ||||
-rw-r--r-- | subprojects/freeamo/src/make-wiki | 103 | ||||
-rw-r--r-- | subprojects/freeamo/src/search-pages | 43 |
8 files changed, 332 insertions, 152 deletions
diff --git a/subprojects/freeamo/default.conf b/subprojects/freeamo/default.conf new file mode 100644 index 0000000..bd4d01d --- /dev/null +++ b/subprojects/freeamo/default.conf @@ -0,0 +1,20 @@ +# Read the API reference documentation for parameter values: +# https://addons-server.readthedocs.io/en/latest/topics/api/addons.html + +# Mozilla API +api_version= + +# Mozilla API parameters: Add-ons -> Search +app="" +appversion= +page_size= +platform="" +sort="" +type="" + +# Mozilla API parameters: Add-ons -> Detail +minimum_average_daily_users= + +# Extra +notes="" +mediawikibot__bot_password__Username="" # Not mandatory diff --git a/subprojects/freeamo/freeamo b/subprojects/freeamo/freeamo index dec2d5c..bd1df0e 100755 --- a/subprojects/freeamo/freeamo +++ b/subprojects/freeamo/freeamo @@ -15,20 +15,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -source freeamo.conf - scriptsrc=$(readlink -f -- "${BASH_SOURCE[0]}") run_path=$(dirname "${scriptsrc}" || echo .) filename=$(basename "$0"); +source "$run_path/src/main_functions" +source "$run_path/freeamo.conf" + + +#source "$run_path/src/err" + output_file_1="$run_path/build/index-search-pages.txt"; output_file_2="$run_path/build/index-search-pages--free-licenses.txt" -# source "$run_path/build/err" - # Evaluate debug mode if [ "$1" == "--debug" ] || [ "$2" == "--debug" ]; then - + debug=true debug_arg="--debug"; @@ -55,14 +57,31 @@ if [[ $exit == true ]]; then exit 1; fi case "$1" in - ""|--debug) + ""|--debug|--normal-mode) - $0 --disclaimer "$debug_arg" - $0 --remove-build "$debug_arg" - $0 --search-pages "$debug_arg" - $0 --get-licenses "$debug_arg" - $0 --make-directory.fsf.org-wiki "$debug_arg" + freeamo_command="$0 --conf-note $debug_arg :" + echo "$freeamo_command"; $freeamo_command + + freeamo_command="$0 --remove-build $debug_arg :" + echo "$freeamo_command"; $freeamo_command + freeamo_command="$0 --conf-copy $debug_arg :" + echo "$freeamo_command"; $freeamo_command + + freeamo_command="$0 --search-pages $debug_arg :" + echo "$freeamo_command"; $freeamo_command + + freeamo_command="$0 --get-licenses $debug_arg :" + echo "$freeamo_command"; $freeamo_command + + if [ ! "$1" == "--normal-mode" ]; then + + freeamo_command="$0 --make-directory.fsf.org-wiki $debug_arg :" + echo "$freeamo_command"; $freeamo_command + + fi + + ;; -help) @@ -70,10 +89,13 @@ case "$1" in [ "$1" = "" ] && echo "Usage: $filename [--option] [--debug] OPTIONS - $0 --disclaimer +f.org-wiki (they are used by Free Software Directory developers) + $0 --conf-note Display which icecat version the add-on will be compatible with. $0 --remove-build Remove the build directory if it exists. + $0 --conf-copy + Copy the conf file to the build directory. $0 --search-pages Downloads the most popular WebExtensions from https://addons.mozilla.org/en-US/firefox/search/... $0 --get-licenses @@ -100,23 +122,35 @@ Disabled options due to limited development time ;; - --disclaimer) - - echo "$notes" + --conf-note) + echo "$notes" + ;; - + --remove-build) rm -fr "$run_path/build" ;; + + --conf-copy) + + mkdir -p "$run_path/build/" + echo -e "# [This file is a copy used as a receipt to declare the settings that were used to generate the files.]\\n" > "$run_path/build/freeamo.conf" + cat "$run_path/freeamo.conf" >> "$run_path/build/freeamo.conf" + sed -i "/^appversion=/ s/$/ # $appversion at the time/" "$run_path/build/freeamo.conf" + + ;; --search-pages) # This option is named after named after https://addons.mozilla.org/en-US/firefox/search/ - $0 --search-pages--download "$debug_arg" - $0 --search-pages--filter "$debug_arg" + freeamo_command="$0 --search-pages--download $debug_arg :" + echo " ↳$freeamo_command"; $freeamo_command + + freeamo_command="$0 --search-pages--filter $debug_arg :" + echo " ↳$freeamo_command"; $freeamo_command ;; @@ -138,8 +172,11 @@ Disabled options due to limited development time --get-licenses) - $0 --get-licenses--download "$debug_arg" - $0 --get-licenses--filter "$debug_arg" + freeamo_command="$0 --get-licenses--download $debug_arg :" + echo " ↳$freeamo_command"; $freeamo_command + + freeamo_command="$0 --get-licenses--filter $debug_arg :" + echo " ↳$freeamo_command"; $freeamo_command ;; diff --git a/subprojects/freeamo/freeamo-org.conf b/subprojects/freeamo/freeamo-org.conf new file mode 100644 index 0000000..74ea564 --- /dev/null +++ b/subprojects/freeamo/freeamo-org.conf @@ -0,0 +1,19 @@ +# Read the API reference documentation for parameter values: +# https://addons-server.readthedocs.io/en/latest/topics/api/addons.html + +# Mozilla API +api_version=3 + +# Mozilla API parameters: Add-ons -> Search +app="firefox" +appversion=$(debranded_package="icecat"; latest_debranded_trisquel_version); +page_size=50 +platform="linux" +sort="users" +type="extension" + +# Mozilla API parameters: Add-ons -> Detail +minimum_average_daily_users=0 + +# Extra +notes="Extensions for the latest Trisquel GNU/Linux icecat package version." diff --git a/subprojects/freeamo/freeamo.conf b/subprojects/freeamo/freeamo.conf index fd133e7..910871f 100644 --- a/subprojects/freeamo/freeamo.conf +++ b/subprojects/freeamo/freeamo.conf @@ -1,13 +1,20 @@ +# Read the API reference documentation for parameter values: +# https://addons-server.readthedocs.io/en/latest/topics/api/addons.html + +# Mozilla API +api_version=3 + # Mozilla API parameters: Add-ons -> Search app="firefox" -appversion="$(wget -qO- "http://archive.trisquel.info/trisquel/pool/main/i/icecat/?C=N;O=D" | grep icecat_ | head -n 1 | sed "s|\"|\\n|g; s|-|\\n|g" | grep icecat_ | sed "s|icecat_||" | head -n 1)" # Latest icecat version for Trisquel -page_size=50 # page_size=50 -- this is the maximum accepted number, it makes the download much faster: 20 seconds for 10 x page_size=50, 35 seconds for 20 x page_size=25 (default) +appversion=60 # Earliest IceCat version (not released yet) +page_size=50 platform="linux" -sort="users" # sort=users. sort=name not working yet - https://github.com/mozilla/addons-server/issues/7600 +sort="users" type="extension" # Mozilla API parameters: Add-ons -> Detail -minimum_average_daily_users="10000" +minimum_average_daily_users=100000 # Extra -notes="WebExtensions compatible with the latest icecat package version for the latest stable Trisquel GNU/Linux distribution. GNU IceCat is based on Firefox Extended Support Release (ESR)." +notes="Extensions for the latest Trisquel GNU/Linux icecat package version." +mediawikibot__bot_password__Username="wikisysbot" diff --git a/subprojects/freeamo/src/get-licenses b/subprojects/freeamo/src/get-licenses index c7e5559..79b6da4 100644 --- a/subprojects/freeamo/src/get-licenses +++ b/subprojects/freeamo/src/get-licenses @@ -18,12 +18,11 @@ # API: Please adopt SPDX - https://github.com/mozilla/addons-server/issues/8706 # Allow GNU [L]GPL x.x or later - https://github.com/mozilla/addons-server/issues/8707 -cd "build/json/search-pages" +cd "build/json/search-pages" || exit function get-licenses--download { - echo -e "\\nDownloading license pages:" - + # API: Add "last_license" to the search pages - https://github.com/mozilla/addons-server/issues/8945 output_file="$output_file_1"; pv__size="$(wc -l < "$output_file")" @@ -35,8 +34,8 @@ function get-licenses--download { if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then - file=$(echo $table | awk '{print $1}'); - entry=$(echo $table | awk '{print $2}'); + file=$(echo "$table" | awk '{print $1}'); + entry=$(echo "$table" | awk '{print $2}'); source "$run_path/src/main_functions" amo_variables @@ -52,14 +51,12 @@ function get-licenses--download { echo "foo" # Must be here to work with pv - done < "$output_file_1" | progress + done < "$output_file_1" | pv__progress } function get-licenses--filter { - echo -e "\\nFilter license pages:" - output_file="$output_file_1"; pv__size="$(wc -l < "$output_file")" source "$run_path/src/main_functions" @@ -72,8 +69,8 @@ function get-licenses--filter { if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then - file=$(echo $table | awk '{print $1}'); - entry=$(echo $table | awk '{print $2}'); + file=$(echo "$table" | awk '{print $1}'); + entry=$(echo "$table" | awk '{print $2}'); source "$run_path/src/main_functions" amo_variables @@ -101,7 +98,7 @@ function get-licenses--filter { # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files. # echo "$name: $license__amo_full_name" - echo "$file $entry" >> "$output_file_2" + echo "$file $entry $slug" >> "$output_file_2" freedom_status="free"; @@ -120,6 +117,6 @@ function get-licenses--filter { echo "foo" # Must be here to work with pv - done < "$output_file_1" | progress + done < "$output_file_1" | pv__progress } diff --git a/subprojects/freeamo/src/main_functions b/subprojects/freeamo/src/main_functions index a96ada3..7a66cff 100644 --- a/subprojects/freeamo/src/main_functions +++ b/subprojects/freeamo/src/main_functions @@ -15,6 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +function latest_debranded_trisquel_version { + + # All debranded programs starts with "i" so all of them are in Trisquels "/i/" directory + wget -qO- "http://archive.trisquel.info/trisquel/pool/main/i/$debranded_package/?C=N;O=D" | sed "s|\"|\\n|g; s|-|\\n|g" | grep "$debranded_package"_ | head -n 1 | sed "s|"$debranded_package"_||" + +} + function check_files { if [ ! -f "$run_path/FreeAMO-$check_type" ]; then @@ -26,7 +33,7 @@ function check_files { # Make sure all links are using SSL - if grep -q "http://addons.mozilla.org/" $run_path/FreeAMO-$check_type; then + if grep -q "http://addons.mozilla.org/" "$run_path/FreeAMO-$check_type"; then echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)." echo "Exiting" && exit @@ -52,22 +59,82 @@ function line_status { } +# Begin locale code + +function locale_description { + + tmp__enUS=$(jq ".results[$entry].$locale_description" "$file" | sed "s|-||;" | jq .enUS | sed "s|^\"||; s|\"$||;" ) + tmp__enGB=$(jq ".results[$entry].$locale_description" "$file" | sed "s|-||;" | jq .enGB | sed "s|^\"||; s|\"$||;" ) + if [ "$tmp__enUS" != "null" ]; then + + if [ "$locale_description" == "name" ]; then + + name="$tmp__enUS"; + + elif [ "$locale_description" == "summary" ]; then + + summary="$tmp__enUS"; + + elif [ "$locale_description" == "description" ]; then + + description="$tmp__enUS"; + + fi + + + elif [ "$tmp__enGB" != "null" ]; then + + if [ "$locale_description" == "name" ]; then + + name="$tmp__enGB"; + + elif [ "$locale_description" == "summary" ]; then + + summary="$tmp__enGB"; + + elif [ "$locale_description" == "description" ]; then + + description="$tmp__enGB"; + + fi + + else + + tmp=$(jq ".results[$entry].$locale_description[]" "$file" | sed "s|-||;" | jq . | sed "s|^\"||; s|\"$||;"); # Use the first avalible summary if English is not avaliable + + if [ "$locale_description" == "name" ]; then + + name="$tmp"; + + elif [ "$locale_description" == "summary" ]; then + + summary="$tmp"; + + elif [ "$locale_description" == "description" ]; then + + description="$tmp"; + + fi + + + fi + +} + +# End locale code + + function amo_variables() { id="$(jq ".results[$entry].id" "$file")"; average_daily_users="$(jq ".results[$entry].average_daily_users" "$file")"; has_eula="$(jq ".results[$entry].has_eula" "$file")"; slug="$(jq ".results[$entry].slug" "$file" | sed "s|^\"||; s|\"$||;")"; current_version__id=$(jq ".results[$entry].current_version.id" "$file"); - - # Names - name__enUS=$(jq ".results[$entry].name" "$file" | sed "s|-||;" | jq .enUS | sed "s|^\"||; s|\"$||;" ) - name__enGB=$(jq ".results[$entry].name" "$file" | sed "s|-||;" | jq .enGB | sed "s|^\"||; s|\"$||;" ) - if [ "$name__enUS" != "null" ]; then name="$name__enUS"; - elif [ "$name__enGB" != "null" ]; then name="$name__enGB"; - else name=$(jq ".results[$entry].name[]" "$file" | sed "s|-||;" | jq . | sed "s|^\"||; s|\"$||;"); # Use the first avalible name if English is not avaliable - fi - + source "$run_path/src/main_functions" + locale_description="name" + locale_description + if jq .results[$entry].current_version.files[0].permissions "$file" | grep -q "nativeMessaging"; then nativeMessaging__pass=false; @@ -102,60 +169,77 @@ function essential_addon { #function hard_to_evaluate_addon { - # curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging - +# curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging + + +# API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668 +# The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies. + +# Old solution: - # API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668 - # The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies. - - # Old solution: - - # 1: Add this to FreeAMO-dependencies: - #belgium-eid eid-mw - - # 2: Uncomment this code: - # depends_on_external_program=false; - # while IFS= read -r dependencies; do - # - # dependency_slug=$(echo $dependencies | awk '{print $1}'); - # dependency=$(echo "$dependencies" | cut -f 2); - # - # if [ "$dependency_slug" == "$slug" ]; then - # - # depends_on_external_program=true; - # - # fi - # - # done < $run_path/FreeAMO-dependencies +# 1: Add this to FreeAMO-dependencies: +#belgium-eid eid-mw + +# 2: Uncomment this code: +# depends_on_external_program=false; +# while IFS= read -r dependencies; do +# +# dependency_slug=$(echo $dependencies | awk '{print $1}'); +# dependency=$(echo "$dependencies" | cut -f 2); +# +# if [ "$dependency_slug" == "$slug" ]; then +# +# depends_on_external_program=true; +# +# fi +# +# done < $run_path/FreeAMO-dependencies #} -function progress { - - (>&2 echo -en "\r"; pv --progress --line-mode --size $pv__size --eta --timer) > /dev/null - +function pv__progress { + + if [ ! "$pv__discard" = true ]; then + + (pv --progress --line-mode --size "$pv__size" --timer) > /dev/null + + fi + } -function html_to_mediawiki { +function format_description__mixed_html_to_mediawiki { sed " # Remove Mozilla tracking links -s|https://outgoing.prod.mozaws.net/v1/.\{65\}||g; s|%3A|:|g; s|%23|#|g +s|https://outgoing.prod.mozaws.net/v1/.\\{65\\}||g; s|%3A|:|g; s|%23|#|g -# Fix general links -s|<a rel=\\\\\\\"nofollow\\\\\\\" href=\\\\\\\"|[|g -s|\\\\\\\">| |g -s|</a>|]|g +# Remove reduntant backslashes. Pandoc will convert quotes regardless. +s|\\\\\\\"|\\\"|g -# Fix line breaks -s|\\\\n|\\n|g -# Fix quotes -s|\\\\\\\"|\\\"|g +" + +} + +function format_description__after_pandoc { + + sed " + +### Fix formatting that cannot be fixed by Pandoc since it's not HTML + +# Fix line breaks. +s|\\\\n|\\n|g -# Fix bullets, many developers failed to use <li> needed to make line break -s|\\n- |\\n* |g +# Remove \t +s|\\\\t||g " + # Change local links to external links + +} + +function format_description { + + format_description__mixed_html_to_mediawiki | pandoc -f html -t mediawiki | format_description__after_pandoc } diff --git a/subprojects/freeamo/src/make-wiki b/subprojects/freeamo/src/make-wiki index 19e782f..118f257 100644 --- a/subprojects/freeamo/src/make-wiki +++ b/subprojects/freeamo/src/make-wiki @@ -15,20 +15,23 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -cd "build/json/search-pages" +cd "build/json/search-pages" || exit -echo -e "\\nGenerating .wiki files for https://directory.fsf.org/wiki/:" +if [ "$minimum_average_daily_users" == "100000" ]; then -echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\" + echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\" ! WebExtension ! Description ! Claimed license ! Users ! Rating -! Updated" > "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki" +! Updated +! Featured" > "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki" +fi -pv__size=$(ls . | wc -l) + +pv__size=$(find . -not -name . | wc -l) output_file="$output_file_2"; source "$run_path/src/main_functions" @@ -38,8 +41,8 @@ while IFS= read -r table; do if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then - file=$(echo $table | awk '{print $1}'); - entry=$(echo $table | awk '{print $2}'); + file=$(echo "$table" | awk '{print $1}'); + entry=$(echo "$table" | awk '{print $2}'); source "$run_path/src/main_functions" amo_variables @@ -55,27 +58,35 @@ while IFS= read -r table; do elif [ "$license__amo_full_name" == "Mozilla Public License Version 1.1" ]; then license__fsd_short_name="MPLv1.1"; fi + # Start description code + source "$run_path/src/main_functions" - full_description=$(jq .results[$entry].description $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | html_to_mediawiki) - short_description=$(jq .results[$entry].summary $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | html_to_mediawiki) + locale_description="summary" + locale_description + locale_description="description" + locale_description + short_description=$(echo "$summary" | format_description) + full_description=$(echo "$description" | format_description) + + # End description code - homepage_url=$(jq .results[$entry].homepage $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - version=$(jq .results[$entry].current_version.version $file | sed "s|^\"||; s|\"$||;"); - version_date=$(jq .results[$entry].last_updated $file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g"); - version_id=$(jq .results[$entry].current_version.id $file); - license_copyright=$(jq .results[$entry].authors[].name $file | sed "s|^\"||; s|\"$||;"); - bayesian_average=$(jq .results[$entry].ratings.bayesian_average $file); - bayesian_average__simple=$(printf "%.1f" "$(echo "$bayesian_average" | sed "s|\.|,|")"); + # The homepage field is mandatory so let's remove "null" from it because we don't need to evaluat it + homepage_url=$(jq .results[$entry].homepage "$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|null||"); + license_copyright=$(jq .results[$entry].authors[].name "$file" | sed "s|^\"||; s|\"$||;"); + bayesian_average=$(jq .results[$entry].ratings.bayesian_average "$file"); + bayesian_average__simple=$(printf "%.1f" "${bayesian_average//./,}"); + is_featured=$(jq .results[$entry].is_featured "$file"); + last_review_date=$(date +"%Y/%m/%d"); - last_review_by="wikisysbot"; - submitted_by="wikisysbot"; + last_review_by="$mediawikibot__bot_password__Username"; + submitted_by="$mediawikibot__bot_password__Username"; - if [[ "$(jq .results[$entry].current_version.compatibility.seamonkey $file)" != "null" ]]; then + if [[ "$(jq .results[$entry].current_version.compatibility.seamonkey "$file")" != "null" ]]; then set_seamonkey="Iceape,"; fi - if [[ "$(jq .results[$entry].current_version.compatibility.firefox $file)" != "null" ]]; then + if [[ "$(jq .results[$entry].current_version.compatibility.firefox "$file")" != "null" ]]; then set_icecat="IceCat,"; fi @@ -83,7 +94,7 @@ while IFS= read -r table; do # set_icecatmobile="IceCatMobile,"; # fi - if [[ "$(jq .results[$entry].current_version.compatibility.thunderbird $file)" != "null" ]]; then + if [[ "$(jq .results[$entry].current_version.compatibility.thunderbird "$file")" != "null" ]]; then set_thunderbird="Icedove,"; fi @@ -97,7 +108,7 @@ while IFS= read -r table; do if [[ $github_true != "" ]]; then # Always use https, and remove anchors - homepage_url__for__vcs_checkout_command="https://github.com$(echo $github_true | sed "s|https://github.com||; s|http://github.com||; s|#|\n|;" | head -n 1)"; + homepage_url__for__vcs_checkout_command="https://github.com$(echo "$github_true" | sed "s|https://github.com||; s|http://github.com||; s|#|\\n|;" | head -n 1)"; # Remove trailing slash homepage_url__for__vcs_checkout_command="${homepage_url__for__vcs_checkout_command%/}" vcs_checkout_command="git clone $homepage_url__for__vcs_checkout_command.git"; @@ -114,11 +125,12 @@ while IFS= read -r table; do fi - support_url=$(jq .results[$entry].support_url $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + support_url=$(jq .results[$entry].support_url "$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + last_updated=$(jq .results[$entry].last_updated $file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g"); # List JSON structure: cat json/$file | js '.' # XML value "homepage" is not always set in AMO API, but the XML require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically. - echo "{{Entry" > $run_path/build/wiki/$slug.wiki + echo "{{Entry" > "$run_path/build/wiki/$slug.wiki" # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341 # Avoid specific versions since it's not compatible with all IceCat versions: @@ -126,20 +138,30 @@ while IFS= read -r table; do # |Version identifier=$version # |Version date=$version_date - + + # Do add these fields since we refer to the download page in the `|Version download` field that lists compatible versions + # |Version identifier=52.6.0 + # |Version date=2018/02/26 + # |Version status=stable + echo "|Name=$name |Short description=$short_description |Full description=$full_description |Homepage URL=$homepage_url |Extension of=$extension_of |VCS checkout command=$vcs_checkout_command +|Documentation note=Extra data (https://addons.mozilla.org/api/v$api_version/addons/addon/$slug/): +* Users: $average_daily_users +* Rating: $bayesian_average__simple +* Updated: $last_updated +* Featured: $is_featured |Version download=https://addons.mozilla.org/en-US/firefox/addon/$slug/versions/ |Last review by=$last_review_by |Last review date=$last_review_date |Submitted by=$submitted_by -|Submitted date=2018/06/20 +|Submitted date= |Is GNU=$is_gnu -}}" >> $run_path/build/wiki/$slug.wiki +}}" >> "$run_path/build/wiki/$slug.wiki" @@ -149,9 +171,9 @@ while IFS= read -r table; do |License copyright=$license_copyright |License note=\"License: $license__amo_full_name\" listed at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/ -}}" >> $run_path/build/wiki/$slug.wiki +}}" >> "$run_path/build/wiki/$slug.wiki" - if [[ "$support_url" != "" ]]; + if [ "$support_url" != "null" ]; then echo "{{Resource @@ -162,19 +184,30 @@ at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/ fi - + if [ "$minimum_average_daily_users" == "100000" ]; then + echo "|- -| [[$name]] +| [[$slug|$name]] | $short_description | [[License:$license__fsd_short_name|$license__fsd_short_name]] | $average_daily_users | $bayesian_average__simple -| $version_date" >> "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki" +| $last_updated +| $is_featured" >> "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki" + + fi + + fi - echo "foo" # Must be here to work with pv -done < "$output_file_2" | progress + echo "foo" # Must be here to work with pv +done < "$output_file_2" | pv__progress # End template -echo "|}" >> "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki" + +if [ "$minimum_average_daily_users" == "100000" ]; then + + echo "|}" >> "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki" + +fi diff --git a/subprojects/freeamo/src/search-pages b/subprojects/freeamo/src/search-pages index a723601..3e894dc 100644 --- a/subprojects/freeamo/src/search-pages +++ b/subprojects/freeamo/src/search-pages @@ -17,22 +17,11 @@ cd "$run_path/build/json/search-pages" || exit -base_uri="https://addons.mozilla.org/api/v3/addons/search/?platform=$platform&sort=$sort&type=$type&app=$app&appversion=$appversion&page_size=$page_size&page="; +base_uri="https://addons.mozilla.org/api/v$api_version/addons/search/?platform=$platform&sort=$sort&type=$type&app=$app&appversion=$appversion&page_size=$page_size&page="; +echo "$base_uri" function search-pages--download() { - echo -e "\\nDownloading search pages from $base_uri :" | sed "s|page=|page=\$page|" - - echo "This file is used to describe the settings that were used to generate the files. - -build/json/search-pages/ -$wget_uri" > "$run_path/build/declared_settings.txt" - # indicate that the page page variable are the page files - sed -i "s|page=1|page=\$page|g" "$run_path/build/declared_settings.txt" - - - - ##################################################### function wget_page { @@ -57,12 +46,16 @@ $wget_uri" > "$run_path/build/declared_settings.txt" pv__size="2"; + elif [ "$minimum_average_daily_users" -gt "0" ]; then + + pv__discard=true; + echo "Cannot show a progress counter since minimum_average_daily_users is set to more than 0 in freeamo.conf." + else pv__size="$page_count" fi - while [ "$page" -lt "$page_count" ]; do @@ -80,28 +73,17 @@ $wget_uri" > "$run_path/build/declared_settings.txt" fi echo "foo" # Must be here to work with pv - done | progress + done | pv__progress } function search-pages--filter() { - - echo -e "\\nFilter search pages -- no EULA, no native messaging, and minimum average daily users ($minimum_average_daily_users):" - - # source "$run_path/src/main_functions" - # amo_variables - echo " -build/index-search-pages.txt -has_eula=false -nativeMessaging__pass=true -minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/declared_settings.txt" - - pv__size=$(ls . | wc -l) + pv__size=$(find . -not -name . | wc -l) output_file="$output_file_1"; source "$run_path/src/main_functions" - rm -f $output_file_1 + rm -f "$output_file_1" # Sort by file modification time stamp for the downloaded JSON files find . -type f | sort -n | while read -r file; do @@ -111,6 +93,7 @@ minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/de entry="0"; # Should not be 1 + # Should not be limited to 3 in --debug mode. for id in $(jq ".results[].average_daily_users" "$file"); do source "$run_path/src/main_functions" @@ -119,7 +102,7 @@ minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/de essential_addon if [ "$essential_addon" = true ]; then - echo -e "$filename\\t$entry" >> "$output_file_1" + echo -e "$filename\\t$entry\\t$slug" >> "$output_file_1" fi @@ -131,6 +114,6 @@ minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/de echo "foo" # Must be here to work with pv - done | progress + done | pv__progress } |