From b868025432f83cfe9ecd0e2a626cf312d2123fb1 Mon Sep 17 00:00:00 2001 From: david Date: Wed, 25 Jul 2018 13:32:41 +0200 Subject: Updated FreeAMO --- subprojects/freeamo/src/get-add-ons | 96 +++++++++ subprojects/freeamo/src/main_functions | 21 +- subprojects/freeamo/src/make-wiki | 12 +- subprojects/freeamo/src/make-wiki (copy) | 224 +++++++++++++++++++++ subprojects/freeamo/src/search-pages | 3 +- .../src/unmaintained/download-free-webextensions | 50 ----- 6 files changed, 345 insertions(+), 61 deletions(-) create mode 100644 subprojects/freeamo/src/get-add-ons create mode 100644 subprojects/freeamo/src/make-wiki (copy) delete mode 100644 subprojects/freeamo/src/unmaintained/download-free-webextensions (limited to 'subprojects/freeamo/src') diff --git a/subprojects/freeamo/src/get-add-ons b/subprojects/freeamo/src/get-add-ons new file mode 100644 index 0000000..6038bd6 --- /dev/null +++ b/subprojects/freeamo/src/get-add-ons @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2017 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +cd "build/json/search-pages" || exit + +function get-add-ons--download { + + + output_file="$output_file_2"; + + if [ "$debug" = true ]; then + + pv__size=$(cat "$output_file" | head -n 3 | wc -l) + + else + + pv__size="$(wc -l < "$output_file")" + + fi + + echo "Download $pv__size add-ons" + + while IFS= read -r table; do + + ((line++)) + + file=$(echo "$table" | awk '{print $1}'); + entry=$(echo "$table" | awk '{print $2}'); + + source "$run_path/src/main_functions" + cd "$run_path/build/json/search-pages" || exit + amo_variables + + cd "$run_path/build/add-ons" || exit + # Add-ons not avalible for GNU/Linux will be ignored (used to be very few though). + wget -nc -q "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc -q "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" + + echo "foo" # Must be here to work with pv + + done < "$output_file" | pv__progress + +} + + +function get-add-ons--filter { + + rm -f "$run_path/build/index-search-pages--free-licenses--license-file-status.txt" + + output_file="$output_file_2"; + + if [ "$debug" = true ]; then + + pv__size=$(cat "$output_file" | head -n 3 | wc -l) + + else + + pv__size="$(wc -l < "$output_file")" + + fi + + echo "Filter $pv__size add-ons " + + while IFS= read -r table; do + + ((line++)) + + file=$(echo "$table" | awk '{print $1}'); + entry=$(echo "$table" | awk '{print $2}'); + + source "$run_path/src/main_functions" + cd "$run_path/build/json/search-pages" || exit + amo_variables + + cd "$run_path/build/add-ons" || exit + license_file_text=$(unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -E "^COPYING$|^COPYING.txt$|^COPYING.md$|^LICENSE$|^LICENSE.txt$|^LICENSE.md$" | sed ':a;N;$!ba;s/\n/\t/g' || echo "NOLICENSEFILEFOUND"); + echo -e "$file\\t$entry\\t$slug\\taddon-$id-latest.xpi\\t$license_file_text" >> "$run_path/build/index-search-pages--free-licenses--license-file-status.txt" + + echo "foo" # Must be here to work with pv + + done < "$output_file" | pv__progress + +} diff --git a/subprojects/freeamo/src/main_functions b/subprojects/freeamo/src/main_functions index 782f896..3eb673e 100644 --- a/subprojects/freeamo/src/main_functions +++ b/subprojects/freeamo/src/main_functions @@ -62,9 +62,22 @@ function line_status { # Begin locale code function locale_description { + +echo "####################" tmp__enUS=$(jq ".results[$entry].$locale_description" "$file" | sed "s|-||;" | jq .enUS | sed "s|^\"||; s|\"$||;" ) tmp__enGB=$(jq ".results[$entry].$locale_description" "$file" | sed "s|-||;" | jq .enGB | sed "s|^\"||; s|\"$||;" ) + + tmp=$(jq ".results[$entry].$locale_description[]" "$file" | head -n 1 | sed "s|^\"||; s|\"$||;"); # Use the first avalible if English is not avaliable + +# David will work on this later +# if [ "$tmp" == "" ]; then +# +# # Some entries doesn't provide any `description` field but only a `summary` field. +# tmp=$(jq ".results[$entry].summary[]" "$file" | head -n 1 | sed "s|-||;"); +# +#fi + if [ "$tmp__enUS" != "null" ]; then if [ "$locale_description" == "name" ]; then @@ -98,10 +111,8 @@ function locale_description { fi - else + elif [ "$tmp" != "null" ]; then - tmp=$(jq ".results[$entry].$locale_description[]" "$file" | sed "s|-||;" | jq . | sed "s|^\"||; s|\"$||;"); # Use the first avalible summary if English is not avaliable - if [ "$locale_description" == "name" ]; then name="$tmp"; @@ -115,9 +126,11 @@ function locale_description { description="$tmp"; fi - fi + +echo "$name +.results[$entry].$locale_description[] $file" } diff --git a/subprojects/freeamo/src/make-wiki b/subprojects/freeamo/src/make-wiki index 531116d..93b768d 100644 --- a/subprojects/freeamo/src/make-wiki +++ b/subprojects/freeamo/src/make-wiki @@ -30,21 +30,21 @@ if [ "$minimum_average_daily_users" == "100000" ]; then fi +output_file="$output_file_2"; if [ "$debug" = true ]; then - - pv__size=$(find . -not -name . | head -n 3 | wc -l) + + pv__size=$(cat "$output_file_2" | head -n 3 | wc -l) else - pv__size=$(find . -not -name . | wc -l) + pv__size=$(cat "$output_file_2" | wc -l) fi -output_file="$output_file_2"; source "$run_path/src/main_functions" -echo "Make $pv__size .wiki pages" +echo "Generate $pv__size .wiki pages." while IFS= read -r table; do @@ -213,7 +213,7 @@ at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/ fi echo "foo" # Must be here to work with pv -done < "$output_file_2" | pv__progress +done < "$output_file_2" # | pv__progress # End template diff --git a/subprojects/freeamo/src/make-wiki (copy) b/subprojects/freeamo/src/make-wiki (copy) new file mode 100644 index 0000000..a122592 --- /dev/null +++ b/subprojects/freeamo/src/make-wiki (copy) @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2018 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +cd "build/json/search-pages" || exit + +if [ "$minimum_average_daily_users" == "100000" ]; then + + echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\" +! WebExtension +! Description +! Claimed license +! Users +! Rating +! Updated +! Featured" > "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki" + +fi + +output_file="$output_file_2"; + +if [ "$debug" = true ]; then + + pv__size=$(cat "$output_file_2" | head -n 3 | wc -l) + +else + + pv__size=$(cat "$output_file_2" | wc -l) + +fi + +source "$run_path/src/main_functions" + +echo "Generate $pv__size .wiki pages." + +while IFS= read -r table; do + + ((++line)) + + if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then + + file=$(echo "$table" | awk '{print $1}'); + entry=$(echo "$table" | awk '{print $2}'); + + source "$run_path/src/main_functions" + amo_variables + amo_variables__licenses + + if [ "$license__amo_full_name" == "BSD License" ]; then license__fsd_short_name="BSD 2Clause"; + elif [ "$license__amo_full_name" == "GNU General Public License, version 3.0" ]; then license__fsd_short_name="GPLv3"; + elif [ "$license__amo_full_name" == "GNU General Public License, version 2.0" ]; then license__fsd_short_name="GPLv2"; + elif [ "$license__amo_full_name" == "GNU Lesser General Public License, version 3.0" ]; then license__fsd_short_name="LGPLv3"; + elif [ "$license__amo_full_name" == "GNU Lesser General Public License, version 2.1" ]; then license__fsd_short_name="LGPLv2.1"; + elif [ "$license__amo_full_name" == "MIT/X11 License" ]; then license__fsd_short_name="X11"; + elif [ "$license__amo_full_name" == "Mozilla Public License, version 2.0" ]; then license__fsd_short_name="MPLv2.0"; + elif [ "$license__amo_full_name" == "Mozilla Public License Version 1.1" ]; then license__fsd_short_name="MPLv1.1"; + fi + + # Start description code + + source "$run_path/src/main_functions" + locale_description="summary" + locale_description + locale_description="description" + locale_description + short_description=$(echo "$summary" | format_description) + full_description=$(echo "$description" | format_description) + + # End description code + + # The homepage field is mandatory so let's remove "null" from it because we don't need to evaluat it + homepage_url=$(jq .results[$entry].homepage "$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|null||"); + license_copyright=$(jq .results[$entry].authors[].name "$file" | sed "s|^\"||; s|\"$||;"); + bayesian_average=$(jq .results[$entry].ratings.bayesian_average "$file"); + bayesian_average__simple=$(printf "%.1f" "${bayesian_average//./,}"); + + is_featured=$(jq .results[$entry].is_featured "$file"); + + last_review_date=$(date +"%Y/%m/%d"); + last_review_by="$mediawikibot__bot_password__Username"; + submitted_by="$mediawikibot__bot_password__Username"; + + if [[ "$(jq .results[$entry].current_version.compatibility.seamonkey "$file")" != "null" ]]; then + set_seamonkey="Iceape,"; + fi + + if [[ "$(jq .results[$entry].current_version.compatibility.firefox "$file")" != "null" ]]; then + set_icecat="IceCat,"; + fi + + # if [[ "$(jq .results[$entry].current_version.compatibility.android $file)" != "null" ]] || [[ "$(jq .results[$entry].current_version.compatibility.mobile $file)" != "null" ]]; then + # set_icecatmobile="IceCatMobile,"; + # fi + + if [[ "$(jq .results[$entry].current_version.compatibility.thunderbird "$file")" != "null" ]]; then + set_thunderbird="Icedove,"; + fi + + # Do not list IceCatMobile + extension_of=$(echo "$set_seamonkey$set_icecat$set_thunderbird" | sed "s|,$||;"); + + + + github_true=$(echo "$homepage_url" | grep "github.com"); + + if [[ $github_true != "" ]]; then + + # Always use https, and remove anchors + homepage_url__for__vcs_checkout_command="https://github.com$(echo "$github_true" | sed "s|https://github.com||; s|http://github.com||; s|#|\\n|;" | head -n 1)"; + # Remove trailing slash + homepage_url__for__vcs_checkout_command="${homepage_url__for__vcs_checkout_command%/}" + vcs_checkout_command="git clone $homepage_url__for__vcs_checkout_command.git"; + + fi + + if [ "$name" == "GNU LibreJS" ]; then + + is_gnu="Yes"; + + else + + is_gnu="No"; + + fi + + support_url=$(jq .results[$entry].support_url "$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + last_updated=$(jq .results[$entry].last_updated $file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g"); + + # List JSON structure: cat json/$file | js '.' + # XML value "homepage" is not always set in AMO API, but the XML require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically. + echo "{{Entry" > "$run_path/build/wiki/$slug.wiki" + + # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341 + # Avoid specific versions since it's not compatible with all IceCat versions: + # |Version download=https://addons.mozilla.org/firefox/downloads/latest/$id/addon-$version_id-latest.xpi + # |Version identifier=$version + # |Version date=$version_date + + + # Do add these fields since we refer to the download page in the `|Version download` field that lists compatible versions + # |Version identifier=52.6.0 + # |Version date=2018/02/26 + # |Version status=stable + + echo "|Name=$name +|Short description=$short_description +|Full description=$full_description +|Homepage URL=$homepage_url +|Extension of=$extension_of +|VCS checkout command=$vcs_checkout_command +|Documentation note=Extra data (https://addons.mozilla.org/api/v$api_version/addons/addon/$slug/): +* Users: $average_daily_users +* Rating: $bayesian_average__simple +* Updated: $last_updated +* Featured: $is_featured +|Version download=https://addons.mozilla.org/en-US/firefox/addon/$slug/versions/ +|Last review by=$last_review_by +|Last review date=$last_review_date +|Submitted by=$submitted_by +|Submitted date= +|Is GNU=$is_gnu +}}" >> "$run_path/build/wiki/$slug.wiki" + + + + # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/ + echo "{{Project license +|License=$license__fsd_short_name +|License copyright=$license_copyright +|License note=\"License: $license__amo_full_name\" listed +at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/ +}}" >> "$run_path/build/wiki/$slug.wiki" + + if [ "$support_url" != "null" ]; + then + + echo "{{Resource +|Resource audience=Users +|Resource kind=Support +|Resource URL=$support_url +}}" >> "$run_path/build/wiki/$slug.wiki" + + fi + + if [ "$minimum_average_daily_users" == "100000" ]; then + + echo "|- +| [[$slug|$name]] +| $short_description +| [[License:$license__fsd_short_name|$license__fsd_short_name]] +| $average_daily_users +| $bayesian_average__simple +| $last_updated +| $is_featured" >> "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki" + + fi + + + + fi + + echo "$name" # Must be here to work with pv +done < "$output_file_2" # | pv__progress +# End template + + +if [ "$minimum_average_daily_users" == "100000" ]; then + + echo "|}" >> "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki" + +fi diff --git a/subprojects/freeamo/src/search-pages b/subprojects/freeamo/src/search-pages index 45ed716..603c750 100644 --- a/subprojects/freeamo/src/search-pages +++ b/subprojects/freeamo/src/search-pages @@ -56,6 +56,7 @@ function search-pages--download() { pv__discard=true; echo "Cannot show a progress counter since minimum_average_daily_users is set to more than 0 in freeamo.conf." + extra_text="max" else @@ -75,7 +76,7 @@ function search-pages--download() { if ( [ "$debug" = true ] && [ "$page" -le "3" ] || [ "$debug" = false ] ) && ( [ "$average_daily_users" -ge "$minimum_average_daily_users" ] ); then source "$run_path/src/main_functions" - echo "Downloading search page: $page of $page_count" + echo "Downloading search page: $page of $extra_text $page_count" wget_page else diff --git a/subprojects/freeamo/src/unmaintained/download-free-webextensions b/subprojects/freeamo/src/unmaintained/download-free-webextensions deleted file mode 100644 index 7084b65..0000000 --- a/subprojects/freeamo/src/unmaintained/download-free-webextensions +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -# This file is part of FreeAMO -# Copyright (C) 2017 David Hedlund -# -# FreeAMO is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# FreeAMO is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -cd build || exit -rm -fr free_webextensions -mkdir free_webextensions -cd free_webextensions || exit - - - -# Begin template -unset file -while IFS= read -r table; do - - verify_variables "$1" - - if [ "$approved_program" = true ]; then - - # End template - - id=$(echo $table | awk '{print $2}'); - file=$(echo $table | awk '{print $3}'); - slug=$(echo $table | awk '{print $4}'); - - # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). - wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" - - - - # Begin template - fi - - ((line++)) - -done < $run_path/build/MERGED-ALL-freedom_status.txt -# End template -- cgit v1.2.3