diff options
Diffstat (limited to 'subprojects/freeamo')
| -rw-r--r-- | subprojects/freeamo/README.md | 48 | ||||
| -rwxr-xr-x | subprojects/freeamo/freeamo | 35 | ||||
| -rw-r--r-- | subprojects/freeamo/freeamo.conf | 2 | ||||
| -rw-r--r-- | subprojects/freeamo/src/get-add-ons | 96 | ||||
| -rw-r--r-- | subprojects/freeamo/src/main_functions | 21 | ||||
| -rw-r--r-- | subprojects/freeamo/src/make-wiki | 12 | ||||
| -rw-r--r-- | subprojects/freeamo/src/make-wiki (copy) | 224 | ||||
| -rw-r--r-- | subprojects/freeamo/src/search-pages | 3 | ||||
| -rw-r--r-- | subprojects/freeamo/src/unmaintained/download-free-webextensions | 50 |
9 files changed, 399 insertions, 92 deletions
diff --git a/subprojects/freeamo/README.md b/subprojects/freeamo/README.md index 208749d..b6a90a0 100644 --- a/subprojects/freeamo/README.md +++ b/subprojects/freeamo/README.md @@ -1,19 +1,25 @@ ##################################################################### -About this project - -This project has been developed to syncronize addons.mozilla.org (AMO) -WebExtension entries that is compatible with IceCat (version 60+) to -the Free Software Directory entries -(https://directory.fsf.org/wiki/Collection:IceCat), example -https://addons.mozilla.org/en-US/firefox/addon/librejs/ with -https://directory.fsf.org/wiki/LibreJS. For more about IceCat see the -`About IceCat` section and read the `About WebExtensions` section to -find out why legacy add-ons are excluded. - -Icedove add-ons is not synced yet because Thunderbird still hasn't -implemented WebExtension support, but it's discussed by the Mozilla -developers, see https://bugzilla.mozilla.org/show_bug.cgi?id=1396172 -For more about Icedove see the `About Icedove` section. +About FreeAMO + +FreeAMO is a project used to get meta-data for free add-ons at +addons.mozilla.org (AMO) used to produce free add-on repositories for +free Mozilla-based programs. contain nonfree add-ons and Mozilla +Foundation's addons.mozilla.org contains nonfree add-ons, and their +programs including Firefox, Thunderbird, SeaMonkey, and Sunbird are +trademarked and thus nonfree. + +FreeAMO ships with configuration files used to find free add-ons for +the latest icecat version for Trisquel GNU/Linux. For more about +IceCat see the `About IceCat` section and read the `About +WebExtensions` section to find out why legacy add-ons are +excluded. Icedove add-ons is not synced yet because Thunderbird still +hasn't implemented WebExtension support, but it's discussed by the +Mozilla developers, see +https://bugzilla.mozilla.org/show_bug.cgi?id=1396172 For more about +Icedove see the `About Icedove` section. FreeAMO has an option to +generate .wiki files designed to be imported to +https://directory.fsf.org/wiki/, example: +https://directory.fsf.org/wiki/LibreJS This script uses the Mozilla add-on API (https://addons-server.readthedocs.io/en/latest/topics/api/addons.html), @@ -30,18 +36,6 @@ that can be synced to the Free Software Directory while add-ons that has these issues will generated on a list so we can contact the developers and ask them to fix them. -I've decided to set the uppder limit for WebExtensions to have 10 000 -daily users or more so we have a chance to work on the most relevant -add-ons since there are many thousands of add-ons on -addons.mozilla.org. However a few add-ons that has less than 10 000 -daily users are highly relevant (for example -https://addons.mozilla.org/en-US/firefox/addon/librejs/ at this -writing) so the script will be able to include add-ons from customized -so called Collections from addons.mozilla.org (example, -https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/) -where they can be managed, or from a simple text document that lists -the addons.mozilla.org URL's. - ##################################################################### About addons.mozilla.org diff --git a/subprojects/freeamo/freeamo b/subprojects/freeamo/freeamo index bd1df0e..2c036e9 100755 --- a/subprojects/freeamo/freeamo +++ b/subprojects/freeamo/freeamo @@ -74,6 +74,9 @@ case "$1" in freeamo_command="$0 --get-licenses $debug_arg :" echo "$freeamo_command"; $freeamo_command + freeamo_command="$0 --get-add-ons $debug_arg :" + echo "$freeamo_command"; $freeamo_command + if [ ! "$1" == "--normal-mode" ]; then freeamo_command="$0 --make-directory.fsf.org-wiki $debug_arg :" @@ -81,7 +84,6 @@ case "$1" in fi - ;; -help) @@ -100,6 +102,8 @@ f.org-wiki (they are used by Free Software Directory developers) Downloads the most popular WebExtensions from https://addons.mozilla.org/en-US/firefox/search/... $0 --get-licenses The licenses are not added to the general JSON file. This option will download the neccesary license JSON files for all add-ons. + $0 --get-add-ons + Download the installable add-ons (the .xpi files). $0 --make-directory.fsf.org-wiki Generates .wiki files specifically for https://directory.fsf.org/wiki/ @@ -194,12 +198,37 @@ Disabled options due to limited development time get-licenses--filter ;; + + --get-add-ons) + + freeamo_command="$0 --get-add-ons--download $debug_arg :" + echo " ↳$freeamo_command"; $freeamo_command + + freeamo_command="$0 --get-add-ons--filter $debug_arg :" + echo " ↳$freeamo_command"; $freeamo_command + + ;; + + --get-add-ons--download) + + mkdir -p "$run_path/build/add-ons" + source src/get-add-ons + get-add-ons--download + + ;; + + --get-add-ons--filter) + + source src/get-add-ons + get-add-ons--filter + + ;; --make-directory.fsf.org-wiki) mkdir -p "$run_path/build/wiki"; source src/make-wiki - + ;; - + esac diff --git a/subprojects/freeamo/freeamo.conf b/subprojects/freeamo/freeamo.conf index 526427f..3660dd0 100644 --- a/subprojects/freeamo/freeamo.conf +++ b/subprojects/freeamo/freeamo.conf @@ -13,7 +13,7 @@ sort="users" type="extension" # Mozilla API parameters: Add-ons -> Detail -minimum_average_daily_users=0 +minimum_average_daily_users=50000 # Extra notes="Extensions for the latest Trisquel GNU/Linux icecat package version." diff --git a/subprojects/freeamo/src/get-add-ons b/subprojects/freeamo/src/get-add-ons new file mode 100644 index 0000000..6038bd6 --- /dev/null +++ b/subprojects/freeamo/src/get-add-ons @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2017 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +cd "build/json/search-pages" || exit + +function get-add-ons--download { + + + output_file="$output_file_2"; + + if [ "$debug" = true ]; then + + pv__size=$(cat "$output_file" | head -n 3 | wc -l) + + else + + pv__size="$(wc -l < "$output_file")" + + fi + + echo "Download $pv__size add-ons" + + while IFS= read -r table; do + + ((line++)) + + file=$(echo "$table" | awk '{print $1}'); + entry=$(echo "$table" | awk '{print $2}'); + + source "$run_path/src/main_functions" + cd "$run_path/build/json/search-pages" || exit + amo_variables + + cd "$run_path/build/add-ons" || exit + # Add-ons not avalible for GNU/Linux will be ignored (used to be very few though). + wget -nc -q "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc -q "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" + + echo "foo" # Must be here to work with pv + + done < "$output_file" | pv__progress + +} + + +function get-add-ons--filter { + + rm -f "$run_path/build/index-search-pages--free-licenses--license-file-status.txt" + + output_file="$output_file_2"; + + if [ "$debug" = true ]; then + + pv__size=$(cat "$output_file" | head -n 3 | wc -l) + + else + + pv__size="$(wc -l < "$output_file")" + + fi + + echo "Filter $pv__size add-ons " + + while IFS= read -r table; do + + ((line++)) + + file=$(echo "$table" | awk '{print $1}'); + entry=$(echo "$table" | awk '{print $2}'); + + source "$run_path/src/main_functions" + cd "$run_path/build/json/search-pages" || exit + amo_variables + + cd "$run_path/build/add-ons" || exit + license_file_text=$(unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -E "^COPYING$|^COPYING.txt$|^COPYING.md$|^LICENSE$|^LICENSE.txt$|^LICENSE.md$" | sed ':a;N;$!ba;s/\n/\t/g' || echo "NOLICENSEFILEFOUND"); + echo -e "$file\\t$entry\\t$slug\\taddon-$id-latest.xpi\\t$license_file_text" >> "$run_path/build/index-search-pages--free-licenses--license-file-status.txt" + + echo "foo" # Must be here to work with pv + + done < "$output_file" | pv__progress + +} diff --git a/subprojects/freeamo/src/main_functions b/subprojects/freeamo/src/main_functions index 782f896..3eb673e 100644 --- a/subprojects/freeamo/src/main_functions +++ b/subprojects/freeamo/src/main_functions @@ -62,9 +62,22 @@ function line_status { # Begin locale code function locale_description { + +echo "####################" tmp__enUS=$(jq ".results[$entry].$locale_description" "$file" | sed "s|-||;" | jq .enUS | sed "s|^\"||; s|\"$||;" ) tmp__enGB=$(jq ".results[$entry].$locale_description" "$file" | sed "s|-||;" | jq .enGB | sed "s|^\"||; s|\"$||;" ) + + tmp=$(jq ".results[$entry].$locale_description[]" "$file" | head -n 1 | sed "s|^\"||; s|\"$||;"); # Use the first avalible if English is not avaliable + +# David will work on this later +# if [ "$tmp" == "" ]; then +# +# # Some entries doesn't provide any `description` field but only a `summary` field. +# tmp=$(jq ".results[$entry].summary[]" "$file" | head -n 1 | sed "s|-||;"); +# +#fi + if [ "$tmp__enUS" != "null" ]; then if [ "$locale_description" == "name" ]; then @@ -98,10 +111,8 @@ function locale_description { fi - else + elif [ "$tmp" != "null" ]; then - tmp=$(jq ".results[$entry].$locale_description[]" "$file" | sed "s|-||;" | jq . | sed "s|^\"||; s|\"$||;"); # Use the first avalible summary if English is not avaliable - if [ "$locale_description" == "name" ]; then name="$tmp"; @@ -115,9 +126,11 @@ function locale_description { description="$tmp"; fi - fi + +echo "$name +.results[$entry].$locale_description[] $file" } diff --git a/subprojects/freeamo/src/make-wiki b/subprojects/freeamo/src/make-wiki index 531116d..93b768d 100644 --- a/subprojects/freeamo/src/make-wiki +++ b/subprojects/freeamo/src/make-wiki @@ -30,21 +30,21 @@ if [ "$minimum_average_daily_users" == "100000" ]; then fi +output_file="$output_file_2"; if [ "$debug" = true ]; then - - pv__size=$(find . -not -name . | head -n 3 | wc -l) + + pv__size=$(cat "$output_file_2" | head -n 3 | wc -l) else - pv__size=$(find . -not -name . | wc -l) + pv__size=$(cat "$output_file_2" | wc -l) fi -output_file="$output_file_2"; source "$run_path/src/main_functions" -echo "Make $pv__size .wiki pages" +echo "Generate $pv__size .wiki pages." while IFS= read -r table; do @@ -213,7 +213,7 @@ at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/ fi echo "foo" # Must be here to work with pv -done < "$output_file_2" | pv__progress +done < "$output_file_2" # | pv__progress # End template diff --git a/subprojects/freeamo/src/make-wiki (copy) b/subprojects/freeamo/src/make-wiki (copy) new file mode 100644 index 0000000..a122592 --- /dev/null +++ b/subprojects/freeamo/src/make-wiki (copy) @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2018 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +cd "build/json/search-pages" || exit + +if [ "$minimum_average_daily_users" == "100000" ]; then + + echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\" +! WebExtension +! Description +! Claimed license +! Users +! Rating +! Updated +! Featured" > "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki" + +fi + +output_file="$output_file_2"; + +if [ "$debug" = true ]; then + + pv__size=$(cat "$output_file_2" | head -n 3 | wc -l) + +else + + pv__size=$(cat "$output_file_2" | wc -l) + +fi + +source "$run_path/src/main_functions" + +echo "Generate $pv__size .wiki pages." + +while IFS= read -r table; do + + ((++line)) + + if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then + + file=$(echo "$table" | awk '{print $1}'); + entry=$(echo "$table" | awk '{print $2}'); + + source "$run_path/src/main_functions" + amo_variables + amo_variables__licenses + + if [ "$license__amo_full_name" == "BSD License" ]; then license__fsd_short_name="BSD 2Clause"; + elif [ "$license__amo_full_name" == "GNU General Public License, version 3.0" ]; then license__fsd_short_name="GPLv3"; + elif [ "$license__amo_full_name" == "GNU General Public License, version 2.0" ]; then license__fsd_short_name="GPLv2"; + elif [ "$license__amo_full_name" == "GNU Lesser General Public License, version 3.0" ]; then license__fsd_short_name="LGPLv3"; + elif [ "$license__amo_full_name" == "GNU Lesser General Public License, version 2.1" ]; then license__fsd_short_name="LGPLv2.1"; + elif [ "$license__amo_full_name" == "MIT/X11 License" ]; then license__fsd_short_name="X11"; + elif [ "$license__amo_full_name" == "Mozilla Public License, version 2.0" ]; then license__fsd_short_name="MPLv2.0"; + elif [ "$license__amo_full_name" == "Mozilla Public License Version 1.1" ]; then license__fsd_short_name="MPLv1.1"; + fi + + # Start description code + + source "$run_path/src/main_functions" + locale_description="summary" + locale_description + locale_description="description" + locale_description + short_description=$(echo "$summary" | format_description) + full_description=$(echo "$description" | format_description) + + # End description code + + # The homepage field is mandatory so let's remove "null" from it because we don't need to evaluat it + homepage_url=$(jq .results[$entry].homepage "$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|null||"); + license_copyright=$(jq .results[$entry].authors[].name "$file" | sed "s|^\"||; s|\"$||;"); + bayesian_average=$(jq .results[$entry].ratings.bayesian_average "$file"); + bayesian_average__simple=$(printf "%.1f" "${bayesian_average//./,}"); + + is_featured=$(jq .results[$entry].is_featured "$file"); + + last_review_date=$(date +"%Y/%m/%d"); + last_review_by="$mediawikibot__bot_password__Username"; + submitted_by="$mediawikibot__bot_password__Username"; + + if [[ "$(jq .results[$entry].current_version.compatibility.seamonkey "$file")" != "null" ]]; then + set_seamonkey="Iceape,"; + fi + + if [[ "$(jq .results[$entry].current_version.compatibility.firefox "$file")" != "null" ]]; then + set_icecat="IceCat,"; + fi + + # if [[ "$(jq .results[$entry].current_version.compatibility.android $file)" != "null" ]] || [[ "$(jq .results[$entry].current_version.compatibility.mobile $file)" != "null" ]]; then + # set_icecatmobile="IceCatMobile,"; + # fi + + if [[ "$(jq .results[$entry].current_version.compatibility.thunderbird "$file")" != "null" ]]; then + set_thunderbird="Icedove,"; + fi + + # Do not list IceCatMobile + extension_of=$(echo "$set_seamonkey$set_icecat$set_thunderbird" | sed "s|,$||;"); + + + + github_true=$(echo "$homepage_url" | grep "github.com"); + + if [[ $github_true != "" ]]; then + + # Always use https, and remove anchors + homepage_url__for__vcs_checkout_command="https://github.com$(echo "$github_true" | sed "s|https://github.com||; s|http://github.com||; s|#|\\n|;" | head -n 1)"; + # Remove trailing slash + homepage_url__for__vcs_checkout_command="${homepage_url__for__vcs_checkout_command%/}" + vcs_checkout_command="git clone $homepage_url__for__vcs_checkout_command.git"; + + fi + + if [ "$name" == "GNU LibreJS" ]; then + + is_gnu="Yes"; + + else + + is_gnu="No"; + + fi + + support_url=$(jq .results[$entry].support_url "$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + last_updated=$(jq .results[$entry].last_updated $file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g"); + + # List JSON structure: cat json/$file | js '.' + # XML value "homepage" is not always set in AMO API, but the XML require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically. + echo "{{Entry" > "$run_path/build/wiki/$slug.wiki" + + # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341 + # Avoid specific versions since it's not compatible with all IceCat versions: + # |Version download=https://addons.mozilla.org/firefox/downloads/latest/$id/addon-$version_id-latest.xpi + # |Version identifier=$version + # |Version date=$version_date + + + # Do add these fields since we refer to the download page in the `|Version download` field that lists compatible versions + # |Version identifier=52.6.0 + # |Version date=2018/02/26 + # |Version status=stable + + echo "|Name=$name +|Short description=$short_description +|Full description=$full_description +|Homepage URL=$homepage_url +|Extension of=$extension_of +|VCS checkout command=$vcs_checkout_command +|Documentation note=Extra data (https://addons.mozilla.org/api/v$api_version/addons/addon/$slug/): +* Users: $average_daily_users +* Rating: $bayesian_average__simple +* Updated: $last_updated +* Featured: $is_featured +|Version download=https://addons.mozilla.org/en-US/firefox/addon/$slug/versions/ +|Last review by=$last_review_by +|Last review date=$last_review_date +|Submitted by=$submitted_by +|Submitted date= +|Is GNU=$is_gnu +}}" >> "$run_path/build/wiki/$slug.wiki" + + + + # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/ + echo "{{Project license +|License=$license__fsd_short_name +|License copyright=$license_copyright +|License note=\"License: $license__amo_full_name\" listed +at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/ +}}" >> "$run_path/build/wiki/$slug.wiki" + + if [ "$support_url" != "null" ]; + then + + echo "{{Resource +|Resource audience=Users +|Resource kind=Support +|Resource URL=$support_url +}}" >> "$run_path/build/wiki/$slug.wiki" + + fi + + if [ "$minimum_average_daily_users" == "100000" ]; then + + echo "|- +| [[$slug|$name]] +| $short_description +| [[License:$license__fsd_short_name|$license__fsd_short_name]] +| $average_daily_users +| $bayesian_average__simple +| $last_updated +| $is_featured" >> "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki" + + fi + + + + fi + + echo "$name" # Must be here to work with pv +done < "$output_file_2" # | pv__progress +# End template + + +if [ "$minimum_average_daily_users" == "100000" ]; then + + echo "|}" >> "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki" + +fi diff --git a/subprojects/freeamo/src/search-pages b/subprojects/freeamo/src/search-pages index 45ed716..603c750 100644 --- a/subprojects/freeamo/src/search-pages +++ b/subprojects/freeamo/src/search-pages @@ -56,6 +56,7 @@ function search-pages--download() { pv__discard=true; echo "Cannot show a progress counter since minimum_average_daily_users is set to more than 0 in freeamo.conf." + extra_text="max" else @@ -75,7 +76,7 @@ function search-pages--download() { if ( [ "$debug" = true ] && [ "$page" -le "3" ] || [ "$debug" = false ] ) && ( [ "$average_daily_users" -ge "$minimum_average_daily_users" ] ); then source "$run_path/src/main_functions" - echo "Downloading search page: $page of $page_count" + echo "Downloading search page: $page of $extra_text $page_count" wget_page else diff --git a/subprojects/freeamo/src/unmaintained/download-free-webextensions b/subprojects/freeamo/src/unmaintained/download-free-webextensions deleted file mode 100644 index 7084b65..0000000 --- a/subprojects/freeamo/src/unmaintained/download-free-webextensions +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -# This file is part of FreeAMO -# Copyright (C) 2017 David Hedlund -# -# FreeAMO is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# FreeAMO is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -cd build || exit -rm -fr free_webextensions -mkdir free_webextensions -cd free_webextensions || exit - - - -# Begin template -unset file -while IFS= read -r table; do - - verify_variables "$1" - - if [ "$approved_program" = true ]; then - - # End template - - id=$(echo $table | awk '{print $2}'); - file=$(echo $table | awk '{print $3}'); - slug=$(echo $table | awk '{print $4}'); - - # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). - wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" - - - - # Begin template - fi - - ((line++)) - -done < $run_path/build/MERGED-ALL-freedom_status.txt -# End template |