diff options
Diffstat (limited to 'addons.mozilla.org-fsd')
| -rw-r--r-- | addons.mozilla.org-fsd/get-data/TODO | 2 | ||||
| -rwxr-xr-x | addons.mozilla.org-fsd/get-data/license-reporter | 363 | ||||
| -rwxr-xr-x | addons.mozilla.org-fsd/get-data/license-reporter (another copy) | 528 | ||||
| -rwxr-xr-x | addons.mozilla.org-fsd/get-data/license-reporter (copy) | 442 | ||||
| -rw-r--r-- | addons.mozilla.org-fsd/get-data/license-reporter-collections | 1 | ||||
| -rwxr-xr-x | addons.mozilla.org-fsd/get-data/license-reporter~ | 377 | ||||
| -rwxr-xr-x | addons.mozilla.org-fsd/sync-data/fsd-amo-bot | 164 |
7 files changed, 315 insertions, 1562 deletions
diff --git a/addons.mozilla.org-fsd/get-data/TODO b/addons.mozilla.org-fsd/get-data/TODO index 7c808ab..52b5c0e 100644 --- a/addons.mozilla.org-fsd/get-data/TODO +++ b/addons.mozilla.org-fsd/get-data/TODO @@ -1,3 +1,5 @@ +* Merged https://directory.fsf.org/wiki/Free_Software_Directory:IceCat_WebExtensions_(proposed)/About to the README + The author of the package, licenseutils, has implemented an option per request that make it possible to use detect licenses, its evoked with: $ licensing detect * diff --git a/addons.mozilla.org-fsd/get-data/license-reporter b/addons.mozilla.org-fsd/get-data/license-reporter index 5298037..3a73598 100755 --- a/addons.mozilla.org-fsd/get-data/license-reporter +++ b/addons.mozilla.org-fsd/get-data/license-reporter @@ -34,7 +34,7 @@ if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi -minimum_average_daily_users="1000000"; +minimum_average_daily_users="100000"; SCRIPTSRC=$(readlink -f "$0" || echo "$0") RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .) @@ -69,6 +69,20 @@ function check_files { } +function line_status { + + + if [ "$file" == "$previous_file" ]; then + + ((line++)) + + else + + line="0"; # Should not be 1 + + fi + +} case "$1" in @@ -83,14 +97,24 @@ OPTIONS $0 --make-repository-list Downloads add-on data for free add-ons from collections shared by defined users in license-reporter-repositories. Example: https://addons.mozilla.org/en-US/firefox/collections/mozilla/ - $0 --make-collection-list $0 --make-custom-list $0 --make-search-list + $0 --get-licenses $0 --merge-lists + $0 --generate-proposed-list $0 --download-licenses-json $0 --download-free-webextensions-for-gnu-and-linux $0 --verify-license-copy + + --all-recommended + $0 --fresh-build + $0 --make-search-list + $0 --merge-lists + $0 --get-licenses + $0 --merged-free + $0 --make-wiki + " && exit 1 ;; @@ -224,15 +248,18 @@ OPTIONS page="1" - + function foo() { # Do not evaluate the number of extensions seen in https://addons.mozilla.org/en-US/firefox/search/ in a Quantum based browser since that will hide legacy add-ons. - uri_query="page=$page&platform=linux&sort=users&type=extension&tag=firefox57"; + uri_query="page=$page&platform=linux&sort=users&type=extension"; + # Get the most popular WebExtensions wget "https://addons.mozilla.org/api/v3/addons/search/?$uri_query" -O "index.html?$uri_query.json" + ((page++)) + } @@ -244,7 +271,6 @@ OPTIONS while [ "$(jq ".results[].average_daily_users" "index.html?$uri_query.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do - ((page++)) foo done @@ -256,14 +282,59 @@ OPTIONS find index*.json -type f -printf "%Tc %p\n" | sort -n | awk '{print $NF}' | while read -r file; do - line="0"; + + + line="0"; # Should not be 1 for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do + + + if [ "$(jq ".results[$line].has_eula" "$file")" == "false" ]; then + + has_eula="eulafree"; + + else + + has_eula="eula"; + + fi + + + # Clean titles + + name=$(jq .results[$line].name $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + + if [ "$name" == "null" ]; then + + name=$(jq .results[$line].name $RUN_PATH/build/json/$file | sed "s|en-GB|enGB|;" | jq .enGB | sed "s|^\"||; s|\"$||;"); + + fi + + + # Clean titles. Example of bad titles: "Ghostery – Privacy Ad Blocker" + + # - + # Hard to filter out "-" since many add-ons use it, see for example https://addons.mozilla.org/en-US/firefox/search/?platform=linux&q=1-click + # Flash Video Downloader - YouTube HD Download [4K] + + name=$(echo $name | sed ' +s/ — /\n/g; +s/ – /\n/g; +s/ - /\n/g; # looks identical to the prevoius line but is unique +s/: /\n/g; +s/(/\n/g; +s/,/\n/g; +' | head -n 1 | sed 's/ $//'); + + + + + # Exclude add-ons with to low average daily users from index.html* if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then - echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file\t$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")" >> ../merged-search.txt + echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file\t$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")\t$name\t$has_eula" >> ../merged-search.txt fi @@ -290,17 +361,22 @@ OPTIONS cd build/ || exit mkdir -p json/current_versions cd json/current_versions || exit + + line="0"; # Should not be 1 - line="0"; while IFS= read -r table; do - ((line++)) - id=$(echo $table | awk '{print $2}'); file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}'); + line_status + + # echo "$file $line $id" + wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json" + previous_file="$file"; + done < $RUN_PATH/build/MERGED-ALL.txt ;; @@ -309,75 +385,254 @@ OPTIONS cd build/ || exit cp -a MERGED-ALL.txt MERGED-ALL-freedom_status.txt - line="0"; + line="1"; # Should not be "0" while IFS= read -r table; do - ((line++)) - id=$(echo $table | awk '{print $2}'); - license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - + license=$(jq .license.name "$RUN_PATH/build/json/current_versions/$id.json" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". + # See https://addons.mozilla.org/en-US/developers/addon/<your add-on>/ownership for the current add-on list. Note that not all licenses are added to the list. + # DuckDuckGo Privacy Essentials is distributed under Apache License 2.0. + # Apache License, version 2.0 is not on the add-on license list - https://github.com/mozilla/addons-server/issues/8545 if - [ "$license" == "GNU General Public License, version 3.0" ] || + [ "$license" == "Apache License, Version 2.0" ] || # Custom license + [ "$license" == "Apache License, version 2.0" ] || # Custom license + [ "$license" == "BSD License" ] || + [ "$license" == "GNU General Public License, version 3.0" ] || [ "$license" == "GNU General Public License, version 2.0" ] || [ "$license" == "GNU Lesser General Public License, version 3.0" ] || [ "$license" == "GNU Lesser General Public License, version 2.1" ] || + [ "$license" == "MIT/X11 License" ] || [ "$license" == "Mozilla Public License, version 2.0" ] || - [ "$license" == "Mozilla Public License Version 1.1" ] || - [ "$license" == "BSD License" ] || - [ "$license" == "MIT/X11 License" ] + [ "$license" == "Mozilla Public License Version 1.1" ] then # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files. sed -i "$line""s/$/\\tfree/" MERGED-ALL-freedom_status.txt - else + elif [ "$license" == "All Rights Reserved" ]; then + sed -i "$line""s/$/\\tnonfree/" MERGED-ALL-freedom_status.txt + else + + sed -i "$line""s/$/\\tunknown/" MERGED-ALL-freedom_status.txt + fi + # Add license for analysis + # don't use "/" as delimeters since some licenses are "MIT/X11 License" + sed -i "$line""s|$|\\t$license|" MERGED-ALL-freedom_status.txt + + ((line++)) + done < $RUN_PATH/build/MERGED-ALL.txt ;; - --generate-proposed-list) + --make-wiki) - line="0"; + cd build/ || exit + if [ ! -d "wiki" ]; then mkdir wiki; fi + + echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\" +! WebExtension +! Description +! Claimed license +! Users +! Rating +! Updated" > "wiki/IceCat WebExtensions (proposed).wiki" + + + unset file while IFS= read -r table; do - ((line++)) + if [ -z ${file+x} ] || [ "$file" != "$(echo $table | awk '{print $3}')" ]; then + + line="0"; # should not be 1 + + fi + average_daily_users=$(echo $table | awk '{print $1}'); id=$(echo $table | awk '{print $2}'); - source=$(echo $table | awk '{print $3}'); + file=$(echo $table | awk '{print $3}'); + name=$(echo "$table" | cut -f 5); + has_eula=$(echo "$table" | cut -f 6); + freedom_status=$(echo "$table" | cut -f 7); + license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - freedom_status=$(echo $table | awk '{print $5}'); - name=$(jq .results[$line].name $RUN_PATH/build/json/$source | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + firefox_max_version="$(jq .results[$line].current_version.compatibility.firefox.max $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|.\*||")"; + - if [ "$name" == "null" ]; then + # Values are either an integer or "*" + re='^[0-9]+$' + if [[ $firefox_max_version =~ $re ]] && [ "$firefox_max_version" -ge 60 ] || [ "$firefox_max_version" == '*' ]; then - name=$(jq .results[$line].name $RUN_PATH/build/json/$source | sed "s|en-GB|enGB|;" | jq .enGB | sed "s|^\"||; s|\"$||;"); +# echo "$firefox_max_version" + firefox_max_version__pass=true; + + else + unset firefox_max_version__pass + fi + + if [ "$freedom_status" == "free" ] && [ "$has_eula" == "eulafree" ] && [ "$firefox_max_version__pass" = true ]; then - # Clean titles. Example of bad titles: "Ghostery – Privacy Ad Blocker" - if [[ $name = *"–"* ]]; then + # require min version 60 ddd - name=$(echo $name | sed 's/–/\n/g; s/-/\n/g' | head -n 1); + #################################### + # Build the core wiki pages - fi + average_daily_users=$(echo $table | awk '{print $1}'); + id=$(echo $table | awk '{print $2}'); + file=$(echo $table | awk '{print $3}'); + name=$(echo "$table" | cut -f 5); + has_eula=$(echo "$table" | cut -f 6); + freedom_status=$(echo "$table" | cut -f 7); + + license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + + + if [ "$license" == "Apache License, Version 2.0" ]; then license="Apache2.0"; + elif [ "$license" == "Apache License, version 2.0" ]; then license="Apache2.0"; + elif [ "$license" == "BSD License" ]; then license="BSD 2Clause"; + elif [ "$license" == "GNU General Public License, version 3.0" ]; then license="GPLv3"; + elif [ "$license" == "GNU General Public License, version 2.0" ]; then license="GPLv2"; + elif [ "$license" == "GNU Lesser General Public License, version 3.0" ]; then license="LGPLv3"; + elif [ "$license" == "GNU Lesser General Public License, version 2.1" ]; then license="LGPLv2.1"; + elif [ "$license" == "MIT/X11 License" ]; then license="X11"; + elif [ "$license" == "Mozilla Public License, version 2.0" ]; then license="MPLv2.0"; + elif [ "$license" == "Mozilla Public License Version 1.1" ]; then license="MPLv1.1"; + fi - if [ "$freedom_status" == "free" ]; then - echo "{{Checkme item -|Section= -|Package=$name -|Package note= -}}" + + + + short_description=$(jq .results[$line].summary $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|\\\n|\n|g; s|\\\||g"); + +# echo "$short_description" + + full_description=$(jq .results[$line].description $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + homepage_url=$(jq .results[$line].homepage $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + version=$(jq .results[$line].current_version.version $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;"); + version_date=$(jq .results[$line].last_updated $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g"); + version_id=$(jq .results[$line].current_version.id $RUN_PATH/build/json/$file); + license_copyright=$(jq .results[$line].authors[].name $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;"); + bayesian_average=$(jq .results[$line].ratings.bayesian_average $RUN_PATH/build/json/$file); + bayesian_average__simple=$(printf "%.1f" "$(echo "$bayesian_average" | sed "s|\.|,|")"); + + last_review_date=$(date +"%Y/%m/%d"); + last_review_by="wikisysbot"; + submitted_by="wikisysbot"; + + if [[ "$(jq .results[$line].current_version.compatibility.seamonkey $RUN_PATH/build/json/$file)" != "null" ]]; then + set_seamonkey="Iceape,"; + fi + + if [[ "$(jq .results[$line].current_version.compatibility.firefox $RUN_PATH/build/json/$file)" != "null" ]]; then + set_icecat="IceCat,"; + fi + + # if [[ "$(jq .results[$line].current_version.compatibility.android $RUN_PATH/build/json/$file)" != "null" ]] || [[ "$(jq .results[$line].current_version.compatibility.mobile $RUN_PATH/build/json/$file)" != "null" ]]; then + # set_icecatmobile="IceCatMobile,"; + # fi + + if [[ "$(jq .results[$line].current_version.compatibility.thunderbird $RUN_PATH/build/json/$file)" != "null" ]]; then + set_thunderbird="Icedove,"; + fi + + # Do not list IceCatMobile + extension_of=$(echo "$set_seamonkey$set_icecat$set_thunderbird" | sed "s|,$||;"); + + + + github_true=$(echo "$homepage_url" | grep "github.com"); + + if [[ $github_true != "" ]]; then + + # Always use https, and remove anchors + homepage_url__for__vcs_checkout_command="https://github.com$(echo $github_true | sed "s|https://github.com||; s|http://github.com||; s|#|\n|;" | head -n 1)"; + # Remove trailing slash + homepage_url__for__vcs_checkout_command="${homepage_url__for__vcs_checkout_command%/}" + vcs_checkout_command="git clone $homepage_url__for__vcs_checkout_command.git"; + fi + + if [ "$name" == "GNU LibreJS" ]; then + + is_gnu="Yes"; + + else + + is_gnu="No"; + + fi + + support_url=$(jq .results[$line].support_url $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + + # List JSON structure: cat json/$file | js '.' + # XML value "homepage" is not always set in AMO API, but the XML require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically. + echo "{{Entry" > wiki/$id.wiki + + # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341 + echo "|Name=$name +|Short description=$short_description +|Full description=$full_description +|Homepage URL=$homepage_url +|Extension of=$extension_of +|VCS checkout command=$vcs_checkout_command +|Version identifier=$version +|Version date=$version_date +|Version download=https://addons.mozilla.org/firefox/downloads/latest/$id/addon-$version_id-latest.xpi +|Last review by=$last_review_by +|Last review date=$last_review_date +|Submitted by=$submitted_by +|Submitted date=2018/06/20 +|Is GNU=$is_gnu +}}" >> wiki/$id.wiki + + + # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/ + echo "{{Project license +|License=$license +|License copyright=$license_copyright +}}" >> wiki/$id.wiki + + if [[ "$support_url" != "" ]]; + then + + echo "{{Resource +|Resource audience=Users +|Resource kind=Support +|Resource URL=$support_url +}}" >> "wiki/$id.wiki" + + fi + + + echo "|- +| [[$name]] +| $short_description +| [[License:$license|$license]] +| $average_daily_users +| $bayesian_average__simple +| $version_date" >> "wiki/IceCat WebExtensions (proposed).wiki" + + + + + fi + + ((line++)) + done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt + echo "|}" >> "wiki/IceCat WebExtensions (proposed).wiki" + ;; --download-free-webextensions-for-gnu-and-linux) @@ -393,8 +648,8 @@ OPTIONS ((line++)) - # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). - wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" + # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). + wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" done < ../license.name.enUS.txt @@ -516,18 +771,24 @@ $count_reports reports to submit"; ;; - --all) - - $0 --fresh-build "$2" - $0 --make-repository-list "$2" - $0 --make-collection-list "$2" - $0 --make-custom-list "$2" - $0 --make-search-list "$2" - $0 --merge-lists "$2" - # $0 --download-licenses-json "$2" - # $0 --download-free-webextensions-for-gnu-and-linux "$2" - # $0 --verify-license-copy "$2" + --make-xml) + # Deprecated. + # sha1sum file.xml + # However. It's not possible to insert the checsum in <sha1> for the page since it will break the checksum itself. + # Get number of edits: + revid="$(wget -qO- "https://directory.fsf.org/w/api.php?action=query&list=recentchanges&rclimit=1&format=json" | jq .query.recentchanges[0].revid)"; + xml_header="$(wget -qO- "https://directory.fsf.org/wiki/Special:Export/Main_Page" | head -n 42)"; + + ;; + + --all-recommended) + $0 --fresh-build "$2" + $0 --make-search-list "$2" + $0 --merge-lists "$2" + $0 --get-licenses "$2" + $0 --merged-free "$2" + $0 --make-wiki "$2" ;; diff --git a/addons.mozilla.org-fsd/get-data/license-reporter (another copy) b/addons.mozilla.org-fsd/get-data/license-reporter (another copy) deleted file mode 100755 index a297883..0000000 --- a/addons.mozilla.org-fsd/get-data/license-reporter (another copy) +++ /dev/null @@ -1,528 +0,0 @@ -#!/usr/bin/env bash - -# This file is part of license-reporter -# Copyright (C) 2017, 2018 David Hedlund -# -# license-reporter is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# license-reporter is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -#if [ "$data" == "search" ]; then search_page_variable='.results[]'; fi - - -# Specifically for titles: en-US is sometimes unused, en-GB used to be used then. -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.name.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-GB|enGB|g" "$json_file" | jq "$search_page_variable.name.enGB" | sed "s|^\"||; s|\"$||;" - -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.homepage.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_email.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_url.enUS" | sed "s|^\"||; s|\"$||;" - - -# Licenses -# sed "s|en-US|enUS|g" "$slug.json" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;" - -if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi - -minimum_average_daily_users="1000000"; - -SCRIPTSRC=$(readlink -f "$0" || echo "$0") -RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .) -filename=$(basename "$0"); -export -p SCRIPTSRC RUN_PATH minimum_average_daily_users debug - -if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi -if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi - -if [[ $exit == true ]]; then exit 1; fi - - -function check_files { - - if [ ! -f "$RUN_PATH/license-reporter-$check_type" ]; then - - echo "$RUN_PATH/license-reporter-$check_type doesn't exist."; - echo "Exiting" && exit - - else - - - # Make sure all links are using SSL - if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-$check_type; then - - echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)." - echo "Exiting" && exit - - fi - - fi - -} - - - -case "$1" in - - ""|-help) - - [ "$1" = "" ] && echo "Usage: $filename [--option] [--debug] - -OPTIONS - $0 --all - $0 --fresh-build - $0 --make-repository-list - Downloads add-on data for free add-ons from collections shared by defined users in license-reporter-repositories. Example: - https://addons.mozilla.org/en-US/firefox/collections/mozilla/ - - $0 --make-collection-list - $0 --make-custom-list - $0 --make-search-list - $0 --merge-lists - $0 --download-licenses-json - $0 --download-free-webextensions-for-gnu-and-linux - $0 --verify-license-copy -" && exit 1 - - ;; - - --fresh-build) - - if [ -d "build" ]; then - - rm -fr build - - fi - - mkdir -p build - - ;; - - --make-repository-list) - - # It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs. - # This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync - # your addons.mozilla.org collection list with a separate text file. - - # https://github.com/mozilla/addons/issues/722 - # Cannot list collections in desired formats - # Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list). - # So we have to parse it from HTML. - - - # Get the links to the collections - - function get_repositories { - - check_type="repositories"; - check_files - - cd build/ || exit - - for i in $(cat ../license-reporter-repositories); do - - user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;"); - rm -f repository-$user - - - function wget_repository { - - match="^ <a href=\"/en-US/firefox/collections/" - wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user - - } - - # Figure out if there are several pages - number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||"); - - if [ -z "$number_of_pages" ]; then - - echo "$i doesn't have any sub pages"; - wget_repository - - else - - echo "$i has sub pages"; - - page_number=1; - - while [ $page_number -le $number_of_pages ] - do - - wget_repository__page_extension="?page=$page_number"; - echo "$wget_repository__page_extension" - wget_repository - page_number=$(( $page_number + 1 )) - - done - - fi - - done - } - get_repositories - - - # Download the JSON files from the collections - - cd .. - for x in $(find build/ -name repository* | sort); do - - for i in $(cat $x); do - - collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); - wget $i"format%3Ajson" -O "build/collection-$collection_name.json" - - done - - done - - ;; - - --make-collection-list) - - check_type="collections"; - check_files - - cd build/ || exit - for i in $(cat ../license-reporter-collections); do - - # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson - collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); - wget $i"format%3Ajson" -O "collection-$collection_name.json" - - done - - ;; - - --make-custom-list) - - check_type="custom"; - check_files - - src/download-custom - - ;; - - --make-search-list) - - - cd build/ || exit - mkdir json - cd json || exit - - # API documentation: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html - - function foo() { - - # Get the most popular WebExtensions - wget "https://addons.mozilla.org/api/v3/addons/search/?sort=users&page=$page&tag=firefox57" -O "index.html?sort=users&page=$page&tag=firefox57.json" - - } - - page="1" - foo - - - - if [ "$debug" == false ]; then - - while [ "$(jq ".results[].average_daily_users" "index.html?sort=users&page=$page&tag=firefox57.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do - - ((page++)) - foo - - done - - fi - - - # Sort by file modification time stamp for the downloaded JSON files - find index*.json -type f -printf "%Tc %p\n" | sort -n | awk '{print $NF}' | while read -r file; do - - - line="0"; - - for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do - - # Exclude add-ons with to low average daily users from index.html* - if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then - - echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file" >> ../merged-search.txt - - - fi - - ((line++)) - - done - - - done - - ;; - - --merge-lists) - - cd build || exit - - cat merged-search.txt >> MERGED-ALL.txt - cat merged-custom.txt >> MERGED-ALL.txt - - ;; - --get-licenses) - - cd build/ || exit - mkdir json/current_versions - cd json/current_versions - - line="0"; - while IFS= read -r table; do - - ((line++)) - - id=$(echo $table | awk '{print $2}'); - file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}'); - - wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json" - - done < $RUN_PATH/build/MERGED-ALL.txt - - ;; - --merged-free) - - cd build/ || exit - cp -a MERGED-ALL.txt MERGED-ALL-freedom_status.txt - - line="0"; - while IFS= read -r table; do - - ((line++)) - - id=$(echo $table | awk '{print $2}'); - license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - - # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". - if - [ "$license" == "GNU General Public License, version 3.0" ] || - [ "$license" == "GNU General Public License, version 2.0" ] || - [ "$license" == "GNU Lesser General Public License, version 3.0" ] || - [ "$license" == "GNU Lesser General Public License, version 2.1" ] || - [ "$license" == "Mozilla Public License, version 2.0" ] || - [ "$license" == "Mozilla Public License Version 1.1" ] || - [ "$license" == "BSD License" ] || - [ "$license" == "MIT/X11 License" ] - then - - # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the JSON file for index.html?sort=users&page=1&tag=firefox57.json for example - sed -i "$line""s/$/\\tfree/" MERGED-ALL-freedom_status.txt - - else - - sed -i "$line""s/$/\\tnonfree/" MERGED-ALL-freedom_status.txt - - fi - - done < $RUN_PATH/build/MERGED-ALL.txt - - ;; - --generate-proposed-list) - - line="0"; - while IFS= read -r table; do - - ((line++)) - - id=$(echo $table | awk '{print $2}'); - source=$(echo $table | awk '{print $3}'); - license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - freedom_status=$(echo $table | awk '{print $4}'); - name=$(jq .results[$line].name $RUN_PATH/build/json/$source | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - - if [ "$name" == "null" ]; then - - name=$(jq .results[$line].name $RUN_PATH/build/json/$source | sed "s|en-GB|enGB|;" | jq .enGB | sed "s|^\"||; s|\"$||;"); - - fi - - - # Clean titles. Example of bad titles: "Ghostery – Privacy Ad Blocker" - if [[ $name = *"–"* ]]; then - - name=$(echo $name | sed 's/–/\n/g; s/-/\n/g' | head -n 1); - - fi - - if [ "$freedom_status" == "free" ]; then - echo "{{Checkme item -|Section= -|Package=$name -|Package note= -}}" - fi - - done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt - - ;; - - --download-free-webextensions-for-gnu-and-linux) - - cd build || exit - rm -fr free_webextensions_for_GNU_and_Linux - mkdir free_webextensions_for_GNU_and_Linux - cd free_webextensions_for_GNU_and_Linux || exit - - line="0"; - - while IFS= read -r license; do - - ((line++)) - - # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). - wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" - - done < ../license.name.enUS.txt - - - ;; - --verify-license-copy) - - cd build || exit - rm -fr VERIFY-LICENSE-COPY.txt - cd free_webextensions_for_GNU_and_Linux || exit - - - while IFS= read -r id; do - - - - - if [ ! -f "addon-$id-latest.xpi" ]; then - - echo "Nonfree" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE$"; then - - echo "LICENSE" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.txt$"; then - - echo "LICENSE.txt" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.md$"; then - - echo "LICENSE.md" >> ../VERIFY-LICENSE-COPY.txt - - - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING$"; then - - echo "COPYING" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.txt$"; then - - echo "COPYING.txt" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.md$"; then - - echo "COPYING.md" >> ../VERIFY-LICENSE-COPY.txt - - else - - echo "Request" >> ../VERIFY-LICENSE-COPY.txt - - fi - - done < ../id.txt - - - - line="0"; - - while IFS= read -r status; do - - - ((line++)) - - average_daily_users="$(sed -n ${line}p ../average_daily_users.txt)"; - id="$(sed -n ${line}p ../id.txt)"; - slug="$(sed -n ${line}p ../slug.txt)"; - license_name_en_US="$(sed -n ${line}p ../license.name.enUS.txt)"; - homepage_en_US="$(sed -n ${line}p ../homepage.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; - support_email_en_US="$(sed -n ${line}p ../support_email.en-US.txt)"; - support_url_en_US="$(sed -n ${line}p ../support_url.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; - - - - if [ "$license_name_en_US" == "GNU General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/gpl-3.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU General Public License, version 2.0" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/lgpl-3.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 2.1" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "Mozilla Public License, version 2.0" ]; then license_txt="https://www.mozilla.org/media/MPL/2.0/index.txt"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "Mozilla Public License Version 1.1" ]; then license_txt="https://www.mozilla.org/media/MPL/1.1/index.txt"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "BSD License" ]; then license_txt="https://directory.fsf.org/wiki?title=License:FreeBSD"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "MIT/X11 License" ]; then license_txt="https://directory.fsf.org/wiki/License:X11"; license_file_name="LICENSE"; - fi - - - if [ "$(sed -n ${line}p ../name.en-US.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-US.txt)"; - elif [ "$(sed -n ${line}p ../name.en-GB.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-GB.txt)"; - else name="$slug"; - fi - - - if [ "$status" = "Request" ]; then - - ((count_reports++)) - - # Authors that don't provide contact information will be ignored, its impossible to track conversations in Reviews anyway, because they will be deleted. - - if [ "$homepage_en_US" != "null" ] || [ "$support_url_en_US" != "null" ] || [ "$support_email_en_US" != "null" ]; then - - decimals="$(echo "$average_daily_users" | sed -r ':L;s=\b([0-9]+)([0-9]{3})\b=\1,\2=g;t L')"; - - echo " -$decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/addon/$slug/ - Request: \"You have distributed $name under $license_name_en_US on https://addons.mozilla.org/en-US/firefox/addon/$slug/, but you have not added a copy of the license. can you please download the text of the license from $license_txt and add save it to file name $license_file_name and put it in the root directory of you program?\" by contacting:"; - - if [ "$homepage_en_US" != "null" ]; then echo " * Homepage (please submit Pull Request for $license_file_name if possible): $homepage_en_US"; fi - if [ "$support_url_en_US" != "null" ]; then echo " * Support URL (please submit Pull Request for $license_file_name if possible): $support_url_en_US"; fi - if [ "$support_email_en_US" != "null" ]; then echo " * Support E-mail: $support_email_en_US"; fi - - fi - - fi - - done < ../VERIFY-LICENSE-COPY.txt - - echo " -######################### -$count_reports reports to submit"; - - ;; - - --all) - - $0 --fresh-build "$2" - $0 --make-repository-list "$2" - $0 --make-collection-list "$2" - $0 --make-custom-list "$2" - $0 --make-search-list "$2" - $0 --merge-lists "$2" - # $0 --download-licenses-json "$2" - # $0 --download-free-webextensions-for-gnu-and-linux "$2" - # $0 --verify-license-copy "$2" - - ;; - - -esac diff --git a/addons.mozilla.org-fsd/get-data/license-reporter (copy) b/addons.mozilla.org-fsd/get-data/license-reporter (copy) deleted file mode 100755 index dd46854..0000000 --- a/addons.mozilla.org-fsd/get-data/license-reporter (copy) +++ /dev/null @@ -1,442 +0,0 @@ -#!/usr/bin/env bash - -# This file is part of license-reporter -# Copyright (C) 2017, 2018 David Hedlund -# -# license-reporter is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# license-reporter is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -#if [ "$data" == "search" ]; then search_page_variable='.results[]'; fi - - -# Specifically for titles: en-US is sometimes unused, en-GB used to be used then. -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.name.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-GB|enGB|g" "$json_file" | jq "$search_page_variable.name.enGB" | sed "s|^\"||; s|\"$||;" - -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.homepage.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_email.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_url.enUS" | sed "s|^\"||; s|\"$||;" - - -# Licenses -# sed "s|en-US|enUS|g" "$slug.json" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;" - -if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi - -minimum_average_daily_users="10000"; - -SCRIPTSRC=$(readlink -f "$0" || echo "$0") -RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .) -filename=$(basename "$0"); -export -p SCRIPTSRC RUN_PATH minimum_average_daily_users debug - -if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi -if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi - -if [[ $exit == true ]]; then exit 1; fi - - -function check_files { - - if [ ! -f "$RUN_PATH/license-reporter-$check_type" ]; then - - echo "$RUN_PATH/license-reporter-$check_type doesn't exist."; - echo "Exiting" && exit - - else - - - # Make sure all links are using SSL - if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-$check_type; then - - echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)." - echo "Exiting" && exit - - fi - - fi - -} - - - -case "$1" in - - ""|-help) - - [ "$1" = "" ] && echo "Usage: $filename [--option] [--debug] - -OPTIONS - $0 --all - $0 --fresh-build - $0 --make-repository-list - $0 --make-collection-list - $0 --make-custom-list - $0 --make-search-list - $0 --merge-lists - $0 --download-licenses-json - $0 --download-free-webextensions-for-gnu-and-linux - $0 --verify-license-copy -" && exit 1 - - ;; - - --fresh-build) - - if [ -d "build" ]; then - - rm -fr build - - fi - - mkdir -p build - - ;; - - --make-repository-list) - - # It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs. - # This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync - # your addons.mozilla.org collection list with a separate text file. - - # https://github.com/mozilla/addons/issues/722 - # Cannot list collections in desired formats - # Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list). - # So we have to parse it from HTML. - - - function get_repositories { - - check_type="repositories"; - check_files - - cd build/ || exit - - for i in $(cat ../license-reporter-repositories); do - - user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;"); - rm -f repository-$user - - - function wget_repository { - - match="^ <a href=\"/en-US/firefox/collections/" - wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user - - } - - # Figure out if there are several pages - number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||"); - - if [ -z "$number_of_pages" ]; then - - echo "$i doesn't have any sub pages"; - wget_repository - - else - - echo "$i has sub pages"; - - page_number=1; - - while [ $page_number -le $number_of_pages ] - do - - wget_repository__page_extension="?page=$page_number"; - echo "$wget_repository__page_extension" - wget_repository - page_number=$(( $page_number + 1 )) - - done - - fi - - done - } - -# get_repositories - - for x in $(find . -name repository* ); do - - echo $x; - - done - - ;; - - --make-collection-list) - - check_type="collections"; - check_files - - cd build/ || exit - for i in $(cat ../license-reporter-collections); do - - # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson - collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); - wget $i"format%3Ajson" -O "collection-$collection_name.json" - - done - - ;; - - --make-custom-list) - - check_type="custom"; - check_files - - src/download-custom - - ;; - - --make-search-list) - - - cd build/ || exit - mkdir json - cd json || exit - - # API documentation: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html - - function foo() { - - # Get the most popular WebExtensions - wget "https://addons.mozilla.org/api/v3/addons/search/?sort=users&page=$page&tag=firefox57" -O "index.html?sort=users&page=$page&tag=firefox57.json" - - } - - page="1" - foo - - - - if [ "$debug" == false ]; then - - while [ "$(jq ".results[].average_daily_users" "index.html?sort=users&page=$page&tag=firefox57.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do - - ((page++)) - foo - - done - - fi - - - find index*.json -type f -printf "%f\n" | while read -r file; do - - - line="0"; - - for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do - - # Exclude add-ons with to low average daily users from index.html* - if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then - - echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file" >> ../merged-search.txt - - - fi - - ((line++)) - - done - - - done - - ;; - - --merge-lists) - - cd build || exit - - cat merged-search.txt >> MERGED-ALL.txt - cat merged-custom.txt >> MERGED-ALL.txt - - ;; - --get-licenses) - - # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "current_versions/$(jq ".results[$line].current_version.id" "$file").json" - - ;; - --download-free-webextensions-for-gnu-and-linux) - - cd build || exit - rm -fr free_webextensions_for_GNU_and_Linux - mkdir free_webextensions_for_GNU_and_Linux - cd free_webextensions_for_GNU_and_Linux || exit - - line="0"; - - while IFS= read -r license; do - - ((line++)) - id="$(sed -n ${line}p ../id.txt)"; - slug="$(sed -n ${line}p ../slug.txt)"; - - # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". - if - [ "$license" == "GNU General Public License, version 3.0" ] || - [ "$license" == "GNU General Public License, version 2.0" ] || - [ "$license" == "GNU Lesser General Public License, version 3.0" ] || - [ "$license" == "GNU Lesser General Public License, version 2.1" ] || - [ "$license" == "Mozilla Public License, version 2.0" ] || - [ "$license" == "Mozilla Public License Version 1.1" ] || - [ "$license" == "BSD License" ] || - [ "$license" == "MIT/X11 License" ] - then - - # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). - wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" - - - fi - - - done < ../license.name.enUS.txt - - - ;; - --verify-license-copy) - - cd build || exit - rm -fr VERIFY-LICENSE-COPY.txt - cd free_webextensions_for_GNU_and_Linux || exit - - - while IFS= read -r id; do - - - - - if [ ! -f "addon-$id-latest.xpi" ]; then - - echo "Nonfree" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE$"; then - - echo "LICENSE" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.txt$"; then - - echo "LICENSE.txt" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.md$"; then - - echo "LICENSE.md" >> ../VERIFY-LICENSE-COPY.txt - - - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING$"; then - - echo "COPYING" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.txt$"; then - - echo "COPYING.txt" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.md$"; then - - echo "COPYING.md" >> ../VERIFY-LICENSE-COPY.txt - - else - - echo "Request" >> ../VERIFY-LICENSE-COPY.txt - - fi - - done < ../id.txt - - - - line="0"; - - while IFS= read -r status; do - - - ((line++)) - - average_daily_users="$(sed -n ${line}p ../average_daily_users.txt)"; - id="$(sed -n ${line}p ../id.txt)"; - slug="$(sed -n ${line}p ../slug.txt)"; - license_name_en_US="$(sed -n ${line}p ../license.name.enUS.txt)"; - homepage_en_US="$(sed -n ${line}p ../homepage.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; - support_email_en_US="$(sed -n ${line}p ../support_email.en-US.txt)"; - support_url_en_US="$(sed -n ${line}p ../support_url.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; - - - - if [ "$license_name_en_US" == "GNU General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/gpl-3.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU General Public License, version 2.0" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/lgpl-3.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 2.1" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "Mozilla Public License, version 2.0" ]; then license_txt="https://www.mozilla.org/media/MPL/2.0/index.txt"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "Mozilla Public License Version 1.1" ]; then license_txt="https://www.mozilla.org/media/MPL/1.1/index.txt"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "BSD License" ]; then license_txt="https://directory.fsf.org/wiki?title=License:FreeBSD"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "MIT/X11 License" ]; then license_txt="https://directory.fsf.org/wiki/License:X11"; license_file_name="LICENSE"; - fi - - - if [ "$(sed -n ${line}p ../name.en-US.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-US.txt)"; - elif [ "$(sed -n ${line}p ../name.en-GB.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-GB.txt)"; - else name="$slug"; - fi - - - if [ "$status" = "Request" ]; then - - ((count_reports++)) - - # Authors that don't provide contact information will be ignored, its impossible to track conversations in Reviews anyway, because they will be deleted. - - if [ "$homepage_en_US" != "null" ] || [ "$support_url_en_US" != "null" ] || [ "$support_email_en_US" != "null" ]; then - - decimals="$(echo "$average_daily_users" | sed -r ':L;s=\b([0-9]+)([0-9]{3})\b=\1,\2=g;t L')"; - - echo " -$decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/addon/$slug/ - Request: \"You have distributed $name under $license_name_en_US on https://addons.mozilla.org/en-US/firefox/addon/$slug/, but you have not added a copy of the license. can you please download the text of the license from $license_txt and add save it to file name $license_file_name and put it in the root directory of you program?\" by contacting:"; - - if [ "$homepage_en_US" != "null" ]; then echo " * Homepage (please submit Pull Request for $license_file_name if possible): $homepage_en_US"; fi - if [ "$support_url_en_US" != "null" ]; then echo " * Support URL (please submit Pull Request for $license_file_name if possible): $support_url_en_US"; fi - if [ "$support_email_en_US" != "null" ]; then echo " * Support E-mail: $support_email_en_US"; fi - - fi - - fi - - done < ../VERIFY-LICENSE-COPY.txt - - echo " -######################### -$count_reports reports to submit"; - - ;; - - --all) - - $0 --fresh-build "$2" - $0 --make-repository-list "$2" - $0 --make-collection-list "$2" - $0 --make-custom-list "$2" - $0 --make-search-list "$2" - $0 --merge-lists "$2" - # $0 --download-licenses-json "$2" - # $0 --download-free-webextensions-for-gnu-and-linux "$2" - # $0 --verify-license-copy "$2" - - ;; - - -esac diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-collections b/addons.mozilla.org-fsd/get-data/license-reporter-collections index 0e3f693..1d58d53 100644 --- a/addons.mozilla.org-fsd/get-data/license-reporter-collections +++ b/addons.mozilla.org-fsd/get-data/license-reporter-collections @@ -1 +1,2 @@ +https://addons.mozilla.org/en-US/firefox/collections/NateN1222/librejs-compatibility-add-ons/ https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/ diff --git a/addons.mozilla.org-fsd/get-data/license-reporter~ b/addons.mozilla.org-fsd/get-data/license-reporter~ deleted file mode 100755 index d843b24..0000000 --- a/addons.mozilla.org-fsd/get-data/license-reporter~ +++ /dev/null @@ -1,377 +0,0 @@ -#!/usr/bin/env bash - -# This file is part of license-reporter -# Copyright (C) 2017 David Hedlund -# -# license-reporter is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# license-reporter is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -#if [ "$data" == "search" ]; then search_page_variable='.results[]'; fi - - -# Specifically for titles: en-US is sometimes unused, en-GB used to be used then. -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.name.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-GB|enGB|g" "$json_file" | jq "$search_page_variable.name.enGB" | sed "s|^\"||; s|\"$||;" - -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.homepage.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_email.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_url.enUS" | sed "s|^\"||; s|\"$||;" - - -# Licenses -# sed "s|en-US|enUS|g" "$slug.json" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;" - -if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi - -minimum_average_daily_users="10000"; - -SCRIPTSRC=$(readlink -f "$0" || echo "$0") -RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .) -filename=$(basename "$0"); -export -p SCRIPTSRC RUN_PATH minimum_average_daily_users debug - -if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi -if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi - -if [[ $exit == true ]]; then exit 1; fi - - - - -case "$1" in - - ""|-help) - - [ "$1" = "" ] && echo "Usage: $filename [--option] [--debug] - -OPTIONS - $0 --all - $0 --fresh-build - $0 --make-collection-list - $0 --make-custom-list - $0 --make-search-list - $0 --merge-lists - $0 --download-licenses-json - $0 --download-free-webextensions-for-gnu-and-linux - $0 --verify-license-copy -" && exit 1 - - ;; - - --fresh-build) - - if [ -d "build" ]; then - - rm -fr build - - fi - - mkdir -p build - - ;; - - --make-repository-list) - - # It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs. - # This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync - # your addons.mozilla.org collection list with a separate text file. - - # Cannot list collections in desired formats - https://github.com/mozilla/addons/issues/722 - - ;; - - --make-collection-list) - - if [ ! -f "$RUN_PATH/license-reporter-collections" ]; then - - echo "$RUN_PATH/license-reporter-collections doesn't exist. Skipping custom list."; - - else - - - # Make sure all repositories are using SSL - if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-repository; then - - echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)." - - else - - cd build/ || exit - for i in $(cat ../license-reporter-collections); do - - # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextension-replacements/format%3Ajson - collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); - wget $i"format%3Ajson" -O "collection-$collection_name.json" - - done - - fi - - fi - - ;; - - --make-custom-list) - - if [ ! -f "$RUN_PATH/license-reporter-custom" ]; then - - echo "$RUN_PATH/license-reporter-custom doesn't exist. Skipping custom list."; - - else - - src/download-custom - - fi - - ;; - - --make-search-list) - - - cd build/ || exit - mkdir json - cd json || exit - - # API documentation: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html - - function foo() { - - # Get the most popular WebExtensions - wget "https://addons.mozilla.org/api/v3/addons/search/?sort=users&page=$page&tag=firefox57" -O "index.html?sort=users&page=$page&tag=firefox57.json" - - } - - page="1" - foo - - - - if [ "$debug" == false ]; then - - while [ "$(jq ".results[].average_daily_users" "index.html?sort=users&page=$page&tag=firefox57.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do - - ((page++)) - foo - - done - - fi - - - find index*.json -type f -printf "%f\n" | while read -r file; do - - - line="0"; - - for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do - - # Exclude add-ons with to low average daily users from index.html* - if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then - - echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file" >> ../merged-search.txt - - - fi - - ((line++)) - - done - - - done - - ;; - - --merge-lists) - - cd build || exit - - cat merged-search.txt >> MERGED-ALL.txt - cat merged-custom.txt >> MERGED-ALL.txt - - ;; - --get-licenses) - - # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "current_versions/$(jq ".results[$line].current_version.id" "$file").json" - - ;; - --download-free-webextensions-for-gnu-and-linux) - - cd build || exit - rm -fr free_webextensions_for_GNU_and_Linux - mkdir free_webextensions_for_GNU_and_Linux - cd free_webextensions_for_GNU_and_Linux || exit - - line="0"; - - while IFS= read -r license; do - - ((line++)) - id="$(sed -n ${line}p ../id.txt)"; - slug="$(sed -n ${line}p ../slug.txt)"; - - # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". - if - [ "$license" == "GNU General Public License, version 3.0" ] || - [ "$license" == "GNU General Public License, version 2.0" ] || - [ "$license" == "GNU Lesser General Public License, version 3.0" ] || - [ "$license" == "GNU Lesser General Public License, version 2.1" ] || - [ "$license" == "Mozilla Public License, version 2.0" ] || - [ "$license" == "Mozilla Public License Version 1.1" ] || - [ "$license" == "BSD License" ] || - [ "$license" == "MIT/X11 License" ] - then - - # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). - wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" - - - fi - - - done < ../license.name.enUS.txt - - - ;; - --verify-license-copy) - - cd build || exit - rm -fr VERIFY-LICENSE-COPY.txt - cd free_webextensions_for_GNU_and_Linux || exit - - - while IFS= read -r id; do - - - - - if [ ! -f "addon-$id-latest.xpi" ]; then - - echo "Nonfree" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE$"; then - - echo "LICENSE" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.txt$"; then - - echo "LICENSE.txt" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.md$"; then - - echo "LICENSE.md" >> ../VERIFY-LICENSE-COPY.txt - - - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING$"; then - - echo "COPYING" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.txt$"; then - - echo "COPYING.txt" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.md$"; then - - echo "COPYING.md" >> ../VERIFY-LICENSE-COPY.txt - - else - - echo "Request" >> ../VERIFY-LICENSE-COPY.txt - - fi - - done < ../id.txt - - - - line="0"; - - while IFS= read -r status; do - - - ((line++)) - - average_daily_users="$(sed -n ${line}p ../average_daily_users.txt)"; - id="$(sed -n ${line}p ../id.txt)"; - slug="$(sed -n ${line}p ../slug.txt)"; - license_name_en_US="$(sed -n ${line}p ../license.name.enUS.txt)"; - homepage_en_US="$(sed -n ${line}p ../homepage.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; - support_email_en_US="$(sed -n ${line}p ../support_email.en-US.txt)"; - support_url_en_US="$(sed -n ${line}p ../support_url.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; - - - - if [ "$license_name_en_US" == "GNU General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/gpl-3.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU General Public License, version 2.0" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/lgpl-3.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 2.1" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "Mozilla Public License, version 2.0" ]; then license_txt="https://www.mozilla.org/media/MPL/2.0/index.txt"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "Mozilla Public License Version 1.1" ]; then license_txt="https://www.mozilla.org/media/MPL/1.1/index.txt"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "BSD License" ]; then license_txt="https://directory.fsf.org/wiki?title=License:FreeBSD"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "MIT/X11 License" ]; then license_txt="https://directory.fsf.org/wiki/License:X11"; license_file_name="LICENSE"; - fi - - - if [ "$(sed -n ${line}p ../name.en-US.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-US.txt)"; - elif [ "$(sed -n ${line}p ../name.en-GB.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-GB.txt)"; - else name="$slug"; - fi - - - if [ "$status" = "Request" ]; then - - ((count_reports++)) - - # Authors that don't provide contact information will be ignored, its impossible to track conversations in Reviews anyway, because they will be deleted. - - if [ "$homepage_en_US" != "null" ] || [ "$support_url_en_US" != "null" ] || [ "$support_email_en_US" != "null" ]; then - - decimals="$(echo "$average_daily_users" | sed -r ':L;s=\b([0-9]+)([0-9]{3})\b=\1,\2=g;t L')"; - - echo " -$decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/addon/$slug/ - Request: \"You have distributed $name under $license_name_en_US on https://addons.mozilla.org/en-US/firefox/addon/$slug/, but you have not added a copy of the license. can you please download the text of the license from $license_txt and add save it to file name $license_file_name and put it in the root directory of you program?\" by contacting:"; - - if [ "$homepage_en_US" != "null" ]; then echo " * Homepage (please submit Pull Request for $license_file_name if possible): $homepage_en_US"; fi - if [ "$support_url_en_US" != "null" ]; then echo " * Support URL (please submit Pull Request for $license_file_name if possible): $support_url_en_US"; fi - if [ "$support_email_en_US" != "null" ]; then echo " * Support E-mail: $support_email_en_US"; fi - - fi - - fi - - done < ../VERIFY-LICENSE-COPY.txt - - echo " -######################### -$count_reports reports to submit"; - - ;; - - --all) - - $0 --fresh-build "$2" - $0 --make-collection-list "$2" - $0 --make-custom-list "$2" - $0 --make-search-list "$2" - $0 --merge-lists "$2" - # $0 --download-licenses-json "$2" - # $0 --download-free-webextensions-for-gnu-and-linux "$2" - # $0 --verify-license-copy "$2" - - ;; - - -esac diff --git a/addons.mozilla.org-fsd/sync-data/fsd-amo-bot b/addons.mozilla.org-fsd/sync-data/fsd-amo-bot index 14b008d..4947ae7 100755 --- a/addons.mozilla.org-fsd/sync-data/fsd-amo-bot +++ b/addons.mozilla.org-fsd/sync-data/fsd-amo-bot @@ -115,168 +115,6 @@ case "$1" in ;; - --make-xml) - - if [ ! -d "build/fsd" ]; then mkdir -p build/fsd; fi - - # Get number of edits: - wget "https://directory.fsf.org/w/api.php?action=query&list=recentchanges&rclimit=1" -O fsdedits.xml - oldrevid="$(grep revid fsdedits.xml | sed "s|"||g; s|revid|\nrevid|; s|old_revid|\n|" | grep revid | sed "s|revid=||")" - newrevid=$(expr 1 + $oldrevid) - - cd build/ || exit - - while read url; do - - retrive - # List JSON structure: cat amo/json/$slug.json | js '.' - # XML value "homepage" is not always set in AMO API, but the FSD require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically. - echo "<mediawiki xmlns=\"http://www.mediawiki.org/xml/export-0.7/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.mediawiki.org/xml/export-0.7/ http://www.mediawiki.org/xml/export-0.7.xsd\" version=\"0.7\" xml:lang=\"en\"> - <siteinfo> - <sitename>Free Software Directory</sitename> - <base>https://directory.fsf.org/wiki/Main_Page</base> - <generator>MediaWiki 1.20.2</generator> - <case>first-letter</case> - <namespaces> - <namespace key=\"-2\" case=\"first-letter\">Media</namespace> - <namespace key=\"-1\" case=\"first-letter\">Special</namespace> - <namespace key=\"0\" case=\"first-letter\" /> - <namespace key=\"1\" case=\"first-letter\">Talk</namespace> - <namespace key=\"2\" case=\"first-letter\">User</namespace> - <namespace key=\"3\" case=\"first-letter\">User talk</namespace> - <namespace key=\"4\" case=\"first-letter\">Free Software Directory</namespace> - <namespace key=\"5\" case=\"first-letter\">Free Software Directory talk</namespace> - <namespace key=\"6\" case=\"first-letter\">File</namespace> - <namespace key=\"7\" case=\"first-letter\">File talk</namespace> - <namespace key=\"8\" case=\"first-letter\">MediaWiki</namespace> - <namespace key=\"9\" case=\"first-letter\">MediaWiki talk</namespace> - <namespace key=\"10\" case=\"first-letter\">Template</namespace> - <namespace key=\"11\" case=\"first-letter\">Template talk</namespace> - <namespace key=\"12\" case=\"first-letter\">Help</namespace> - <namespace key=\"13\" case=\"first-letter\">Help talk</namespace> - <namespace key=\"14\" case=\"first-letter\">Category</namespace> - <namespace key=\"15\" case=\"first-letter\">Category talk</namespace> - <namespace key=\"102\" case=\"first-letter\">Property</namespace> - <namespace key=\"103\" case=\"first-letter\">Property talk</namespace> - <namespace key=\"106\" case=\"first-letter\">Form</namespace> - <namespace key=\"107\" case=\"first-letter\">Form talk</namespace> - <namespace key=\"108\" case=\"first-letter\">Concept</namespace> - <namespace key=\"109\" case=\"first-letter\">Concept talk</namespace> - <namespace key=\"170\" case=\"first-letter\">Filter</namespace> - <namespace key=\"171\" case=\"first-letter\">Filter talk</namespace> - <namespace key=\"500\" case=\"first-letter\">Review</namespace> - <namespace key=\"501\" case=\"first-letter\">Review talk</namespace> - <namespace key=\"502\" case=\"first-letter\">License</namespace> - <namespace key=\"503\" case=\"first-letter\">License talk</namespace> - <namespace key=\"510\" case=\"first-letter\">Collection</namespace> - <namespace key=\"511\" case=\"first-letter\">Collection talk</namespace> - <namespace key=\"520\" case=\"first-letter\">Org</namespace> - <namespace key=\"521\" case=\"first-letter\">Org talk</namespace> - </namespaces> - </siteinfo> - <page> - <title>IceCat/$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.name.enUS' | sed "s|^\"||; s|\"$||;")</title> - <revision> - <id>$newrevid</id> - <timestamp>$(date -u +%Y-%m-%dT%H:%M:%SZ)</timestamp> - <contributor> - <username>amo-import</username> - <id>1501</id> - </contributor> - <sha1>kj1p9nd2p15bqr2tb7jkihshlxow11y</sha1> - <text xml:space=\"preserve\" bytes=\"2687\">{{Entry" > fsd/$slug.xml - - # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341 - echo "|Name=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.name.enUS' | sed "s|^\"||; s|\"$||;") -|Short description=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.summary.enUS' | sed "s|^\"||; s|\"$||; s|\\\n|\n|g; s|\\\\\"|\"|g;") -|Full description=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.description.enUS' | sed "s|^\"||; s|\"$||; s|\\\n|\n|g; s|\\\\\"|\"|g;") -|Homepage URL=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | sed "s|^\"||; s|\"$||;")" >> fsd/$slug.xml - - - if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.seamonkey')" != "null" ]]; then - set_seamonkey="SeaMonkey,"; - fi - - if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.firefox')" != "null" ]]; then - set_icecat="IceCat,"; - fi - - if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.android')" != "null" ]]; then - set_icecatmobile="IceCatMobile,"; - fi - - if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.mobile')" != "null" ]]; then - set_icecatmobile="IceCatMobile,"; - fi - - if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.thunderbird')" != "null" ]]; then - set_thunderbird="Thunderbird,"; - fi - - echo "|Extension of=$(echo "$set_seamonkey$set_icecat$set_icecatmobile$set_thunderbird" | sed "s|,$||;")" >> fsd/$slug.xml - - - if [[ "$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | grep "github.com")" != "null" ]]; then - echo "|VCS checkout command=git clone $(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | sed "s|^\"||; s|\"$||;")" >> fsd/$slug.xml - fi - - echo "|VCS checkout command=git clone $(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | sed "s|^\"||; s|\"$||;")"; - echo "|Version identifier=$(cat amo/json/$slug.json | jq '.current_version.version' | sed "s|^\"||; s|\"$||;") -|Version date=$(cat amo/json/$slug.json | jq '.last_updated' | sed "s|^\"||; s|\"$||;") -|Version download=https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$(cat amo/json/$slug.json | jq '.current_version.id')-latest.xpi -|Last review by=FSDbot -|Last review date=$(date +"%Y/%m/%d") -|Submitted by=FSDbot -|Submitted date=2017/02/01 -|Is GNU=No -}}" >> fsd/$slug.xml - - - # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/ - echo "{{Project license - -# Stuck here (have to use old API until this have been fixed): https://github.com/mozilla/addons-server/issues/3936 - -|License=$(cat amo/json.versions/$slug.json | sed "s|-||g;" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;") -|License copyright=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.authors[].name' | sed "s|^\"||; s|\"$||;") -}}" >> fsd/$slug.xml - - # Homepage URL can be changed by the community so lets do a backup here - echo "{{Resource -|Resource audience=amo-import -|Resource kind=General -|Resource URL=$url -}}" >> fsd/$slug.xml - - - if [[ "$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.support_url[]' | sed "s|^\"||; s|\"$||;")" != "" ]]; - then - - echo "{{Resource -|Resource audience=Users -|Resource kind=Support -|Resource URL=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.support_url[]' | sed "s|^\"||; s|\"$||;") -}}" >> "fsd/$slug.xml" - - fi - - # Add "E-mail" here when "Support E-mail" have been added to the API: https://github.com/mozilla/addons-server/issues/3901 - - - echo "{{Featured}}</text> - </revision> - </page> -</mediawiki>" >> "fsd/$slug.xml" - - cat "fsd/$slug.xml" - - - - done < ../amo-repo.txt - - - ;; - --all) @@ -284,8 +122,6 @@ case "$1" in $0 --get-xpi-url $0 --get-xpi $0 --xpi-extract - $0 --make-xml - ;; *) |