aboutsummaryrefslogtreecommitdiff
path: root/addons.mozilla.org-fsd/get-data
diff options
context:
space:
mode:
Diffstat (limited to 'addons.mozilla.org-fsd/get-data')
-rwxr-xr-xaddons.mozilla.org-fsd/get-data/license-reporter371
1 files changed, 262 insertions, 109 deletions
diff --git a/addons.mozilla.org-fsd/get-data/license-reporter b/addons.mozilla.org-fsd/get-data/license-reporter
index 47c28a8..9bf4263 100755
--- a/addons.mozilla.org-fsd/get-data/license-reporter
+++ b/addons.mozilla.org-fsd/get-data/license-reporter
@@ -34,7 +34,7 @@
if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi
-minimum_average_daily_users="100000";
+minimum_average_daily_users="10000";
SCRIPTSRC=$(readlink -f "$0" || echo "$0")
RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .)
@@ -85,6 +85,131 @@ function line_status {
}
+function verify_variables {
+
+ if [ -z ${file+x} ] || [ "$file" != "$(echo $table | awk '{print $3}')" ]; then
+
+ line="0"; # should not be 1
+
+ fi
+
+ average_daily_users=$(echo $table | awk '{print $1}');
+ id=$(echo $table | awk '{print $2}');
+ file=$(echo $table | awk '{print $3}');
+ slug=$(echo $table | awk '{print $4}');
+ name=$(echo "$table" | cut -f 5);
+ has_eula=$(echo "$table" | cut -f 6);
+
+
+
+ firefox_max_version="$(jq .results[$line].current_version.compatibility.firefox.max $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|.\*||")";
+ nativeMessaging="$(jq .results[$line].current_version.files[0].permissions $RUN_PATH/build/json/$file | grep "nativeMessaging")";
+
+ # Values are either an integer or "*"
+ re='^[0-9]+$'
+ if [[ $firefox_max_version =~ $re ]] && [ "$firefox_max_version" -ge 60 ] || [ "$firefox_max_version" == '*' ]; then
+
+ # echo "$firefox_max_version"
+ firefox_max_version__pass=true;
+
+ else
+
+ unset firefox_max_version__pass
+
+ fi
+
+ if [ ! -n "$nativeMessaging" ]; then
+
+ nativeMessaging__pass=true;
+
+ else
+
+ nativeMessaging__pass=false;
+
+ fi
+
+ echo "$nativeMessaging__pass"
+
+
+ # curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging
+
+
+ # API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668
+ # The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies.
+
+ # Old solution:
+
+ # 1: Add this to license-reporter-dependencies:
+ #belgium-eid eid-mw
+
+ # 2: Uncomment this code:
+ # depends_on_external_program=false;
+ # while IFS= read -r dependencies; do
+ #
+ # dependency_slug=$(echo $dependencies | awk '{print $1}');
+ # dependency=$(echo "$dependencies" | cut -f 2);
+ #
+ # if [ "$dependency_slug" == "$slug" ]; then
+ #
+ # depends_on_external_program=true;
+ #
+ # fi
+ #
+ # done < $RUN_PATH/license-reporter-dependencies
+
+
+
+
+ approved_program=false;
+ if [ "$has_eula" == "eulafree" ] && [ "$firefox_max_version__pass" = true ] && [ "$nativeMessaging__pass" = true ]; then
+
+ if [ "$1" == "--get-licenses" ]; then
+
+ approved_program=true;
+
+ else
+
+ license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+
+
+
+ # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License".
+ # See https://addons.mozilla.org/en-US/developers/addon/<your add-on>/ownership for the current add-on list. Note that not all licenses are added to the list.
+ # DuckDuckGo Privacy Essentials is distributed under Apache License 2.0.
+ # Apache License, version 2.0 is not on the add-on license list - https://github.com/mozilla/addons-server/issues/8545
+ if
+ [ "$license" == "Apache License, Version 2.0" ] || # Custom license
+ [ "$license" == "Apache License, version 2.0" ] || # Custom license
+ [ "$license" == "BSD License" ] ||
+ [ "$license" == "GNU General Public License, version 3.0" ] ||
+ [ "$license" == "GNU General Public License, version 2.0" ] ||
+ [ "$license" == "GNU Lesser General Public License, version 3.0" ] ||
+ [ "$license" == "GNU Lesser General Public License, version 2.1" ] ||
+ [ "$license" == "MIT/X11 License" ] ||
+ [ "$license" == "Mozilla Public License, version 2.0" ] ||
+ [ "$license" == "Mozilla Public License Version 1.1" ]
+ then
+
+ # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files.
+ freedom_status="free";
+
+ elif [ "$license" == "All Rights Reserved" ]; then
+
+ freedom_status="nonfree";
+
+ else
+
+ freedom_status="unknown";
+ fi
+
+ [ "$freedom_status" == "free" ]
+
+ fi
+
+ fi
+
+}
+
case "$1" in
""|-help)
@@ -110,8 +235,10 @@ OPTIONS
$0 --get-licenses
The licenses are not added to the general JSON file. This option will download the neccesary license JSON files for all add-ons.
$0 --download-licenses-json
- $0 --download-free-webextensions-for-gnu-and-linux
+ $0 --download-free-webextensions
+ Downloads the latest version of the free WebExtensions.
$0 --verify-license-copy
+ Search for license files in the root directory in the downloaded WebExtensions (.xpi files)
$0 --merged-free
Generates MERGED-ALL-freedom_status.txt
$0 --make-wiki
@@ -367,78 +494,57 @@ s/,/\n/g;
;;
--get-licenses)
+ # API: Please adopt SPDX - https://github.com/mozilla/addons-server/issues/8706
+ # Allow GNU [L]GPL x.x or later - https://github.com/mozilla/addons-server/issues/8707
+
cd build/ || exit
mkdir -p json/current_versions
cd json/current_versions || exit
line="0"; # Should not be 1
-
+ all_lines="1" # Should not be 0
+
+ # Begin template
+ unset file
while IFS= read -r table; do
- id=$(echo $table | awk '{print $2}');
- file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}');
+ verify_variables "$1"
- line_status
-
- # echo "$file $line $id"
-
- wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json"
-
- previous_file="$file";
-
- done < $RUN_PATH/build/MERGED-ALL.txt
-
- ;;
- --merged-free)
-
- cd build/ || exit
- cp -a MERGED-ALL.txt MERGED-ALL-freedom_status.txt
-
- line="1"; # Should not be "0"
- while IFS= read -r table; do
+ if [ "$approved_program" = true ]; then
+ # End template
- id=$(echo $table | awk '{print $2}');
- license=$(jq .license.name "$RUN_PATH/build/json/current_versions/$id.json" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
-
- # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License".
- # See https://addons.mozilla.org/en-US/developers/addon/<your add-on>/ownership for the current add-on list. Note that not all licenses are added to the list.
- # DuckDuckGo Privacy Essentials is distributed under Apache License 2.0.
- # Apache License, version 2.0 is not on the add-on license list - https://github.com/mozilla/addons-server/issues/8545
- if
- [ "$license" == "Apache License, Version 2.0" ] || # Custom license
- [ "$license" == "Apache License, version 2.0" ] || # Custom license
- [ "$license" == "BSD License" ] ||
- [ "$license" == "GNU General Public License, version 3.0" ] ||
- [ "$license" == "GNU General Public License, version 2.0" ] ||
- [ "$license" == "GNU Lesser General Public License, version 3.0" ] ||
- [ "$license" == "GNU Lesser General Public License, version 2.1" ] ||
- [ "$license" == "MIT/X11 License" ] ||
- [ "$license" == "Mozilla Public License, version 2.0" ] ||
- [ "$license" == "Mozilla Public License Version 1.1" ]
- then
+ echo "ddd";
+
+ id=$(echo $table | awk '{print $2}');
+ file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}');
+ has_eula=$(echo "$table" | cut -f 6);
+
+ # line_status
- # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files.
- sed -i "$line""s/$/\\tfree/" MERGED-ALL-freedom_status.txt
+ total=$(wc -l < $RUN_PATH/build/MERGED-ALL.txt)
+ echo "#######################################
+Entry: $all_lines (approved) / $total"
- elif [ "$license" == "All Rights Reserved" ]; then
+
+ # echo $(jq ".results[$line].slug" "$file");
+
+ echo "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/"
- sed -i "$line""s/$/\\tnonfree/" MERGED-ALL-freedom_status.txt
+ # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json"
- else
+ previous_file="$file";
+ ((all_lines++))
- sed -i "$line""s/$/\\tunknown/" MERGED-ALL-freedom_status.txt
-
+
+ # Begin template
fi
- # Add license for analysis
- # don't use "/" as delimeters since some licenses are "MIT/X11 License"
- sed -i "$line""s|$|\\t$license|" MERGED-ALL-freedom_status.txt
-
((line++))
done < $RUN_PATH/build/MERGED-ALL.txt
-
+ # End template
+
;;
--make-wiki)
@@ -453,44 +559,15 @@ s/,/\n/g;
! Rating
! Updated" > "wiki/IceCat WebExtensions (proposed).wiki"
-
+
+ # Begin template
unset file
while IFS= read -r table; do
- if [ -z ${file+x} ] || [ "$file" != "$(echo $table | awk '{print $3}')" ]; then
-
- line="0"; # should not be 1
-
- fi
-
- average_daily_users=$(echo $table | awk '{print $1}');
- id=$(echo $table | awk '{print $2}');
- file=$(echo $table | awk '{print $3}');
- name=$(echo "$table" | cut -f 5);
- has_eula=$(echo "$table" | cut -f 6);
- freedom_status=$(echo "$table" | cut -f 7);
-
- license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
- firefox_max_version="$(jq .results[$line].current_version.compatibility.firefox.max $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|.\*||")";
-
-
- # Values are either an integer or "*"
- re='^[0-9]+$'
- if [[ $firefox_max_version =~ $re ]] && [ "$firefox_max_version" -ge 60 ] || [ "$firefox_max_version" == '*' ]; then
-
-# echo "$firefox_max_version"
- firefox_max_version__pass=true;
-
- else
-
- unset firefox_max_version__pass
-
- fi
+ verify_variables "$1"
- if [ "$freedom_status" == "free" ] && [ "$has_eula" == "eulafree" ] && [ "$firefox_max_version__pass" = true ]; then
-
-
- # require min version 60 ddd
+ if [ "$approved_program" = true ]; then
+ # End template
####################################
# Build the core wiki pages
@@ -498,6 +575,7 @@ s/,/\n/g;
average_daily_users=$(echo $table | awk '{print $1}');
id=$(echo $table | awk '{print $2}');
file=$(echo $table | awk '{print $3}');
+ slug=$(echo $table | awk '{print $4}');
name=$(echo "$table" | cut -f 5);
has_eula=$(echo "$table" | cut -f 6);
freedom_status=$(echo "$table" | cut -f 7);
@@ -520,11 +598,24 @@ s/,/\n/g;
- short_description=$(jq .results[$line].summary $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|\\\n|\n|g; s|\\\||g");
+ short_description_raw=$(jq .results[$line].summary $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|\\\n|\n|g; s|\\\||g;
+s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues
+s|http|\nhttp|g;
+");
+ short_description_again=$(echo "$short_description_raw" | sed "/outgoing.prod.mozaws.net/d;");
-# echo "$short_description"
+ short_description=$(echo "$short_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g');
- full_description=$(jq .results[$line].description $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+ full_description_raw=$(jq .results[$line].description $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;
+ s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues
+");
+ full_description_again=$(echo "$full_description_raw" | sed "/outgoing.prod.mozaws.net/d;");
+
+ full_description=$(echo "$full_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g');
+
+
+
+
homepage_url=$(jq .results[$line].homepage $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
version=$(jq .results[$line].current_version.version $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;");
version_date=$(jq .results[$line].last_updated $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g");
@@ -556,7 +647,7 @@ s/,/\n/g;
# Do not list IceCatMobile
extension_of=$(echo "$set_seamonkey$set_icecat$set_thunderbird" | sed "s|,$||;");
-
+
github_true=$(echo "$homepage_url" | grep "github.com");
@@ -603,6 +694,7 @@ s/,/\n/g;
|Is GNU=$is_gnu
}}" >> wiki/$id.wiki
+
# https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/
echo "{{Project license
@@ -632,43 +724,96 @@ s/,/\n/g;
-
+ # Begin template
fi
((line++))
-
done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt
-
+ # End template
+
echo "|}" >> "wiki/IceCat WebExtensions (proposed).wiki"
;;
- --download-free-webextensions-for-gnu-and-linux)
+ --download-free-webextensions)
cd build || exit
- rm -fr free_webextensions_for_GNU_and_Linux
- mkdir free_webextensions_for_GNU_and_Linux
- cd free_webextensions_for_GNU_and_Linux || exit
+ rm -fr free_webextensions
+ mkdir free_webextensions
+ cd free_webextensions || exit
- line="0";
-
- while IFS= read -r license; do
-
- ((line++))
- # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough).
- wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi"
- done < ../license.name.enUS.txt
+ # Begin template
+ unset file
+ while IFS= read -r table; do
+
+ verify_variables "$1"
+
+ if [ "$approved_program" = true ]; then
+
+ # End template
+
+ id=$(echo $table | awk '{print $2}');
+ file=$(echo $table | awk '{print $3}');
+ slug=$(echo $table | awk '{print $4}');
+
+ # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough).
+ wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi"
+
+
+ # Begin template
+ fi
+
+ ((line++))
+
+ done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt
+ # End template
;;
--verify-license-copy)
cd build || exit
rm -fr VERIFY-LICENSE-COPY.txt
- cd free_webextensions_for_GNU_and_Linux || exit
+ cd free_webextensions || exit
+
+
+
+ # Begin template
+ unset file
+ while IFS= read -r table; do
+
+ verify_variables "$1"
+
+ if [ "$approved_program" = true ]; then
+ # End template
+
+ id=$(echo $table | awk '{print $2}');
+ file=$(echo $table | awk '{print $3}');
+ slug=$(echo $table | awk '{print $4}');
+
+ echo "--------------------------
+$name ($id): $license";
+ unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING"
+ unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.txt"
+ unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.md"
+ unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE"
+ unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.txt"
+ unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.md"
+
+ # Begin template
+ fi
+
+ ((line++))
+
+ done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt
+ # End template
+
+
+ ;;
+ --verify-license-copy-old)
while IFS= read -r id; do
@@ -782,7 +927,15 @@ $count_reports reports to submit";
--make-xml)
# Deprecated.
- # sha1sum file.xml
+
+
+ # XML pages look like this:
+ #...
+ # <sha1>tlybh1k9mnxz92kccg46hqv9jvyrkp0</sha1>
+ # </revision>
+ # </page>
+ #</mediawiki>
+ #
# However. It's not possible to insert the checsum in <sha1> for the page since it will break the checksum itself.
# Get number of edits: