aboutsummaryrefslogtreecommitdiff
path: root/addons.mozilla.org-fsd/get-data/license-reporter
diff options
context:
space:
mode:
authordavid <public@beloved.name>2018-06-15 07:29:49 +0200
committerdavid <public@beloved.name>2018-06-15 07:29:49 +0200
commit9ac9fe13e85951e99f1996e4dc2e68d45edb3ec9 (patch)
treecba5bb1991e977bf269cf51604b4a6042cca20b2 /addons.mozilla.org-fsd/get-data/license-reporter
parent4660350001548bf35ab54c633cf0950786d039d5 (diff)
I only marked addons.mozilla.org-fsd for commit
Diffstat (limited to 'addons.mozilla.org-fsd/get-data/license-reporter')
-rwxr-xr-xaddons.mozilla.org-fsd/get-data/license-reporter123
1 files changed, 101 insertions, 22 deletions
diff --git a/addons.mozilla.org-fsd/get-data/license-reporter b/addons.mozilla.org-fsd/get-data/license-reporter
index 68a05f9..5298037 100755
--- a/addons.mozilla.org-fsd/get-data/license-reporter
+++ b/addons.mozilla.org-fsd/get-data/license-reporter
@@ -34,7 +34,7 @@
if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi
-minimum_average_daily_users="10000";
+minimum_average_daily_users="1000000";
SCRIPTSRC=$(readlink -f "$0" || echo "$0")
RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .)
@@ -81,7 +81,7 @@ OPTIONS
$0 --all
$0 --fresh-build
$0 --make-repository-list
- Downloads add-on data from collections shared by defined users in license-reporter-repositories. Example:
+ Downloads add-on data for free add-ons from collections shared by defined users in license-reporter-repositories. Example:
https://addons.mozilla.org/en-US/firefox/collections/mozilla/
$0 --make-collection-list
@@ -222,21 +222,27 @@ OPTIONS
# API documentation: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html
+
+ page="1"
+
function foo() {
+ # Do not evaluate the number of extensions seen in https://addons.mozilla.org/en-US/firefox/search/ in a Quantum based browser since that will hide legacy add-ons.
+ uri_query="page=$page&platform=linux&sort=users&type=extension&tag=firefox57";
+
# Get the most popular WebExtensions
- wget "https://addons.mozilla.org/api/v3/addons/search/?sort=users&page=$page&tag=firefox57" -O "index.html?sort=users&page=$page&tag=firefox57.json"
+ wget "https://addons.mozilla.org/api/v3/addons/search/?$uri_query" -O "index.html?$uri_query.json"
}
- page="1"
+
foo
if [ "$debug" == false ]; then
- while [ "$(jq ".results[].average_daily_users" "index.html?sort=users&page=$page&tag=firefox57.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do
+ while [ "$(jq ".results[].average_daily_users" "index.html?$uri_query.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do
((page++))
foo
@@ -246,17 +252,18 @@ OPTIONS
fi
- find index*.json -type f -printf "%f\n" | while read -r file; do
+ # Sort by file modification time stamp for the downloaded JSON files
+ find index*.json -type f -printf "%Tc %p\n" | sort -n | awk '{print $NF}' | while read -r file; do
line="0";
for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do
-
+
# Exclude add-ons with to low average daily users from index.html*
if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then
- echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file" >> ../merged-search.txt
+ echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file\t$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")" >> ../merged-search.txt
fi
@@ -280,23 +287,35 @@ OPTIONS
;;
--get-licenses)
- # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "current_versions/$(jq ".results[$line].current_version.id" "$file").json"
+ cd build/ || exit
+ mkdir -p json/current_versions
+ cd json/current_versions || exit
+
+ line="0";
+ while IFS= read -r table; do
+
+ ((line++))
+
+ id=$(echo $table | awk '{print $2}');
+ file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}');
+
+ wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json"
+
+ done < $RUN_PATH/build/MERGED-ALL.txt
;;
- --download-free-webextensions-for-gnu-and-linux)
+ --merged-free)
- cd build || exit
- rm -fr free_webextensions_for_GNU_and_Linux
- mkdir free_webextensions_for_GNU_and_Linux
- cd free_webextensions_for_GNU_and_Linux || exit
-
- line="0";
+ cd build/ || exit
+ cp -a MERGED-ALL.txt MERGED-ALL-freedom_status.txt
- while IFS= read -r license; do
-
+ line="0";
+ while IFS= read -r table; do
+
((line++))
- id="$(sed -n ${line}p ../id.txt)";
- slug="$(sed -n ${line}p ../slug.txt)";
+
+ id=$(echo $table | awk '{print $2}');
+ license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
# This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License".
if
@@ -310,12 +329,72 @@ OPTIONS
[ "$license" == "MIT/X11 License" ]
then
- # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough).
- wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi"
+ # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files.
+ sed -i "$line""s/$/\\tfree/" MERGED-ALL-freedom_status.txt
+ else
+
+ sed -i "$line""s/$/\\tnonfree/" MERGED-ALL-freedom_status.txt
fi
+ done < $RUN_PATH/build/MERGED-ALL.txt
+
+ ;;
+ --generate-proposed-list)
+
+ line="0";
+ while IFS= read -r table; do
+
+ ((line++))
+
+ id=$(echo $table | awk '{print $2}');
+ source=$(echo $table | awk '{print $3}');
+ license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+ freedom_status=$(echo $table | awk '{print $5}');
+ name=$(jq .results[$line].name $RUN_PATH/build/json/$source | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+
+ if [ "$name" == "null" ]; then
+
+ name=$(jq .results[$line].name $RUN_PATH/build/json/$source | sed "s|en-GB|enGB|;" | jq .enGB | sed "s|^\"||; s|\"$||;");
+
+ fi
+
+
+ # Clean titles. Example of bad titles: "Ghostery – Privacy Ad Blocker"
+ if [[ $name = *"–"* ]]; then
+
+ name=$(echo $name | sed 's/–/\n/g; s/-/\n/g' | head -n 1);
+
+ fi
+
+ if [ "$freedom_status" == "free" ]; then
+ echo "{{Checkme item
+|Section=
+|Package=$name
+|Package note=
+}}"
+ fi
+
+ done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt
+
+ ;;
+
+ --download-free-webextensions-for-gnu-and-linux)
+
+ cd build || exit
+ rm -fr free_webextensions_for_GNU_and_Linux
+ mkdir free_webextensions_for_GNU_and_Linux
+ cd free_webextensions_for_GNU_and_Linux || exit
+
+ line="0";
+
+ while IFS= read -r license; do
+
+ ((line++))
+
+ # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough).
+ wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi"
done < ../license.name.enUS.txt