diff options
Diffstat (limited to 'addons.mozilla.org-fsd/get-data/license-reporter')
| -rwxr-xr-x | addons.mozilla.org-fsd/get-data/license-reporter | 123 |
1 files changed, 101 insertions, 22 deletions
diff --git a/addons.mozilla.org-fsd/get-data/license-reporter b/addons.mozilla.org-fsd/get-data/license-reporter index 68a05f9..5298037 100755 --- a/addons.mozilla.org-fsd/get-data/license-reporter +++ b/addons.mozilla.org-fsd/get-data/license-reporter @@ -34,7 +34,7 @@ if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi -minimum_average_daily_users="10000"; +minimum_average_daily_users="1000000"; SCRIPTSRC=$(readlink -f "$0" || echo "$0") RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .) @@ -81,7 +81,7 @@ OPTIONS $0 --all $0 --fresh-build $0 --make-repository-list - Downloads add-on data from collections shared by defined users in license-reporter-repositories. Example: + Downloads add-on data for free add-ons from collections shared by defined users in license-reporter-repositories. Example: https://addons.mozilla.org/en-US/firefox/collections/mozilla/ $0 --make-collection-list @@ -222,21 +222,27 @@ OPTIONS # API documentation: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html + + page="1" + function foo() { + # Do not evaluate the number of extensions seen in https://addons.mozilla.org/en-US/firefox/search/ in a Quantum based browser since that will hide legacy add-ons. + uri_query="page=$page&platform=linux&sort=users&type=extension&tag=firefox57"; + # Get the most popular WebExtensions - wget "https://addons.mozilla.org/api/v3/addons/search/?sort=users&page=$page&tag=firefox57" -O "index.html?sort=users&page=$page&tag=firefox57.json" + wget "https://addons.mozilla.org/api/v3/addons/search/?$uri_query" -O "index.html?$uri_query.json" } - page="1" + foo if [ "$debug" == false ]; then - while [ "$(jq ".results[].average_daily_users" "index.html?sort=users&page=$page&tag=firefox57.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do + while [ "$(jq ".results[].average_daily_users" "index.html?$uri_query.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do ((page++)) foo @@ -246,17 +252,18 @@ OPTIONS fi - find index*.json -type f -printf "%f\n" | while read -r file; do + # Sort by file modification time stamp for the downloaded JSON files + find index*.json -type f -printf "%Tc %p\n" | sort -n | awk '{print $NF}' | while read -r file; do line="0"; for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do - + # Exclude add-ons with to low average daily users from index.html* if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then - echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file" >> ../merged-search.txt + echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file\t$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")" >> ../merged-search.txt fi @@ -280,23 +287,35 @@ OPTIONS ;; --get-licenses) - # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "current_versions/$(jq ".results[$line].current_version.id" "$file").json" + cd build/ || exit + mkdir -p json/current_versions + cd json/current_versions || exit + + line="0"; + while IFS= read -r table; do + + ((line++)) + + id=$(echo $table | awk '{print $2}'); + file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}'); + + wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json" + + done < $RUN_PATH/build/MERGED-ALL.txt ;; - --download-free-webextensions-for-gnu-and-linux) + --merged-free) - cd build || exit - rm -fr free_webextensions_for_GNU_and_Linux - mkdir free_webextensions_for_GNU_and_Linux - cd free_webextensions_for_GNU_and_Linux || exit - - line="0"; + cd build/ || exit + cp -a MERGED-ALL.txt MERGED-ALL-freedom_status.txt - while IFS= read -r license; do - + line="0"; + while IFS= read -r table; do + ((line++)) - id="$(sed -n ${line}p ../id.txt)"; - slug="$(sed -n ${line}p ../slug.txt)"; + + id=$(echo $table | awk '{print $2}'); + license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". if @@ -310,12 +329,72 @@ OPTIONS [ "$license" == "MIT/X11 License" ] then - # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). - wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" + # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files. + sed -i "$line""s/$/\\tfree/" MERGED-ALL-freedom_status.txt + else + + sed -i "$line""s/$/\\tnonfree/" MERGED-ALL-freedom_status.txt fi + done < $RUN_PATH/build/MERGED-ALL.txt + + ;; + --generate-proposed-list) + + line="0"; + while IFS= read -r table; do + + ((line++)) + + id=$(echo $table | awk '{print $2}'); + source=$(echo $table | awk '{print $3}'); + license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + freedom_status=$(echo $table | awk '{print $5}'); + name=$(jq .results[$line].name $RUN_PATH/build/json/$source | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + + if [ "$name" == "null" ]; then + + name=$(jq .results[$line].name $RUN_PATH/build/json/$source | sed "s|en-GB|enGB|;" | jq .enGB | sed "s|^\"||; s|\"$||;"); + + fi + + + # Clean titles. Example of bad titles: "Ghostery – Privacy Ad Blocker" + if [[ $name = *"–"* ]]; then + + name=$(echo $name | sed 's/–/\n/g; s/-/\n/g' | head -n 1); + + fi + + if [ "$freedom_status" == "free" ]; then + echo "{{Checkme item +|Section= +|Package=$name +|Package note= +}}" + fi + + done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt + + ;; + + --download-free-webextensions-for-gnu-and-linux) + + cd build || exit + rm -fr free_webextensions_for_GNU_and_Linux + mkdir free_webextensions_for_GNU_and_Linux + cd free_webextensions_for_GNU_and_Linux || exit + + line="0"; + + while IFS= read -r license; do + + ((line++)) + + # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). + wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" done < ../license.name.enUS.txt |