diff options
Diffstat (limited to 'addons.mozilla.org-fsd/get-data/license-reporter (copy)')
| -rwxr-xr-x | addons.mozilla.org-fsd/get-data/license-reporter (copy) | 216 |
1 files changed, 171 insertions, 45 deletions
diff --git a/addons.mozilla.org-fsd/get-data/license-reporter (copy) b/addons.mozilla.org-fsd/get-data/license-reporter (copy) index cc76e8c..dd46854 100755 --- a/addons.mozilla.org-fsd/get-data/license-reporter (copy) +++ b/addons.mozilla.org-fsd/get-data/license-reporter (copy) @@ -1,7 +1,7 @@ #!/usr/bin/env bash # This file is part of license-reporter -# Copyright (C) 2017 David Hedlund +# Copyright (C) 2017, 2018 David Hedlund # # license-reporter is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -38,6 +38,7 @@ minimum_average_daily_users="10000"; SCRIPTSRC=$(readlink -f "$0" || echo "$0") RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .) +filename=$(basename "$0"); export -p SCRIPTSRC RUN_PATH minimum_average_daily_users debug if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi @@ -46,7 +47,51 @@ if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi if [[ $exit == true ]]; then exit 1; fi +function check_files { + + if [ ! -f "$RUN_PATH/license-reporter-$check_type" ]; then + + echo "$RUN_PATH/license-reporter-$check_type doesn't exist."; + echo "Exiting" && exit + + else + + + # Make sure all links are using SSL + if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-$check_type; then + + echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)." + echo "Exiting" && exit + + fi + + fi + +} + + + case "$1" in + + ""|-help) + + [ "$1" = "" ] && echo "Usage: $filename [--option] [--debug] + +OPTIONS + $0 --all + $0 --fresh-build + $0 --make-repository-list + $0 --make-collection-list + $0 --make-custom-list + $0 --make-search-list + $0 --merge-lists + $0 --download-licenses-json + $0 --download-free-webextensions-for-gnu-and-linux + $0 --verify-license-copy +" && exit 1 + + ;; + --fresh-build) if [ -d "build" ]; then @@ -59,18 +104,99 @@ case "$1" in ;; + --make-repository-list) - --make-custom-list) + # It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs. + # This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync + # your addons.mozilla.org collection list with a separate text file. - if [ ! -f "$RUN_PATH/license-reporter-custom" ]; then + # https://github.com/mozilla/addons/issues/722 + # Cannot list collections in desired formats + # Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list). + # So we have to parse it from HTML. - echo "$RUN_PATH/license-reporter-custom don't exist. Skipping custom list."; - - else - src/download-custom - - fi + function get_repositories { + + check_type="repositories"; + check_files + + cd build/ || exit + + for i in $(cat ../license-reporter-repositories); do + + user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;"); + rm -f repository-$user + + + function wget_repository { + + match="^ <a href=\"/en-US/firefox/collections/" + wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user + + } + + # Figure out if there are several pages + number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||"); + + if [ -z "$number_of_pages" ]; then + + echo "$i doesn't have any sub pages"; + wget_repository + + else + + echo "$i has sub pages"; + + page_number=1; + + while [ $page_number -le $number_of_pages ] + do + + wget_repository__page_extension="?page=$page_number"; + echo "$wget_repository__page_extension" + wget_repository + page_number=$(( $page_number + 1 )) + + done + + fi + + done + } + +# get_repositories + + for x in $(find . -name repository* ); do + + echo $x; + + done + + ;; + + --make-collection-list) + + check_type="collections"; + check_files + + cd build/ || exit + for i in $(cat ../license-reporter-collections); do + + # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson + collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); + wget $i"format%3Ajson" -O "collection-$collection_name.json" + + done + + ;; + + --make-custom-list) + + check_type="custom"; + check_files + + src/download-custom ;; @@ -86,7 +212,7 @@ case "$1" in function foo() { # Get the most popular WebExtensions - wget "https://addons.mozilla.org/api/v3/addons/search/?sort=users&page=$page&tag=firefox57" + wget "https://addons.mozilla.org/api/v3/addons/search/?sort=users&page=$page&tag=firefox57" -O "index.html?sort=users&page=$page&tag=firefox57.json" } @@ -97,39 +223,38 @@ case "$1" in if [ "$debug" == false ]; then - while [ "$(jq ".results[].average_daily_users" "index.html?sort=users&page=$page&tag=firefox57" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do + while [ "$(jq ".results[].average_daily_users" "index.html?sort=users&page=$page&tag=firefox57.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do ((page++)) foo done -fi - + fi - find index.html* -type f -printf "%f\n" | while read -r file; do + find index*.json -type f -printf "%f\n" | while read -r file; do - line="0"; - - for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do - # Exclude add-ons with to low average daily users from index.html* - if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then + line="0"; + + for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do - echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file" >> ../merged-search.txt + # Exclude add-ons with to low average daily users from index.html* + if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then - # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "current_versions/$(jq ".results[$line].current_version.id" "$file").json" + echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file" >> ../merged-search.txt - fi - ((line++)) - - done + fi + ((line++)) done + + done + ;; --merge-lists) @@ -140,6 +265,11 @@ fi cat merged-custom.txt >> MERGED-ALL.txt ;; + --get-licenses) + + # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "current_versions/$(jq ".results[$line].current_version.id" "$file").json" + + ;; --download-free-webextensions-for-gnu-and-linux) cd build || exit @@ -158,13 +288,13 @@ fi # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". if [ "$license" == "GNU General Public License, version 3.0" ] || - [ "$license" == "GNU General Public License, version 2.0" ] || - [ "$license" == "GNU Lesser General Public License, version 3.0" ] || - [ "$license" == "GNU Lesser General Public License, version 2.1" ] || - [ "$license" == "Mozilla Public License, version 2.0" ] || - [ "$license" == "Mozilla Public License Version 1.1" ] || - [ "$license" == "BSD License" ] || - [ "$license" == "MIT/X11 License" ] + [ "$license" == "GNU General Public License, version 2.0" ] || + [ "$license" == "GNU Lesser General Public License, version 3.0" ] || + [ "$license" == "GNU Lesser General Public License, version 2.1" ] || + [ "$license" == "Mozilla Public License, version 2.0" ] || + [ "$license" == "Mozilla Public License Version 1.1" ] || + [ "$license" == "BSD License" ] || + [ "$license" == "MIT/X11 License" ] then # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). @@ -293,22 +423,18 @@ $decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/ $count_reports reports to submit"; ;; - --bugs) - - echo " - * Look in the GitHub repository for LICENSE or COPYING. Sometime the add-on is delayed or have not been released to addons.mozilla.org." - - ;; --all) - $0 --fresh-build $2 - $0 --make-custom-list $2 - $0 --make-search-list $2 - $0 --merge-lists $2 - $0 --download-licenses-json $2 - # $0 --download-free-webextensions-for-gnu-and-linux - # $0 --verify-license-copy + $0 --fresh-build "$2" + $0 --make-repository-list "$2" + $0 --make-collection-list "$2" + $0 --make-custom-list "$2" + $0 --make-search-list "$2" + $0 --merge-lists "$2" + # $0 --download-licenses-json "$2" + # $0 --download-free-webextensions-for-gnu-and-linux "$2" + # $0 --verify-license-copy "$2" ;; |