diff options
Diffstat (limited to 'addons.mozilla.org-fsd/get-data')
| -rwxr-xr-x | addons.mozilla.org-fsd/get-data/license-reporter | 371 |
1 files changed, 262 insertions, 109 deletions
diff --git a/addons.mozilla.org-fsd/get-data/license-reporter b/addons.mozilla.org-fsd/get-data/license-reporter index 47c28a8..9bf4263 100755 --- a/addons.mozilla.org-fsd/get-data/license-reporter +++ b/addons.mozilla.org-fsd/get-data/license-reporter @@ -34,7 +34,7 @@ if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi -minimum_average_daily_users="100000"; +minimum_average_daily_users="10000"; SCRIPTSRC=$(readlink -f "$0" || echo "$0") RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .) @@ -85,6 +85,131 @@ function line_status { } +function verify_variables { + + if [ -z ${file+x} ] || [ "$file" != "$(echo $table | awk '{print $3}')" ]; then + + line="0"; # should not be 1 + + fi + + average_daily_users=$(echo $table | awk '{print $1}'); + id=$(echo $table | awk '{print $2}'); + file=$(echo $table | awk '{print $3}'); + slug=$(echo $table | awk '{print $4}'); + name=$(echo "$table" | cut -f 5); + has_eula=$(echo "$table" | cut -f 6); + + + + firefox_max_version="$(jq .results[$line].current_version.compatibility.firefox.max $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|.\*||")"; + nativeMessaging="$(jq .results[$line].current_version.files[0].permissions $RUN_PATH/build/json/$file | grep "nativeMessaging")"; + + # Values are either an integer or "*" + re='^[0-9]+$' + if [[ $firefox_max_version =~ $re ]] && [ "$firefox_max_version" -ge 60 ] || [ "$firefox_max_version" == '*' ]; then + + # echo "$firefox_max_version" + firefox_max_version__pass=true; + + else + + unset firefox_max_version__pass + + fi + + if [ ! -n "$nativeMessaging" ]; then + + nativeMessaging__pass=true; + + else + + nativeMessaging__pass=false; + + fi + + echo "$nativeMessaging__pass" + + + # curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging + + + # API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668 + # The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies. + + # Old solution: + + # 1: Add this to license-reporter-dependencies: + #belgium-eid eid-mw + + # 2: Uncomment this code: + # depends_on_external_program=false; + # while IFS= read -r dependencies; do + # + # dependency_slug=$(echo $dependencies | awk '{print $1}'); + # dependency=$(echo "$dependencies" | cut -f 2); + # + # if [ "$dependency_slug" == "$slug" ]; then + # + # depends_on_external_program=true; + # + # fi + # + # done < $RUN_PATH/license-reporter-dependencies + + + + + approved_program=false; + if [ "$has_eula" == "eulafree" ] && [ "$firefox_max_version__pass" = true ] && [ "$nativeMessaging__pass" = true ]; then + + if [ "$1" == "--get-licenses" ]; then + + approved_program=true; + + else + + license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + + + + # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". + # See https://addons.mozilla.org/en-US/developers/addon/<your add-on>/ownership for the current add-on list. Note that not all licenses are added to the list. + # DuckDuckGo Privacy Essentials is distributed under Apache License 2.0. + # Apache License, version 2.0 is not on the add-on license list - https://github.com/mozilla/addons-server/issues/8545 + if + [ "$license" == "Apache License, Version 2.0" ] || # Custom license + [ "$license" == "Apache License, version 2.0" ] || # Custom license + [ "$license" == "BSD License" ] || + [ "$license" == "GNU General Public License, version 3.0" ] || + [ "$license" == "GNU General Public License, version 2.0" ] || + [ "$license" == "GNU Lesser General Public License, version 3.0" ] || + [ "$license" == "GNU Lesser General Public License, version 2.1" ] || + [ "$license" == "MIT/X11 License" ] || + [ "$license" == "Mozilla Public License, version 2.0" ] || + [ "$license" == "Mozilla Public License Version 1.1" ] + then + + # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files. + freedom_status="free"; + + elif [ "$license" == "All Rights Reserved" ]; then + + freedom_status="nonfree"; + + else + + freedom_status="unknown"; + fi + + [ "$freedom_status" == "free" ] + + fi + + fi + +} + case "$1" in ""|-help) @@ -110,8 +235,10 @@ OPTIONS $0 --get-licenses The licenses are not added to the general JSON file. This option will download the neccesary license JSON files for all add-ons. $0 --download-licenses-json - $0 --download-free-webextensions-for-gnu-and-linux + $0 --download-free-webextensions + Downloads the latest version of the free WebExtensions. $0 --verify-license-copy + Search for license files in the root directory in the downloaded WebExtensions (.xpi files) $0 --merged-free Generates MERGED-ALL-freedom_status.txt $0 --make-wiki @@ -367,78 +494,57 @@ s/,/\n/g; ;; --get-licenses) + # API: Please adopt SPDX - https://github.com/mozilla/addons-server/issues/8706 + # Allow GNU [L]GPL x.x or later - https://github.com/mozilla/addons-server/issues/8707 + cd build/ || exit mkdir -p json/current_versions cd json/current_versions || exit line="0"; # Should not be 1 - + all_lines="1" # Should not be 0 + + # Begin template + unset file while IFS= read -r table; do - id=$(echo $table | awk '{print $2}'); - file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}'); + verify_variables "$1" - line_status - - # echo "$file $line $id" - - wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json" - - previous_file="$file"; - - done < $RUN_PATH/build/MERGED-ALL.txt - - ;; - --merged-free) - - cd build/ || exit - cp -a MERGED-ALL.txt MERGED-ALL-freedom_status.txt - - line="1"; # Should not be "0" - while IFS= read -r table; do + if [ "$approved_program" = true ]; then + # End template - id=$(echo $table | awk '{print $2}'); - license=$(jq .license.name "$RUN_PATH/build/json/current_versions/$id.json" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - - # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". - # See https://addons.mozilla.org/en-US/developers/addon/<your add-on>/ownership for the current add-on list. Note that not all licenses are added to the list. - # DuckDuckGo Privacy Essentials is distributed under Apache License 2.0. - # Apache License, version 2.0 is not on the add-on license list - https://github.com/mozilla/addons-server/issues/8545 - if - [ "$license" == "Apache License, Version 2.0" ] || # Custom license - [ "$license" == "Apache License, version 2.0" ] || # Custom license - [ "$license" == "BSD License" ] || - [ "$license" == "GNU General Public License, version 3.0" ] || - [ "$license" == "GNU General Public License, version 2.0" ] || - [ "$license" == "GNU Lesser General Public License, version 3.0" ] || - [ "$license" == "GNU Lesser General Public License, version 2.1" ] || - [ "$license" == "MIT/X11 License" ] || - [ "$license" == "Mozilla Public License, version 2.0" ] || - [ "$license" == "Mozilla Public License Version 1.1" ] - then + echo "ddd"; + + id=$(echo $table | awk '{print $2}'); + file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}'); + has_eula=$(echo "$table" | cut -f 6); + + # line_status - # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files. - sed -i "$line""s/$/\\tfree/" MERGED-ALL-freedom_status.txt + total=$(wc -l < $RUN_PATH/build/MERGED-ALL.txt) + echo "####################################### +Entry: $all_lines (approved) / $total" - elif [ "$license" == "All Rights Reserved" ]; then + + # echo $(jq ".results[$line].slug" "$file"); + + echo "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" - sed -i "$line""s/$/\\tnonfree/" MERGED-ALL-freedom_status.txt + # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json" - else + previous_file="$file"; + ((all_lines++)) - sed -i "$line""s/$/\\tunknown/" MERGED-ALL-freedom_status.txt - + + # Begin template fi - # Add license for analysis - # don't use "/" as delimeters since some licenses are "MIT/X11 License" - sed -i "$line""s|$|\\t$license|" MERGED-ALL-freedom_status.txt - ((line++)) done < $RUN_PATH/build/MERGED-ALL.txt - + # End template + ;; --make-wiki) @@ -453,44 +559,15 @@ s/,/\n/g; ! Rating ! Updated" > "wiki/IceCat WebExtensions (proposed).wiki" - + + # Begin template unset file while IFS= read -r table; do - if [ -z ${file+x} ] || [ "$file" != "$(echo $table | awk '{print $3}')" ]; then - - line="0"; # should not be 1 - - fi - - average_daily_users=$(echo $table | awk '{print $1}'); - id=$(echo $table | awk '{print $2}'); - file=$(echo $table | awk '{print $3}'); - name=$(echo "$table" | cut -f 5); - has_eula=$(echo "$table" | cut -f 6); - freedom_status=$(echo "$table" | cut -f 7); - - license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - firefox_max_version="$(jq .results[$line].current_version.compatibility.firefox.max $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|.\*||")"; - - - # Values are either an integer or "*" - re='^[0-9]+$' - if [[ $firefox_max_version =~ $re ]] && [ "$firefox_max_version" -ge 60 ] || [ "$firefox_max_version" == '*' ]; then - -# echo "$firefox_max_version" - firefox_max_version__pass=true; - - else - - unset firefox_max_version__pass - - fi + verify_variables "$1" - if [ "$freedom_status" == "free" ] && [ "$has_eula" == "eulafree" ] && [ "$firefox_max_version__pass" = true ]; then - - - # require min version 60 ddd + if [ "$approved_program" = true ]; then + # End template #################################### # Build the core wiki pages @@ -498,6 +575,7 @@ s/,/\n/g; average_daily_users=$(echo $table | awk '{print $1}'); id=$(echo $table | awk '{print $2}'); file=$(echo $table | awk '{print $3}'); + slug=$(echo $table | awk '{print $4}'); name=$(echo "$table" | cut -f 5); has_eula=$(echo "$table" | cut -f 6); freedom_status=$(echo "$table" | cut -f 7); @@ -520,11 +598,24 @@ s/,/\n/g; - short_description=$(jq .results[$line].summary $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|\\\n|\n|g; s|\\\||g"); + short_description_raw=$(jq .results[$line].summary $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|\\\n|\n|g; s|\\\||g; +s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues +s|http|\nhttp|g; +"); + short_description_again=$(echo "$short_description_raw" | sed "/outgoing.prod.mozaws.net/d;"); -# echo "$short_description" + short_description=$(echo "$short_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g'); - full_description=$(jq .results[$line].description $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + full_description_raw=$(jq .results[$line].description $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||; + s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues +"); + full_description_again=$(echo "$full_description_raw" | sed "/outgoing.prod.mozaws.net/d;"); + + full_description=$(echo "$full_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g'); + + + + homepage_url=$(jq .results[$line].homepage $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); version=$(jq .results[$line].current_version.version $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;"); version_date=$(jq .results[$line].last_updated $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g"); @@ -556,7 +647,7 @@ s/,/\n/g; # Do not list IceCatMobile extension_of=$(echo "$set_seamonkey$set_icecat$set_thunderbird" | sed "s|,$||;"); - + github_true=$(echo "$homepage_url" | grep "github.com"); @@ -603,6 +694,7 @@ s/,/\n/g; |Is GNU=$is_gnu }}" >> wiki/$id.wiki + # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/ echo "{{Project license @@ -632,43 +724,96 @@ s/,/\n/g; - + # Begin template fi ((line++)) - done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt - + # End template + echo "|}" >> "wiki/IceCat WebExtensions (proposed).wiki" ;; - --download-free-webextensions-for-gnu-and-linux) + --download-free-webextensions) cd build || exit - rm -fr free_webextensions_for_GNU_and_Linux - mkdir free_webextensions_for_GNU_and_Linux - cd free_webextensions_for_GNU_and_Linux || exit + rm -fr free_webextensions + mkdir free_webextensions + cd free_webextensions || exit - line="0"; - - while IFS= read -r license; do - - ((line++)) - # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). - wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" - done < ../license.name.enUS.txt + # Begin template + unset file + while IFS= read -r table; do + + verify_variables "$1" + + if [ "$approved_program" = true ]; then + + # End template + + id=$(echo $table | awk '{print $2}'); + file=$(echo $table | awk '{print $3}'); + slug=$(echo $table | awk '{print $4}'); + + # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). + wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" + + + # Begin template + fi + + ((line++)) + + done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt + # End template ;; --verify-license-copy) cd build || exit rm -fr VERIFY-LICENSE-COPY.txt - cd free_webextensions_for_GNU_and_Linux || exit + cd free_webextensions || exit + + + + # Begin template + unset file + while IFS= read -r table; do + + verify_variables "$1" + + if [ "$approved_program" = true ]; then + # End template + + id=$(echo $table | awk '{print $2}'); + file=$(echo $table | awk '{print $3}'); + slug=$(echo $table | awk '{print $4}'); + + echo "-------------------------- +$name ($id): $license"; + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.txt" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.md" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.txt" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.md" + + # Begin template + fi + + ((line++)) + + done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt + # End template + + + ;; + --verify-license-copy-old) while IFS= read -r id; do @@ -782,7 +927,15 @@ $count_reports reports to submit"; --make-xml) # Deprecated. - # sha1sum file.xml + + + # XML pages look like this: + #... + # <sha1>tlybh1k9mnxz92kccg46hqv9jvyrkp0</sha1> + # </revision> + # </page> + #</mediawiki> + # # However. It's not possible to insert the checsum in <sha1> for the page since it will break the checksum itself. # Get number of edits: |