diff options
Diffstat (limited to 'addons.mozilla.org-fsd/get-data/license-reporter (copy)')
| -rwxr-xr-x | addons.mozilla.org-fsd/get-data/license-reporter (copy) | 442 |
1 files changed, 0 insertions, 442 deletions
diff --git a/addons.mozilla.org-fsd/get-data/license-reporter (copy) b/addons.mozilla.org-fsd/get-data/license-reporter (copy) deleted file mode 100755 index dd46854..0000000 --- a/addons.mozilla.org-fsd/get-data/license-reporter (copy) +++ /dev/null @@ -1,442 +0,0 @@ -#!/usr/bin/env bash - -# This file is part of license-reporter -# Copyright (C) 2017, 2018 David Hedlund -# -# license-reporter is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# license-reporter is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -#if [ "$data" == "search" ]; then search_page_variable='.results[]'; fi - - -# Specifically for titles: en-US is sometimes unused, en-GB used to be used then. -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.name.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-GB|enGB|g" "$json_file" | jq "$search_page_variable.name.enGB" | sed "s|^\"||; s|\"$||;" - -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.homepage.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_email.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_url.enUS" | sed "s|^\"||; s|\"$||;" - - -# Licenses -# sed "s|en-US|enUS|g" "$slug.json" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;" - -if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi - -minimum_average_daily_users="10000"; - -SCRIPTSRC=$(readlink -f "$0" || echo "$0") -RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .) -filename=$(basename "$0"); -export -p SCRIPTSRC RUN_PATH minimum_average_daily_users debug - -if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi -if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi - -if [[ $exit == true ]]; then exit 1; fi - - -function check_files { - - if [ ! -f "$RUN_PATH/license-reporter-$check_type" ]; then - - echo "$RUN_PATH/license-reporter-$check_type doesn't exist."; - echo "Exiting" && exit - - else - - - # Make sure all links are using SSL - if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-$check_type; then - - echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)." - echo "Exiting" && exit - - fi - - fi - -} - - - -case "$1" in - - ""|-help) - - [ "$1" = "" ] && echo "Usage: $filename [--option] [--debug] - -OPTIONS - $0 --all - $0 --fresh-build - $0 --make-repository-list - $0 --make-collection-list - $0 --make-custom-list - $0 --make-search-list - $0 --merge-lists - $0 --download-licenses-json - $0 --download-free-webextensions-for-gnu-and-linux - $0 --verify-license-copy -" && exit 1 - - ;; - - --fresh-build) - - if [ -d "build" ]; then - - rm -fr build - - fi - - mkdir -p build - - ;; - - --make-repository-list) - - # It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs. - # This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync - # your addons.mozilla.org collection list with a separate text file. - - # https://github.com/mozilla/addons/issues/722 - # Cannot list collections in desired formats - # Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list). - # So we have to parse it from HTML. - - - function get_repositories { - - check_type="repositories"; - check_files - - cd build/ || exit - - for i in $(cat ../license-reporter-repositories); do - - user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;"); - rm -f repository-$user - - - function wget_repository { - - match="^ <a href=\"/en-US/firefox/collections/" - wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user - - } - - # Figure out if there are several pages - number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||"); - - if [ -z "$number_of_pages" ]; then - - echo "$i doesn't have any sub pages"; - wget_repository - - else - - echo "$i has sub pages"; - - page_number=1; - - while [ $page_number -le $number_of_pages ] - do - - wget_repository__page_extension="?page=$page_number"; - echo "$wget_repository__page_extension" - wget_repository - page_number=$(( $page_number + 1 )) - - done - - fi - - done - } - -# get_repositories - - for x in $(find . -name repository* ); do - - echo $x; - - done - - ;; - - --make-collection-list) - - check_type="collections"; - check_files - - cd build/ || exit - for i in $(cat ../license-reporter-collections); do - - # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson - collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); - wget $i"format%3Ajson" -O "collection-$collection_name.json" - - done - - ;; - - --make-custom-list) - - check_type="custom"; - check_files - - src/download-custom - - ;; - - --make-search-list) - - - cd build/ || exit - mkdir json - cd json || exit - - # API documentation: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html - - function foo() { - - # Get the most popular WebExtensions - wget "https://addons.mozilla.org/api/v3/addons/search/?sort=users&page=$page&tag=firefox57" -O "index.html?sort=users&page=$page&tag=firefox57.json" - - } - - page="1" - foo - - - - if [ "$debug" == false ]; then - - while [ "$(jq ".results[].average_daily_users" "index.html?sort=users&page=$page&tag=firefox57.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do - - ((page++)) - foo - - done - - fi - - - find index*.json -type f -printf "%f\n" | while read -r file; do - - - line="0"; - - for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do - - # Exclude add-ons with to low average daily users from index.html* - if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then - - echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file" >> ../merged-search.txt - - - fi - - ((line++)) - - done - - - done - - ;; - - --merge-lists) - - cd build || exit - - cat merged-search.txt >> MERGED-ALL.txt - cat merged-custom.txt >> MERGED-ALL.txt - - ;; - --get-licenses) - - # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "current_versions/$(jq ".results[$line].current_version.id" "$file").json" - - ;; - --download-free-webextensions-for-gnu-and-linux) - - cd build || exit - rm -fr free_webextensions_for_GNU_and_Linux - mkdir free_webextensions_for_GNU_and_Linux - cd free_webextensions_for_GNU_and_Linux || exit - - line="0"; - - while IFS= read -r license; do - - ((line++)) - id="$(sed -n ${line}p ../id.txt)"; - slug="$(sed -n ${line}p ../slug.txt)"; - - # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". - if - [ "$license" == "GNU General Public License, version 3.0" ] || - [ "$license" == "GNU General Public License, version 2.0" ] || - [ "$license" == "GNU Lesser General Public License, version 3.0" ] || - [ "$license" == "GNU Lesser General Public License, version 2.1" ] || - [ "$license" == "Mozilla Public License, version 2.0" ] || - [ "$license" == "Mozilla Public License Version 1.1" ] || - [ "$license" == "BSD License" ] || - [ "$license" == "MIT/X11 License" ] - then - - # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). - wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" - - - fi - - - done < ../license.name.enUS.txt - - - ;; - --verify-license-copy) - - cd build || exit - rm -fr VERIFY-LICENSE-COPY.txt - cd free_webextensions_for_GNU_and_Linux || exit - - - while IFS= read -r id; do - - - - - if [ ! -f "addon-$id-latest.xpi" ]; then - - echo "Nonfree" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE$"; then - - echo "LICENSE" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.txt$"; then - - echo "LICENSE.txt" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.md$"; then - - echo "LICENSE.md" >> ../VERIFY-LICENSE-COPY.txt - - - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING$"; then - - echo "COPYING" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.txt$"; then - - echo "COPYING.txt" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.md$"; then - - echo "COPYING.md" >> ../VERIFY-LICENSE-COPY.txt - - else - - echo "Request" >> ../VERIFY-LICENSE-COPY.txt - - fi - - done < ../id.txt - - - - line="0"; - - while IFS= read -r status; do - - - ((line++)) - - average_daily_users="$(sed -n ${line}p ../average_daily_users.txt)"; - id="$(sed -n ${line}p ../id.txt)"; - slug="$(sed -n ${line}p ../slug.txt)"; - license_name_en_US="$(sed -n ${line}p ../license.name.enUS.txt)"; - homepage_en_US="$(sed -n ${line}p ../homepage.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; - support_email_en_US="$(sed -n ${line}p ../support_email.en-US.txt)"; - support_url_en_US="$(sed -n ${line}p ../support_url.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; - - - - if [ "$license_name_en_US" == "GNU General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/gpl-3.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU General Public License, version 2.0" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/lgpl-3.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 2.1" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "Mozilla Public License, version 2.0" ]; then license_txt="https://www.mozilla.org/media/MPL/2.0/index.txt"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "Mozilla Public License Version 1.1" ]; then license_txt="https://www.mozilla.org/media/MPL/1.1/index.txt"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "BSD License" ]; then license_txt="https://directory.fsf.org/wiki?title=License:FreeBSD"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "MIT/X11 License" ]; then license_txt="https://directory.fsf.org/wiki/License:X11"; license_file_name="LICENSE"; - fi - - - if [ "$(sed -n ${line}p ../name.en-US.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-US.txt)"; - elif [ "$(sed -n ${line}p ../name.en-GB.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-GB.txt)"; - else name="$slug"; - fi - - - if [ "$status" = "Request" ]; then - - ((count_reports++)) - - # Authors that don't provide contact information will be ignored, its impossible to track conversations in Reviews anyway, because they will be deleted. - - if [ "$homepage_en_US" != "null" ] || [ "$support_url_en_US" != "null" ] || [ "$support_email_en_US" != "null" ]; then - - decimals="$(echo "$average_daily_users" | sed -r ':L;s=\b([0-9]+)([0-9]{3})\b=\1,\2=g;t L')"; - - echo " -$decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/addon/$slug/ - Request: \"You have distributed $name under $license_name_en_US on https://addons.mozilla.org/en-US/firefox/addon/$slug/, but you have not added a copy of the license. can you please download the text of the license from $license_txt and add save it to file name $license_file_name and put it in the root directory of you program?\" by contacting:"; - - if [ "$homepage_en_US" != "null" ]; then echo " * Homepage (please submit Pull Request for $license_file_name if possible): $homepage_en_US"; fi - if [ "$support_url_en_US" != "null" ]; then echo " * Support URL (please submit Pull Request for $license_file_name if possible): $support_url_en_US"; fi - if [ "$support_email_en_US" != "null" ]; then echo " * Support E-mail: $support_email_en_US"; fi - - fi - - fi - - done < ../VERIFY-LICENSE-COPY.txt - - echo " -######################### -$count_reports reports to submit"; - - ;; - - --all) - - $0 --fresh-build "$2" - $0 --make-repository-list "$2" - $0 --make-collection-list "$2" - $0 --make-custom-list "$2" - $0 --make-search-list "$2" - $0 --merge-lists "$2" - # $0 --download-licenses-json "$2" - # $0 --download-free-webextensions-for-gnu-and-linux "$2" - # $0 --verify-license-copy "$2" - - ;; - - -esac |