#!/usr/bin/env bash # This file is part of license-reporter # Copyright (C) 2017 David Hedlund # # license-reporter is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # license-reporter is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . #if [ "$data" == "search" ]; then search_page_variable='.results[]'; fi # Specifically for titles: en-US is sometimes unused, en-GB used to be used then. # sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.name.enUS" | sed "s|^\"||; s|\"$||;" # sed "s|en-GB|enGB|g" "$json_file" | jq "$search_page_variable.name.enGB" | sed "s|^\"||; s|\"$||;" # sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.homepage.enUS" | sed "s|^\"||; s|\"$||;" # sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_email.enUS" | sed "s|^\"||; s|\"$||;" # sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_url.enUS" | sed "s|^\"||; s|\"$||;" # Licenses # sed "s|en-US|enUS|g" "$slug.json" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;" if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi minimum_average_daily_users="10000"; SCRIPTSRC=$(readlink -f "$0" || echo "$0") RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .) export -p SCRIPTSRC RUN_PATH minimum_average_daily_users debug if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi if [[ $exit == true ]]; then exit 1; fi case "$1" in --fresh-build) if [ -d "build" ]; then rm -fr build fi mkdir -p build ;; --make-custom-list) if [ ! -f "$RUN_PATH/license-reporter-custom" ]; then echo "$RUN_PATH/license-reporter-custom don't exist. Skipping custom list."; else src/download-custom fi ;; --make-search-list) cd build/ || exit mkdir json cd json || exit # API documentation: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html function foo() { # Get the most popular WebExtensions wget "https://addons.mozilla.org/api/v3/addons/search/?sort=users&page=$page&tag=firefox57" } page="1" foo if [ "$debug" == false ]; then while [ "$(jq ".results[].average_daily_users" "index.html?sort=users&page=$page&tag=firefox57" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do ((page++)) foo done fi find index.html* -type f -printf "%f\n" | while read -r file; do line="0"; for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do # Exclude add-ons with to low average daily users from index.html* if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file" >> ../merged-search.txt # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "current_versions/$(jq ".results[$line].current_version.id" "$file").json" fi ((line++)) done done ;; --merge-lists) cd build || exit cat merged-search.txt >> MERGED-ALL.txt cat merged-custom.txt >> MERGED-ALL.txt ;; --download-free-webextensions-for-gnu-and-linux) cd build || exit rm -fr free_webextensions_for_GNU_and_Linux mkdir free_webextensions_for_GNU_and_Linux cd free_webextensions_for_GNU_and_Linux || exit line="0"; while IFS= read -r license; do ((line++)) id="$(sed -n ${line}p ../id.txt)"; slug="$(sed -n ${line}p ../slug.txt)"; # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". if [ "$license" == "GNU General Public License, version 3.0" ] || [ "$license" == "GNU General Public License, version 2.0" ] || [ "$license" == "GNU Lesser General Public License, version 3.0" ] || [ "$license" == "GNU Lesser General Public License, version 2.1" ] || [ "$license" == "Mozilla Public License, version 2.0" ] || [ "$license" == "Mozilla Public License Version 1.1" ] || [ "$license" == "BSD License" ] || [ "$license" == "MIT/X11 License" ] then # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" fi done < ../license.name.enUS.txt ;; --verify-license-copy) cd build || exit rm -fr VERIFY-LICENSE-COPY.txt cd free_webextensions_for_GNU_and_Linux || exit while IFS= read -r id; do if [ ! -f "addon-$id-latest.xpi" ]; then echo "Nonfree" >> ../VERIFY-LICENSE-COPY.txt elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE$"; then echo "LICENSE" >> ../VERIFY-LICENSE-COPY.txt elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.txt$"; then echo "LICENSE.txt" >> ../VERIFY-LICENSE-COPY.txt elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.md$"; then echo "LICENSE.md" >> ../VERIFY-LICENSE-COPY.txt elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING$"; then echo "COPYING" >> ../VERIFY-LICENSE-COPY.txt elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.txt$"; then echo "COPYING.txt" >> ../VERIFY-LICENSE-COPY.txt elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.md$"; then echo "COPYING.md" >> ../VERIFY-LICENSE-COPY.txt else echo "Request" >> ../VERIFY-LICENSE-COPY.txt fi done < ../id.txt line="0"; while IFS= read -r status; do ((line++)) average_daily_users="$(sed -n ${line}p ../average_daily_users.txt)"; id="$(sed -n ${line}p ../id.txt)"; slug="$(sed -n ${line}p ../slug.txt)"; license_name_en_US="$(sed -n ${line}p ../license.name.enUS.txt)"; homepage_en_US="$(sed -n ${line}p ../homepage.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; support_email_en_US="$(sed -n ${line}p ../support_email.en-US.txt)"; support_url_en_US="$(sed -n ${line}p ../support_url.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; if [ "$license_name_en_US" == "GNU General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/gpl-3.0.txt"; license_file_name="COPYING"; elif [ "$license_name_en_US" == "GNU General Public License, version 2.0" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"; license_file_name="COPYING"; elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/lgpl-3.0.txt"; license_file_name="COPYING"; elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 2.1" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt"; license_file_name="COPYING"; elif [ "$license_name_en_US" == "Mozilla Public License, version 2.0" ]; then license_txt="https://www.mozilla.org/media/MPL/2.0/index.txt"; license_file_name="LICENSE"; elif [ "$license_name_en_US" == "Mozilla Public License Version 1.1" ]; then license_txt="https://www.mozilla.org/media/MPL/1.1/index.txt"; license_file_name="LICENSE"; elif [ "$license_name_en_US" == "BSD License" ]; then license_txt="https://directory.fsf.org/wiki?title=License:FreeBSD"; license_file_name="LICENSE"; elif [ "$license_name_en_US" == "MIT/X11 License" ]; then license_txt="https://directory.fsf.org/wiki/License:X11"; license_file_name="LICENSE"; fi if [ "$(sed -n ${line}p ../name.en-US.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-US.txt)"; elif [ "$(sed -n ${line}p ../name.en-GB.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-GB.txt)"; else name="$slug"; fi if [ "$status" = "Request" ]; then ((count_reports++)) # Authors that don't provide contact information will be ignored, its impossible to track conversations in Reviews anyway, because they will be deleted. if [ "$homepage_en_US" != "null" ] || [ "$support_url_en_US" != "null" ] || [ "$support_email_en_US" != "null" ]; then decimals="$(echo "$average_daily_users" | sed -r ':L;s=\b([0-9]+)([0-9]{3})\b=\1,\2=g;t L')"; echo " $decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/addon/$slug/ Request: \"You have distributed $name under $license_name_en_US on https://addons.mozilla.org/en-US/firefox/addon/$slug/, but you have not added a copy of the license. can you please download the text of the license from $license_txt and add save it to file name $license_file_name and put it in the root directory of you program?\" by contacting:"; if [ "$homepage_en_US" != "null" ]; then echo " * Homepage (please submit Pull Request for $license_file_name if possible): $homepage_en_US"; fi if [ "$support_url_en_US" != "null" ]; then echo " * Support URL (please submit Pull Request for $license_file_name if possible): $support_url_en_US"; fi if [ "$support_email_en_US" != "null" ]; then echo " * Support E-mail: $support_email_en_US"; fi fi fi done < ../VERIFY-LICENSE-COPY.txt echo " ######################### $count_reports reports to submit"; ;; --bugs) echo " * Look in the GitHub repository for LICENSE or COPYING. Sometime the add-on is delayed or have not been released to addons.mozilla.org." ;; --all) $0 --fresh-build $2 $0 --make-custom-list $2 $0 --make-search-list $2 $0 --merge-lists $2 $0 --download-licenses-json $2 # $0 --download-free-webextensions-for-gnu-and-linux # $0 --verify-license-copy ;; esac