path: root/addons.mozilla.org-fsd/get-data/license-reporter~

#!/usr/bin/env bash

#    This file is part of license-reporter
#    Copyright (C) 2017 David Hedlund
#
#    license-reporter is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    license-reporter is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.


#if [ "$data" == "search" ]; then search_page_variable='.results[]'; fi


# Specifically for titles: en-US is sometimes unused, en-GB used to be used then.
# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.name.enUS" | sed "s|^\"||; s|\"$||;"
# sed "s|en-GB|enGB|g" "$json_file" | jq "$search_page_variable.name.enGB" | sed "s|^\"||; s|\"$||;"

# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.homepage.enUS" | sed "s|^\"||; s|\"$||;"
# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_email.enUS" | sed "s|^\"||; s|\"$||;"
# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_url.enUS" | sed "s|^\"||; s|\"$||;"


# Licenses
# sed "s|en-US|enUS|g" "$slug.json" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;"

if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi

minimum_average_daily_users="10000";

SCRIPTSRC=$(readlink -f "$0" || echo "$0")
RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .)
filename=$(basename "$0");
export -p SCRIPTSRC RUN_PATH minimum_average_daily_users debug

if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi
if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi

if [[ $exit == true ]]; then exit 1; fi




case "$1" in

    ""|-help)
	
	[ "$1" = "" ] && echo "Usage: $filename [--option] [--debug]

OPTIONS
    $0 --all
	$0 --fresh-build
        $0 --make-collection-list
        $0 --make-custom-list
	$0 --make-search-list
	$0 --merge-lists
        $0 --download-licenses-json
        $0 --download-free-webextensions-for-gnu-and-linux
        $0 --verify-license-copy
" && exit 1

	;;

    --fresh-build)

        if [ -d "build" ]; then

            rm -fr build
            
        fi

        mkdir -p build

        ;;

    --make-repository-list)

	# It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs.
	# This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync
	# your addons.mozilla.org collection list with a separate text file.

	# Cannot list collections in desired formats - https://github.com/mozilla/addons/issues/722
		
	;;
    
    --make-collection-list)
	
        if [ ! -f "$RUN_PATH/license-reporter-collections" ]; then

            echo "$RUN_PATH/license-reporter-collections doesn't exist. Skipping custom list.";
            
        else


	    # Make sure all repositories are using SSL
	    if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-repository; then
		
		echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)."
		
	    else

		cd build/ || exit
		for i in $(cat ../license-reporter-collections); do
		    
		    # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextension-replacements/format%3Ajson
		    collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||");
		    wget $i"format%3Ajson" -O "collection-$collection_name.json"
		    
		done

	    fi
            
        fi
	
	;;
    
    --make-custom-list)

        if [ ! -f "$RUN_PATH/license-reporter-custom" ]; then

            echo "$RUN_PATH/license-reporter-custom doesn't exist. Skipping custom list.";
            
        else

            src/download-custom
            
        fi

        ;;

    --make-search-list)


        cd build/ || exit
        mkdir json
        cd json || exit

        # API documentation: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html

        function foo() {

            # Get the most popular WebExtensions
            wget "https://addons.mozilla.org/api/v3/addons/search/?sort=users&page=$page&tag=firefox57" -O "index.html?sort=users&page=$page&tag=firefox57.json"

        }

        page="1"
        foo



        if [ "$debug" == false ]; then

            while [ "$(jq ".results[].average_daily_users" "index.html?sort=users&page=$page&tag=firefox57.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do

                ((page++))
                foo

            done

	fi


        find index*.json -type f -printf "%f\n" | while read -r file; do


            line="0";
            
            for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do

                # Exclude add-ons with to low average daily users from index.html*
                if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then

                    echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file" >> ../merged-search.txt


                fi

                ((line++))
                
            done

            
        done

        ;;
    
    --merge-lists)

        cd build || exit

        cat merged-search.txt >> MERGED-ALL.txt
        cat merged-custom.txt >> MERGED-ALL.txt

        ;;
    --get-licenses)

        # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "current_versions/$(jq ".results[$line].current_version.id" "$file").json"
        
        ;;
    --download-free-webextensions-for-gnu-and-linux)
        
        cd build || exit
        rm -fr free_webextensions_for_GNU_and_Linux
        mkdir free_webextensions_for_GNU_and_Linux
        cd free_webextensions_for_GNU_and_Linux || exit

        line="0";
        
	while IFS= read -r license; do
            
            ((line++))
            id="$(sed -n ${line}p ../id.txt)";
            slug="$(sed -n ${line}p ../slug.txt)";

            # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License".
            if
                [ "$license" == "GNU General Public License, version 3.0" ] ||
                    [ "$license" == "GNU General Public License, version 2.0" ] ||
                    [ "$license" == "GNU Lesser General Public License, version 3.0" ] ||
                    [ "$license" == "GNU Lesser General Public License, version 2.1" ] ||
                    [ "$license" == "Mozilla Public License, version 2.0" ] ||
                    [ "$license" == "Mozilla Public License Version 1.1" ] ||
                    [ "$license" == "BSD License" ] ||
                    [ "$license" == "MIT/X11 License" ]
            then

                # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough).
                wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi"

                
            fi


        done < ../license.name.enUS.txt


        ;;
    --verify-license-copy)

        cd build || exit
        rm -fr VERIFY-LICENSE-COPY.txt
        cd free_webextensions_for_GNU_and_Linux || exit


        while IFS= read -r id; do
            



            if [ ! -f "addon-$id-latest.xpi" ]; then
                
                echo "Nonfree" >> ../VERIFY-LICENSE-COPY.txt
                
            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE$"; then
                
                echo "LICENSE" >> ../VERIFY-LICENSE-COPY.txt

            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.txt$"; then
                
                echo "LICENSE.txt" >> ../VERIFY-LICENSE-COPY.txt

            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.md$"; then
                
                echo "LICENSE.md" >> ../VERIFY-LICENSE-COPY.txt



            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING$"; then
                
                echo "COPYING" >> ../VERIFY-LICENSE-COPY.txt

            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.txt$"; then
                
                echo "COPYING.txt" >> ../VERIFY-LICENSE-COPY.txt

            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.md$"; then
                
                echo "COPYING.md" >> ../VERIFY-LICENSE-COPY.txt

            else

                echo "Request" >> ../VERIFY-LICENSE-COPY.txt
                
            fi

        done < ../id.txt



        line="0";

        while IFS= read -r status; do


            ((line++))

            average_daily_users="$(sed -n ${line}p ../average_daily_users.txt)";
            id="$(sed -n ${line}p ../id.txt)";
            slug="$(sed -n ${line}p ../slug.txt)";
            license_name_en_US="$(sed -n ${line}p ../license.name.enUS.txt)";
            homepage_en_US="$(sed -n ${line}p ../homepage.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)";
            support_email_en_US="$(sed -n ${line}p ../support_email.en-US.txt)";
            support_url_en_US="$(sed -n ${line}p ../support_url.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)";



            if [ "$license_name_en_US" == "GNU General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/gpl-3.0.txt"; license_file_name="COPYING";
            elif [ "$license_name_en_US" == "GNU General Public License, version 2.0" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"; license_file_name="COPYING";
            elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/lgpl-3.0.txt"; license_file_name="COPYING";
            elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 2.1" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt"; license_file_name="COPYING";
            elif [ "$license_name_en_US" == "Mozilla Public License, version 2.0" ];  then license_txt="https://www.mozilla.org/media/MPL/2.0/index.txt"; license_file_name="LICENSE";
            elif [ "$license_name_en_US" == "Mozilla Public License Version 1.1" ];  then license_txt="https://www.mozilla.org/media/MPL/1.1/index.txt"; license_file_name="LICENSE";
            elif [ "$license_name_en_US" == "BSD License" ];  then license_txt="https://directory.fsf.org/wiki?title=License:FreeBSD"; license_file_name="LICENSE";
            elif [ "$license_name_en_US" == "MIT/X11 License" ];  then license_txt="https://directory.fsf.org/wiki/License:X11"; license_file_name="LICENSE";
            fi


            if [ "$(sed -n ${line}p ../name.en-US.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-US.txt)";
            elif [ "$(sed -n ${line}p ../name.en-GB.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-GB.txt)";
            else name="$slug";
            fi


            if [ "$status" = "Request" ]; then

                ((count_reports++))

                # Authors that don't provide contact information will be ignored, its impossible to track conversations in Reviews anyway, because they will be deleted.

                if [ "$homepage_en_US" != "null" ] || [ "$support_url_en_US" != "null" ] || [ "$support_email_en_US" != "null" ]; then

                    decimals="$(echo "$average_daily_users" | sed -r ':L;s=\b([0-9]+)([0-9]{3})\b=\1,\2=g;t L')";

                    echo "
$decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/addon/$slug/
  Request: \"You have distributed $name under $license_name_en_US on https://addons.mozilla.org/en-US/firefox/addon/$slug/, but you have not added a copy of the license. can you please download the text of the license from $license_txt and add save it to file name $license_file_name and put it in the root directory of you program?\" by contacting:";

                    if [ "$homepage_en_US" != "null" ]; then echo "    * Homepage (please submit Pull Request for $license_file_name if possible): $homepage_en_US"; fi
                    if [ "$support_url_en_US" != "null" ]; then echo "    * Support URL (please submit Pull Request for $license_file_name if possible): $support_url_en_US"; fi
                    if [ "$support_email_en_US" != "null" ]; then echo "    * Support E-mail: $support_email_en_US"; fi

                fi

            fi

        done < ../VERIFY-LICENSE-COPY.txt

        echo "
#########################
$count_reports reports to submit";

        ;;

    --all)

	$0 --fresh-build "$2"
	$0 --make-collection-list "$2"
        $0 --make-custom-list "$2"
	$0 --make-search-list "$2"
	$0 --merge-lists "$2"
	#        $0 --download-licenses-json "$2"
	#        $0 --download-free-webextensions-for-gnu-and-linux "$2"
	#        $0 --verify-license-copy "$2"

	;;


esac