path: root/addons.mozilla.org-fsd/get-data/license-reporter

#!/usr/bin/env bash
#    This file is part of license-reporter
#    Copyright (C) 2017, 2018 David Hedlund
#
#    license-reporter is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    license-reporter is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

source err

#if [ "$data" == "search" ]; then search_page_variable='.results[]'; fi


# Specifically for titles: en-US is sometimes unused, en-GB used to be used then.
# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.name.enUS" | sed "s|^\"||; s|\"$||;"
# sed "s|en-GB|enGB|g" "$json_file" | jq "$search_page_variable.name.enGB" | sed "s|^\"||; s|\"$||;"

# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.homepage.enUS" | sed "s|^\"||; s|\"$||;"
# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_email.enUS" | sed "s|^\"||; s|\"$||;"
# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_url.enUS" | sed "s|^\"||; s|\"$||;"


# Licenses
# sed "s|en-US|enUS|g" "$slug.json" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;"

if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi

minimum_average_daily_users="10000";

scriptsrc=$(readlink -f -- "${BASH_SOURCE[0]}")
SCRIPTSRC=$(readlink -f "$0" || echo "$0")
RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .)
filename=$(basename "$0");
export -p SCRIPTSRC RUN_PATH minimum_average_daily_users debug

if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi
if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi

if [[ $exit == true ]]; then exit 1; fi


function check_files {

    if [ ! -f "$RUN_PATH/license-reporter-$check_type" ]; then
	
	echo "$RUN_PATH/license-reporter-$check_type doesn't exist.";
	echo "Exiting" && exit
	
    else


	# Make sure all links are using SSL
	if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-$check_type; then
	    
	    echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)."
	    echo "Exiting" && exit
	    
	fi
	
    fi

}

function line_status {

    
    if [ "$file" == "$previous_file" ]; then
        
        ((line++))
        
    else
        
        line="0"; # Should not be 1
        
    fi

}


function verify_variables {

    if [ -z ${file+x} ] || [ "$file" != "$(echo $table | awk '{print $3}')" ]; then

        line="0"; # should not be 1

    fi

    average_daily_users=$(echo $table | awk '{print $1}');
    id=$(echo $table | awk '{print $2}');
    file=$(echo $table | awk '{print $3}');
    slug=$(echo $table | awk '{print $4}');
    name=$(echo "$table" | cut -f 5);
    has_eula=$(echo "$table" | cut -f 6);


   
    firefox_max_version="$(jq .results[$line].current_version.compatibility.firefox.max $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|.\*||")";
    nativeMessaging="$(jq .results[$line].current_version.files[0].permissions $RUN_PATH/build/json/$file | grep "nativeMessaging")";

    # Values are either an integer or "*"
    re='^[0-9]+$'
    if [[ $firefox_max_version =~ $re ]] && [ "$firefox_max_version" -ge 60 ] || [ "$firefox_max_version" == '*' ]; then

        #                echo "$firefox_max_version"
        firefox_max_version__pass=true;

    else

        unset firefox_max_version__pass
        
    fi

    if [ ! -n "$nativeMessaging" ]; then

        nativeMessaging__pass=true;

    else

        nativeMessaging__pass=false;
        
    fi

    echo "$nativeMessaging__pass"

    
    # curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging
    

    # API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668
    # The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies.

    # Old solution:

    # 1: Add this to license-reporter-dependencies:
    #belgium-eid	eid-mw

    # 2: Uncomment this code:
    #    depends_on_external_program=false;
    #    while IFS= read -r dependencies; do
    #
    #        dependency_slug=$(echo $dependencies | awk '{print $1}');
    #        dependency=$(echo "$dependencies" | cut -f 2);
    #
    #        if [ "$dependency_slug" == "$slug" ]; then
    #
    #            depends_on_external_program=true;
    #
    #        fi
    #
    #    done < $RUN_PATH/license-reporter-dependencies




    approved_program=false;
    if [ "$has_eula" == "eulafree" ] && [ "$firefox_max_version__pass" = true ] && [ "$nativeMessaging__pass" = true ]; then

        if [ "$1" == "--get-licenses" ]; then

            approved_program=true;
            
        else

            license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");



            # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License".
            # See https://addons.mozilla.org/en-US/developers/addon/<your add-on>/ownership for the current add-on list. Note that not all licenses are added to the list.
            # DuckDuckGo Privacy Essentials is distributed under Apache License 2.0.
            # Apache License, version 2.0 is not on the add-on license list - https://github.com/mozilla/addons-server/issues/8545
            if
                [ "$license" == "Apache License, Version 2.0" ] || # Custom license
                    [ "$license" == "Apache License, version 2.0" ] || # Custom license
                    [ "$license" == "BSD License" ] ||
                    [ "$license" == "GNU General Public License, version 3.0" ] ||
                    [ "$license" == "GNU General Public License, version 2.0" ] ||
                    [ "$license" == "GNU Lesser General Public License, version 3.0" ] ||
                    [ "$license" == "GNU Lesser General Public License, version 2.1" ] ||
                    [ "$license" == "MIT/X11 License" ] ||
                    [ "$license" == "Mozilla Public License, version 2.0" ] ||
                    [ "$license" == "Mozilla Public License Version 1.1" ]
            then

                # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files.
                freedom_status="free";

            elif [ "$license" == "All Rights Reserved" ]; then

                freedom_status="nonfree";

            else

                freedom_status="unknown";
            fi
            
            [ "$freedom_status" == "free" ]
            
        fi

    fi
    
}

case "$1" in

    ""|-help)
	
	[ "$1" = "" ] && echo "Usage: $filename [--option] [--debug]

OPTIONS
	$0 --fresh-build
            Remove the build directory and create a new one.
	$0 --make-search-list
            Downloads the most popular WebExtensions from https://addons.mozilla.org/en-US/firefox/search/...
        $0 --make-repository-list
            Required file: license-reporter-collections
            File content syntax: https://addons.mozilla.org/en-US/firefox/collections/<user>/
        $0 --make-collection-list
            Required file: license-reporter-collections
            File content syntax: https://addons.mozilla.org/en-US/firefox/collections/<user>/<collection name>
        $0 --make-custom-list
            Required file: license-reporter-custom
            File content syntax: https://addons.mozilla.org/en-US/firefox/addon/<add-on name>/
	$0 --merge-lists
            Merge the lists to MERGED-ALL.txt
	$0 --get-licenses
            The licenses are not added to the general JSON file. This option will download the neccesary license JSON files for all add-ons.
        $0 --download-licenses-json
        $0 --download-free-webextensions
            Downloads the latest version of the free WebExtensions.
        $0 --verify-license-copy
            Search for license files in the root directory in the downloaded WebExtensions (.xpi files)
        $0  --merged-free
            Generates MERGED-ALL-freedom_status.txt
        $0  --make-wiki
            Generates .wiki files

    --all-recommended
        $0  --fresh-build
        $0  --make-search-list
        $0  --merge-lists
        $0  --get-licenses
        $0  --merged-free
        $0  --make-wiki

" && exit 1

	;;

    --fresh-build)

        if [ -d "build" ]; then

            rm -fr $RUN_PATH/build
            
        fi

        mkdir -p $RUN_PATH/build

        ;;

    --make-search-list)


        cd build/ || exit
        mkdir json
        cd json || exit

        # API documentation: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html


        page="1"
        
        function foo() {

            # Do not evaluate the number of extensions seen in https://addons.mozilla.org/en-US/firefox/search/ in a Quantum based browser since that will hide legacy add-ons.
            uri_query="page=$page&platform=linux&sort=users&type=extension";

            
            # Get the most popular WebExtensions
            wget "https://addons.mozilla.org/api/v3/addons/search/?$uri_query" -O "index.html?$uri_query.json"

            ((page++))

        }


        foo



        if [ "$debug" == false ]; then

            while [ "$(jq ".results[].average_daily_users" "index.html?$uri_query.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do

                foo

            done

	fi


        # Sort by file modification time stamp for the downloaded JSON files
        find index*.json -type f -printf "%Tc %p\n" | sort -n | awk '{print $NF}' | while read -r file; do



            
            line="0"; # Should not be 1
            
            for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do


                if [ "$(jq ".results[$line].has_eula" "$file")" == "false" ]; then
                    
                    has_eula="eulafree";
                    
                else
                    
                    has_eula="eula";
                    
                fi


                # Clean titles
                
                name=$(jq .results[$line].name $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");

                if [ "$name" == "null" ]; then

                    name=$(jq .results[$line].name $RUN_PATH/build/json/$file | sed "s|en-GB|enGB|;" | jq .enGB | sed "s|^\"||; s|\"$||;");

                fi
                

                # Clean titles. Example of bad titles: "Ghostery – Privacy Ad Blocker"

                # -
                # Hard to filter out "-" since many add-ons use it, see for example https://addons.mozilla.org/en-US/firefox/search/?platform=linux&q=1-click
                # Flash Video Downloader - YouTube HD Download [4K]
                
                name=$(echo $name | sed '
s/ — /\n/g;
s/ – /\n/g;
s/ - /\n/g; # looks identical to the prevoius line but is unique
s/: /\n/g;
s/(/\n/g;
s/,/\n/g;
' | head -n 1 | sed 's/ $//');




                
                
                # Exclude add-ons with to low average daily users from index.html*
                if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then

                    echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file\t$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")\t$name\t$has_eula" >> ../merged-search.txt


                fi

                ((line++))
                
            done

            
        done

        ;;
    --make-repository-list)

	# It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs.
	# This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync
	# your addons.mozilla.org collection list with a separate text file.

	# https://github.com/mozilla/addons/issues/722
	# Cannot list collections in desired formats
	# Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list).
	# So we have to parse it from HTML.


	# Get the links to the collections
	
	function get_repositories {
	    
	    check_type="repositories";
	    check_files

	    cd build/ || exit

	    for i in $(cat ../license-reporter-repositories); do

		user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;");
		rm -f repository-$user


		function wget_repository {
		    
		    match="^    <a href=\"/en-US/firefox/collections/"
		    wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user
		    
		}

		# Figure out if there are several pages
		number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||");
		
		if [ -z "$number_of_pages" ]; then

		    echo "$i doesn't have any sub pages";
		    wget_repository

		else

		    echo "$i has sub pages";

		    page_number=1;

		    while [ $page_number -le $number_of_pages ]
		    do
			
			wget_repository__page_extension="?page=$page_number";
			echo "$wget_repository__page_extension"
			wget_repository
			page_number=$(( $page_number + 1 ))
			
		    done
		    
		fi
	    	
	    done
	}
	get_repositories


	# Download the JSON files from the collections

	cd ..
	for x in $(find build/ -name repository* | sort); do

	    for i in $(cat $x); do

		collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||");
		wget $i"format%3Ajson" -O "build/collection-$collection_name.json"
		
	    done
	    
	done
	
	;;
    
    --make-collection-list)

	check_type="collections";
	check_files

	cd build/ || exit
	for i in $(cat ../license-reporter-collections); do
	    
	    # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson
	    collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||");
	    wget $i"format%3Ajson" -O "collection-$collection_name.json"
	    
	done
	
	;;
    
    --make-custom-list)

	check_type="custom";
	check_files

        src/download-custom

        ;;
    
    --merge-lists)

        cd build || exit

        cat merged-search.txt >> MERGED-ALL.txt
        cat merged-custom.txt >> MERGED-ALL.txt

        ;;
    --get-licenses)

        # API: Please adopt SPDX - https://github.com/mozilla/addons-server/issues/8706
        # Allow GNU [L]GPL x.x or later - https://github.com/mozilla/addons-server/issues/8707
        
        cd build/ || exit
        mkdir -p json/current_versions
        cd json/current_versions || exit

        line="0"; # Should not be 1
        all_lines="1" # Should not be 0

        # Begin template
        unset file
        while IFS= read -r table; do

            verify_variables "$1"

            if [ "$approved_program" = true ]; then
                # End template

                echo "ddd";
                
                id=$(echo $table | awk '{print $2}');
                file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}');
                has_eula=$(echo "$table" | cut -f 6);
                
                #                line_status

                total=$(wc -l < $RUN_PATH/build/MERGED-ALL.txt)
                echo "#######################################
Entry: $all_lines (approved) / $total"


                #                echo $(jq ".results[$line].slug" "$file");

                echo "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/"

                
                # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json"
                
                previous_file="$file";
                ((all_lines++))


                # Begin template
            fi

            ((line++))
            
        done < $RUN_PATH/build/MERGED-ALL.txt
        # End template
        
        ;;
    --make-wiki)

        cd build/ || exit
	if [ ! -d "wiki" ]; then mkdir wiki; fi

        echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\"
! WebExtension
! Description
! Claimed license
! Users
! Rating
! Updated" > "wiki/IceCat WebExtensions (proposed).wiki"


        # Begin template
        unset file
        while IFS= read -r table; do

            verify_variables "$1"
            
            if [ "$approved_program" = true ]; then
                # End template
                
                ####################################
                # Build the core wiki pages
                
                average_daily_users=$(echo $table | awk '{print $1}');
                id=$(echo $table | awk '{print $2}');
                file=$(echo $table | awk '{print $3}');
                slug=$(echo $table | awk '{print $4}');
                name=$(echo "$table" | cut -f 5);
                has_eula=$(echo "$table" | cut -f 6);
                freedom_status=$(echo "$table" | cut -f 7);
                
                license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");


                if [ "$license" == "Apache License, Version 2.0" ]; then license="Apache2.0";
                elif [ "$license" == "Apache License, version 2.0" ]; then license="Apache2.0";
                elif [ "$license" == "BSD License" ]; then license="BSD 2Clause";
                elif [ "$license" == "GNU General Public License, version 3.0" ]; then license="GPLv3";
                elif [ "$license" == "GNU General Public License, version 2.0" ]; then license="GPLv2";
                elif [ "$license" == "GNU Lesser General Public License, version 3.0" ]; then license="LGPLv3";
                elif [ "$license" == "GNU Lesser General Public License, version 2.1" ]; then license="LGPLv2.1";
                elif [ "$license" == "MIT/X11 License" ]; then license="X11";
                elif [ "$license" == "Mozilla Public License, version 2.0" ]; then license="MPLv2.0";
                elif [ "$license" == "Mozilla Public License Version 1.1" ]; then license="MPLv1.1";
                fi


                

                short_description_raw=$(jq .results[$line].summary $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|\\\n|\n|g; s|\\\||g;
s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues
s|http|\nhttp|g;
");
                short_description_again=$(echo "$short_description_raw" | sed "/outgoing.prod.mozaws.net/d;");

                short_description=$(echo "$short_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g');
                
                full_description_raw=$(jq .results[$line].description $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;
 s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues
");
                full_description_again=$(echo "$full_description_raw" | sed "/outgoing.prod.mozaws.net/d;");

                full_description=$(echo "$full_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g');


                

                homepage_url=$(jq .results[$line].homepage $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
                version=$(jq .results[$line].current_version.version $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;");
                version_date=$(jq .results[$line].last_updated $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g");
                version_id=$(jq .results[$line].current_version.id $RUN_PATH/build/json/$file);
                license_copyright=$(jq .results[$line].authors[].name $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;");
                bayesian_average=$(jq .results[$line].ratings.bayesian_average $RUN_PATH/build/json/$file);
                bayesian_average__simple=$(printf "%.1f" "$(echo "$bayesian_average" | sed "s|\.|,|")");
                
                last_review_date=$(date +"%Y/%m/%d");
                last_review_by="wikisysbot";
                submitted_by="wikisysbot";
                
	        if [[ "$(jq .results[$line].current_version.compatibility.seamonkey $RUN_PATH/build/json/$file)" != "null" ]]; then
		    set_seamonkey="Iceape,";
	        fi

	        if [[ "$(jq .results[$line].current_version.compatibility.firefox $RUN_PATH/build/json/$file)" != "null" ]]; then
		    set_icecat="IceCat,";
	        fi

                #	    if [[ "$(jq .results[$line].current_version.compatibility.android $RUN_PATH/build/json/$file)" != "null" ]] || [[ "$(jq .results[$line].current_version.compatibility.mobile $RUN_PATH/build/json/$file)" != "null" ]]; then
                #		set_icecatmobile="IceCatMobile,";
                #	    fi

	        if [[ "$(jq .results[$line].current_version.compatibility.thunderbird $RUN_PATH/build/json/$file)" != "null" ]]; then
		    set_thunderbird="Icedove,";
	        fi

                # Do not list IceCatMobile
                extension_of=$(echo "$set_seamonkey$set_icecat$set_thunderbird" | sed "s|,$||;");

                

                github_true=$(echo "$homepage_url" | grep "github.com");

                if [[ $github_true != "" ]]; then

                    # Always use https, and remove anchors
                    homepage_url__for__vcs_checkout_command="https://github.com$(echo $github_true | sed "s|https://github.com||; s|http://github.com||; s|#|\n|;" | head -n 1)";
                    # Remove trailing slash
                    homepage_url__for__vcs_checkout_command="${homepage_url__for__vcs_checkout_command%/}"
                    vcs_checkout_command="git clone $homepage_url__for__vcs_checkout_command.git";
                    
                fi

                if [ "$name" == "GNU LibreJS" ]; then
                    
                    is_gnu="Yes";
                    
                else
                    
                    is_gnu="No";
                    
                fi

                support_url=$(jq .results[$line].support_url $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
                
	        # List JSON structure: cat json/$file | js '.'
	        # XML value "homepage" is not always set in AMO API, but the XML require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically.
                echo "{{Entry" > wiki/$id.wiki

	        # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341
                # Avoid specific versions since it's not compatible with all IceCat versions:
                # |Version download=https://addons.mozilla.org/firefox/downloads/latest/$id/addon-$version_id-latest.xpi
                # |Version identifier=$version
                # |Version date=$version_date


                echo "|Name=$name
|Short description=$short_description
|Full description=$full_description
|Homepage URL=$homepage_url
|Extension of=$extension_of
|VCS checkout command=$vcs_checkout_command
|Version download=https://addons.mozilla.org/en-US/firefox/addon/$slug/versions/
|Last review by=$last_review_by
|Last review date=$last_review_date
|Submitted by=$submitted_by
|Submitted date=2018/06/20
|Is GNU=$is_gnu
}}" >> wiki/$id.wiki

                

	        # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/
	        echo "{{Project license
|License=$license
|License copyright=$license_copyright
|License note=\"License: $original_license\" listed
at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/
}}" >> wiki/$id.wiki

	        if [[ "$support_url" != "" ]];
	        then

		    echo "{{Resource
|Resource audience=Users
|Resource kind=Support
|Resource URL=$support_url
}}" >> "wiki/$id.wiki"

	        fi


                echo "|-
| [[$name]]
| $short_description
| [[License:$license|$license]]
| $average_daily_users
| $bayesian_average__simple
| $version_date" >> "wiki/IceCat WebExtensions (proposed).wiki"


                
                # Begin template
            fi

            ((line++))
            
        done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt
        # End template
        
        echo "|}" >> "wiki/IceCat WebExtensions (proposed).wiki"
        
        ;;

    --download-free-webextensions)
        
        cd build || exit
        rm -fr free_webextensions
        mkdir free_webextensions
        cd free_webextensions || exit



        # Begin template
        unset file
        while IFS= read -r table; do

            verify_variables "$1"

            if [ "$approved_program" = true ]; then

                # End template

                id=$(echo $table | awk '{print $2}');
                file=$(echo $table | awk '{print $3}');
                slug=$(echo $table | awk '{print $4}');

                # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough).
                wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi"
                
                

                # Begin template
            fi

            ((line++))
            
        done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt
        # End template

        ;;
    --verify-license-copy)

        cd build || exit
        rm -fr VERIFY-LICENSE-COPY.txt
        cd free_webextensions || exit



        # Begin template
        unset file
        while IFS= read -r table; do

            verify_variables "$1"

            if [ "$approved_program" = true ]; then
                # End template

                id=$(echo $table | awk '{print $2}');
                file=$(echo $table | awk '{print $3}');
                slug=$(echo $table | awk '{print $4}');
                
                echo "--------------------------
$name ($id): $license";
                unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING"
                unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.txt"
                unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.md"
                unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE"
                unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.txt"
                unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.md"
                
                # Begin template
            fi

            ((line++))
            
        done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt
        # End template

        
        ;;
    --verify-license-copy-old)


        while IFS= read -r id; do
            



            if [ ! -f "addon-$id-latest.xpi" ]; then
                
                echo "Nonfree" >> ../VERIFY-LICENSE-COPY.txt
                
            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE$"; then
                
                echo "LICENSE" >> ../VERIFY-LICENSE-COPY.txt

            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.txt$"; then
                
                echo "LICENSE.txt" >> ../VERIFY-LICENSE-COPY.txt

            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.md$"; then
                
                echo "LICENSE.md" >> ../VERIFY-LICENSE-COPY.txt



            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING$"; then
                
                echo "COPYING" >> ../VERIFY-LICENSE-COPY.txt

            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.txt$"; then
                
                echo "COPYING.txt" >> ../VERIFY-LICENSE-COPY.txt

            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.md$"; then
                
                echo "COPYING.md" >> ../VERIFY-LICENSE-COPY.txt

            else

                echo "Request" >> ../VERIFY-LICENSE-COPY.txt
                
            fi

        done < ../id.txt



        line="0";

        while IFS= read -r status; do


            ((line++))

            average_daily_users="$(sed -n ${line}p ../average_daily_users.txt)";
            id="$(sed -n ${line}p ../id.txt)";
            slug="$(sed -n ${line}p ../slug.txt)";
            license_name_en_US="$(sed -n ${line}p ../license.name.enUS.txt)";
            homepage_en_US="$(sed -n ${line}p ../homepage.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)";
            support_email_en_US="$(sed -n ${line}p ../support_email.en-US.txt)";
            support_url_en_US="$(sed -n ${line}p ../support_url.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)";



            if [ "$license_name_en_US" == "GNU General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/gpl-3.0.txt"; license_file_name="COPYING";
            elif [ "$license_name_en_US" == "GNU General Public License, version 2.0" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"; license_file_name="COPYING";
            elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/lgpl-3.0.txt"; license_file_name="COPYING";
            elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 2.1" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt"; license_file_name="COPYING";
            elif [ "$license_name_en_US" == "Mozilla Public License, version 2.0" ];  then license_txt="https://www.mozilla.org/media/MPL/2.0/index.txt"; license_file_name="LICENSE";
            elif [ "$license_name_en_US" == "Mozilla Public License Version 1.1" ];  then license_txt="https://www.mozilla.org/media/MPL/1.1/index.txt"; license_file_name="LICENSE";
            elif [ "$license_name_en_US" == "BSD License" ];  then license_txt="https://directory.fsf.org/wiki?title=License:FreeBSD"; license_file_name="LICENSE";
            elif [ "$license_name_en_US" == "MIT/X11 License" ];  then license_txt="https://directory.fsf.org/wiki/License:X11"; license_file_name="LICENSE";
            fi


            if [ "$(sed -n ${line}p ../name.en-US.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-US.txt)";
            elif [ "$(sed -n ${line}p ../name.en-GB.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-GB.txt)";
            else name="$slug";
            fi


            if [ "$status" = "Request" ]; then

                ((count_reports++))

                # Authors that don't provide contact information will be ignored, its impossible to track conversations in Reviews anyway, because they will be deleted.

                if [ "$homepage_en_US" != "null" ] || [ "$support_url_en_US" != "null" ] || [ "$support_email_en_US" != "null" ]; then

                    decimals="$(echo "$average_daily_users" | sed -r ':L;s=\b([0-9]+)([0-9]{3})\b=\1,\2=g;t L')";

                    echo "
$decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/addon/$slug/
  Request: \"You have distributed $name under $license_name_en_US on https://addons.mozilla.org/en-US/firefox/addon/$slug/, but you have not added a copy of the license. can you please download the text of the license from $license_txt and add save it to file name $license_file_name and put it in the root directory of you program?\" by contacting:";

                    if [ "$homepage_en_US" != "null" ]; then echo "    * Homepage (please submit Pull Request for $license_file_name if possible): $homepage_en_US"; fi
                    if [ "$support_url_en_US" != "null" ]; then echo "    * Support URL (please submit Pull Request for $license_file_name if possible): $support_url_en_US"; fi
                    if [ "$support_email_en_US" != "null" ]; then echo "    * Support E-mail: $support_email_en_US"; fi

                fi

            fi

        done < ../VERIFY-LICENSE-COPY.txt

        echo "
#########################
$count_reports reports to submit";

        ;;

    --make-xml)
        # Deprecated.


        # XML pages look like this:
        #...
        #      <sha1>tlybh1k9mnxz92kccg46hqv9jvyrkp0</sha1>
        #    </revision>
        #  </page>
        #</mediawiki>
        #
        # However. It's not possible to insert the checsum in <sha1> for the page since it will break the checksum itself.

	# Get number of edits:
	revid="$(wget -qO- "https://directory.fsf.org/w/api.php?action=query&list=recentchanges&rclimit=1&format=json" | jq .query.recentchanges[0].revid)";
        xml_header="$(wget -qO- "https://directory.fsf.org/wiki/Special:Export/Main_Page" | head -n 42)";
        
        ;;
    
    --all-recommended)
        $0  --fresh-build "$2"
        $0  --make-search-list "$2"
        $0  --merge-lists "$2"
        $0  --get-licenses "$2"
        $0  --merged-free "$2"
        $0  --make-wiki "$2"
	;;


esac