I only marked addons.mozilla.org-fsd for commit

1 files changed, 528 insertions, 0 deletions

diff --git a/addons.mozilla.org-fsd/get-data/license-reporter (another copy) b/addons.mozilla.org-fsd/get-data/license-reporter (another copy)
new file mode 100755
index 0000000..a297883
--- /dev/null
+++ b/addons.mozilla.org-fsd/get-data/license-reporter (another copy)
@@ -0,0 +1,528 @@
+#!/usr/bin/env bash
+
+#    This file is part of license-reporter
+#    Copyright (C) 2017, 2018 David Hedlund
+#
+#    license-reporter is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    license-reporter is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+#if [ "$data" == "search" ]; then search_page_variable='.results[]'; fi
+
+
+# Specifically for titles: en-US is sometimes unused, en-GB used to be used then.
+# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.name.enUS" | sed "s|^\"||; s|\"$||;"
+# sed "s|en-GB|enGB|g" "$json_file" | jq "$search_page_variable.name.enGB" | sed "s|^\"||; s|\"$||;"
+
+# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.homepage.enUS" | sed "s|^\"||; s|\"$||;"
+# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_email.enUS" | sed "s|^\"||; s|\"$||;"
+# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_url.enUS" | sed "s|^\"||; s|\"$||;"
+
+
+# Licenses
+# sed "s|en-US|enUS|g" "$slug.json" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;"
+
+if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi
+
+minimum_average_daily_users="1000000";
+
+SCRIPTSRC=$(readlink -f "$0" || echo "$0")
+RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .)
+filename=$(basename "$0");
+export -p SCRIPTSRC RUN_PATH minimum_average_daily_users debug
+
+if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi
+if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi
+
+if [[ $exit == true ]]; then exit 1; fi
+
+
+function check_files {
+
+    if [ ! -f "$RUN_PATH/license-reporter-$check_type" ]; then
+	
+	echo "$RUN_PATH/license-reporter-$check_type doesn't exist.";
+	echo "Exiting" && exit
+	
+    else
+
+
+	# Make sure all links are using SSL
+	if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-$check_type; then
+	    
+	    echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)."
+	    echo "Exiting" && exit
+	    
+	fi
+	
+    fi
+
+}
+
+
+
+case "$1" in
+
+    ""|-help)
+	
+	[ "$1" = "" ] && echo "Usage: $filename [--option] [--debug]
+
+OPTIONS
+    $0 --all
+	$0 --fresh-build
+        $0 --make-repository-list
+               Downloads add-on data for free add-ons from collections shared by defined users in license-reporter-repositories. Example:
+               https://addons.mozilla.org/en-US/firefox/collections/mozilla/
+
+        $0 --make-collection-list
+        $0 --make-custom-list
+	$0 --make-search-list
+	$0 --merge-lists
+        $0 --download-licenses-json
+        $0 --download-free-webextensions-for-gnu-and-linux
+        $0 --verify-license-copy
+" && exit 1
+
+	;;
+
+    --fresh-build)
+
+        if [ -d "build" ]; then
+
+            rm -fr build
+            
+        fi
+
+        mkdir -p build
+
+        ;;
+
+    --make-repository-list)
+
+	# It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs.
+	# This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync
+	# your addons.mozilla.org collection list with a separate text file.
+
+	# https://github.com/mozilla/addons/issues/722
+	# Cannot list collections in desired formats
+	# Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list).
+	# So we have to parse it from HTML.
+
+
+	# Get the links to the collections
+	
+	function get_repositories {
+	    
+	    check_type="repositories";
+	    check_files
+
+	    cd build/ || exit
+
+	    for i in $(cat ../license-reporter-repositories); do
+
+		user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;");
+		rm -f repository-$user
+
+
+		function wget_repository {
+		    
+		    match="^    <a href=\"/en-US/firefox/collections/"
+		    wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user
+		    
+		}
+
+		# Figure out if there are several pages
+		number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||");
+		
+		if [ -z "$number_of_pages" ]; then
+
+		    echo "$i doesn't have any sub pages";
+		    wget_repository
+
+		else
+
+		    echo "$i has sub pages";
+
+		    page_number=1;
+
+		    while [ $page_number -le $number_of_pages ]
+		    do
+			
+			wget_repository__page_extension="?page=$page_number";
+			echo "$wget_repository__page_extension"
+			wget_repository
+			page_number=$(( $page_number + 1 ))
+			
+		    done
+		    
+		fi
+	    	
+	    done
+	}
+	get_repositories
+
+
+	# Download the JSON files from the collections
+
+	cd ..
+	for x in $(find build/ -name repository* | sort); do
+
+	    for i in $(cat $x); do
+
+		collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||");
+		wget $i"format%3Ajson" -O "build/collection-$collection_name.json"
+		
+	    done
+	    
+	done
+	
+	;;
+    
+    --make-collection-list)
+
+	check_type="collections";
+	check_files
+
+	cd build/ || exit
+	for i in $(cat ../license-reporter-collections); do
+	    
+	    # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson
+	    collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||");
+	    wget $i"format%3Ajson" -O "collection-$collection_name.json"
+	    
+	done
+	
+	;;
+    
+    --make-custom-list)
+
+	check_type="custom";
+	check_files
+
+        src/download-custom
+
+        ;;
+
+    --make-search-list)
+
+
+        cd build/ || exit
+        mkdir json
+        cd json || exit
+
+        # API documentation: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html
+
+        function foo() {
+
+            # Get the most popular WebExtensions
+            wget "https://addons.mozilla.org/api/v3/addons/search/?sort=users&page=$page&tag=firefox57" -O "index.html?sort=users&page=$page&tag=firefox57.json"
+
+        }
+
+        page="1"
+        foo
+
+
+
+        if [ "$debug" == false ]; then
+
+            while [ "$(jq ".results[].average_daily_users" "index.html?sort=users&page=$page&tag=firefox57.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do
+
+                ((page++))
+                foo
+
+            done
+
+	fi
+
+
+        # Sort by file modification time stamp for the downloaded JSON files
+        find index*.json -type f -printf "%Tc %p\n" | sort -n | awk '{print $NF}' | while read -r file; do
+
+
+            line="0";
+            
+            for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do
+                
+                # Exclude add-ons with to low average daily users from index.html*
+                if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then
+
+                    echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file" >> ../merged-search.txt
+
+
+                fi
+
+                ((line++))
+                
+            done
+
+            
+        done
+
+        ;;
+    
+    --merge-lists)
+
+        cd build || exit
+
+        cat merged-search.txt >> MERGED-ALL.txt
+        cat merged-custom.txt >> MERGED-ALL.txt
+
+        ;;
+    --get-licenses)
+
+        cd build/ || exit
+        mkdir json/current_versions
+        cd json/current_versions
+        
+        line="0";
+        while IFS= read -r table; do
+
+            ((line++))
+
+            id=$(echo $table | awk '{print $2}');
+            file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}');
+
+            wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json"
+            
+        done < $RUN_PATH/build/MERGED-ALL.txt
+        
+        ;;
+    --merged-free)
+        
+        cd build/ || exit
+        cp -a MERGED-ALL.txt MERGED-ALL-freedom_status.txt
+        
+        line="0";
+        while IFS= read -r table; do
+
+            ((line++))
+
+            id=$(echo $table | awk '{print $2}');
+            license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+
+            # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License".
+            if
+                [ "$license" == "GNU General Public License, version 3.0" ] ||
+                    [ "$license" == "GNU General Public License, version 2.0" ] ||
+                    [ "$license" == "GNU Lesser General Public License, version 3.0" ] ||
+                    [ "$license" == "GNU Lesser General Public License, version 2.1" ] ||
+                    [ "$license" == "Mozilla Public License, version 2.0" ] ||
+                    [ "$license" == "Mozilla Public License Version 1.1" ] ||
+                    [ "$license" == "BSD License" ] ||
+                    [ "$license" == "MIT/X11 License" ]
+            then
+
+                # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the JSON file for index.html?sort=users&page=1&tag=firefox57.json for example
+                sed -i "$line""s/$/\\tfree/" MERGED-ALL-freedom_status.txt
+
+            else
+
+                sed -i "$line""s/$/\\tnonfree/" MERGED-ALL-freedom_status.txt
+                
+            fi
+
+        done < $RUN_PATH/build/MERGED-ALL.txt
+
+        ;;
+    --generate-proposed-list)
+
+        line="0";
+        while IFS= read -r table; do
+
+            ((line++))
+
+            id=$(echo $table | awk '{print $2}');
+            source=$(echo $table | awk '{print $3}');
+            license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+            freedom_status=$(echo $table | awk '{print $4}');
+            name=$(jq .results[$line].name $RUN_PATH/build/json/$source | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+
+            if [ "$name" == "null" ]; then
+
+                name=$(jq .results[$line].name $RUN_PATH/build/json/$source | sed "s|en-GB|enGB|;" | jq .enGB | sed "s|^\"||; s|\"$||;");
+
+            fi
+
+
+            # Clean titles. Example of bad titles: "Ghostery – Privacy Ad Blocker"
+            if [[ $name = *"–"* ]]; then
+                
+                name=$(echo $name | sed 's/–/\n/g; s/-/\n/g' | head -n 1);
+                
+            fi
+
+            if [ "$freedom_status" == "free" ]; then
+                echo "{{Checkme item
+|Section=
+|Package=$name
+|Package note=
+}}"
+                fi
+            
+        done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt
+
+        ;;
+
+    --download-free-webextensions-for-gnu-and-linux)
+        
+        cd build || exit
+        rm -fr free_webextensions_for_GNU_and_Linux
+        mkdir free_webextensions_for_GNU_and_Linux
+        cd free_webextensions_for_GNU_and_Linux || exit
+
+        line="0";
+        
+	while IFS= read -r license; do
+            
+            ((line++))
+
+                # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough).
+                wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi"
+
+        done < ../license.name.enUS.txt
+
+
+        ;;
+    --verify-license-copy)
+
+        cd build || exit
+        rm -fr VERIFY-LICENSE-COPY.txt
+        cd free_webextensions_for_GNU_and_Linux || exit
+
+
+        while IFS= read -r id; do
+            
+
+
+
+            if [ ! -f "addon-$id-latest.xpi" ]; then
+                
+                echo "Nonfree" >> ../VERIFY-LICENSE-COPY.txt
+                
+            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE$"; then
+                
+                echo "LICENSE" >> ../VERIFY-LICENSE-COPY.txt
+
+            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.txt$"; then
+                
+                echo "LICENSE.txt" >> ../VERIFY-LICENSE-COPY.txt
+
+            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.md$"; then
+                
+                echo "LICENSE.md" >> ../VERIFY-LICENSE-COPY.txt
+
+
+
+            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING$"; then
+                
+                echo "COPYING" >> ../VERIFY-LICENSE-COPY.txt
+
+            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.txt$"; then
+                
+                echo "COPYING.txt" >> ../VERIFY-LICENSE-COPY.txt
+
+            elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.md$"; then
+                
+                echo "COPYING.md" >> ../VERIFY-LICENSE-COPY.txt
+
+            else
+
+                echo "Request" >> ../VERIFY-LICENSE-COPY.txt
+                
+            fi
+
+        done < ../id.txt
+
+
+
+        line="0";
+
+        while IFS= read -r status; do
+
+
+            ((line++))
+
+            average_daily_users="$(sed -n ${line}p ../average_daily_users.txt)";
+            id="$(sed -n ${line}p ../id.txt)";
+            slug="$(sed -n ${line}p ../slug.txt)";
+            license_name_en_US="$(sed -n ${line}p ../license.name.enUS.txt)";
+            homepage_en_US="$(sed -n ${line}p ../homepage.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)";
+            support_email_en_US="$(sed -n ${line}p ../support_email.en-US.txt)";
+            support_url_en_US="$(sed -n ${line}p ../support_url.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)";
+
+
+
+            if [ "$license_name_en_US" == "GNU General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/gpl-3.0.txt"; license_file_name="COPYING";
+            elif [ "$license_name_en_US" == "GNU General Public License, version 2.0" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"; license_file_name="COPYING";
+            elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/lgpl-3.0.txt"; license_file_name="COPYING";
+            elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 2.1" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt"; license_file_name="COPYING";
+            elif [ "$license_name_en_US" == "Mozilla Public License, version 2.0" ];  then license_txt="https://www.mozilla.org/media/MPL/2.0/index.txt"; license_file_name="LICENSE";
+            elif [ "$license_name_en_US" == "Mozilla Public License Version 1.1" ];  then license_txt="https://www.mozilla.org/media/MPL/1.1/index.txt"; license_file_name="LICENSE";
+            elif [ "$license_name_en_US" == "BSD License" ];  then license_txt="https://directory.fsf.org/wiki?title=License:FreeBSD"; license_file_name="LICENSE";
+            elif [ "$license_name_en_US" == "MIT/X11 License" ];  then license_txt="https://directory.fsf.org/wiki/License:X11"; license_file_name="LICENSE";
+            fi
+
+
+            if [ "$(sed -n ${line}p ../name.en-US.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-US.txt)";
+            elif [ "$(sed -n ${line}p ../name.en-GB.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-GB.txt)";
+            else name="$slug";
+            fi
+
+
+            if [ "$status" = "Request" ]; then
+
+                ((count_reports++))
+
+                # Authors that don't provide contact information will be ignored, its impossible to track conversations in Reviews anyway, because they will be deleted.
+
+                if [ "$homepage_en_US" != "null" ] || [ "$support_url_en_US" != "null" ] || [ "$support_email_en_US" != "null" ]; then
+
+                    decimals="$(echo "$average_daily_users" | sed -r ':L;s=\b([0-9]+)([0-9]{3})\b=\1,\2=g;t L')";
+
+                    echo "
+$decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/addon/$slug/
+  Request: \"You have distributed $name under $license_name_en_US on https://addons.mozilla.org/en-US/firefox/addon/$slug/, but you have not added a copy of the license. can you please download the text of the license from $license_txt and add save it to file name $license_file_name and put it in the root directory of you program?\" by contacting:";
+
+                    if [ "$homepage_en_US" != "null" ]; then echo "    * Homepage (please submit Pull Request for $license_file_name if possible): $homepage_en_US"; fi
+                    if [ "$support_url_en_US" != "null" ]; then echo "    * Support URL (please submit Pull Request for $license_file_name if possible): $support_url_en_US"; fi
+                    if [ "$support_email_en_US" != "null" ]; then echo "    * Support E-mail: $support_email_en_US"; fi
+
+                fi
+
+            fi
+
+        done < ../VERIFY-LICENSE-COPY.txt
+
+        echo "
+#########################
+$count_reports reports to submit";
+
+        ;;
+
+    --all)
+
+	$0 --fresh-build "$2"
+	$0 --make-repository-list "$2"
+	$0 --make-collection-list "$2"
+        $0 --make-custom-list "$2"
+	$0 --make-search-list "$2"
+	$0 --merge-lists "$2"
+	#        $0 --download-licenses-json "$2"
+	#        $0 --download-free-webextensions-for-gnu-and-linux "$2"
+	#        $0 --verify-license-copy "$2"
+
+	;;
+
+
+esac