updated license-reporter

1 files changed, 262 insertions, 109 deletions

diff --git a/addons.mozilla.org-fsd/get-data/license-reporter b/addons.mozilla.org-fsd/get-data/license-reporter
index 47c28a8..9bf4263 100755
--- a/addons.mozilla.org-fsd/get-data/license-reporter
+++ b/addons.mozilla.org-fsd/get-data/license-reporter
@@ -34,7 +34,7 @@
 
 if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi
 
-minimum_average_daily_users="100000";
+minimum_average_daily_users="10000";
 
 SCRIPTSRC=$(readlink -f "$0" || echo "$0")
 RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .)
@@ -85,6 +85,131 @@ function line_status {
 }
 
 
+function verify_variables {
+
+    if [ -z ${file+x} ] || [ "$file" != "$(echo $table | awk '{print $3}')" ]; then
+
+        line="0"; # should not be 1
+
+    fi
+
+    average_daily_users=$(echo $table | awk '{print $1}');
+    id=$(echo $table | awk '{print $2}');
+    file=$(echo $table | awk '{print $3}');
+    slug=$(echo $table | awk '{print $4}');
+    name=$(echo "$table" | cut -f 5);
+    has_eula=$(echo "$table" | cut -f 6);
+
+
+   
+    firefox_max_version="$(jq .results[$line].current_version.compatibility.firefox.max $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|.\*||")";
+    nativeMessaging="$(jq .results[$line].current_version.files[0].permissions $RUN_PATH/build/json/$file | grep "nativeMessaging")";
+
+    # Values are either an integer or "*"
+    re='^[0-9]+$'
+    if [[ $firefox_max_version =~ $re ]] && [ "$firefox_max_version" -ge 60 ] || [ "$firefox_max_version" == '*' ]; then
+
+        #                echo "$firefox_max_version"
+        firefox_max_version__pass=true;
+
+    else
+
+        unset firefox_max_version__pass
+        
+    fi
+
+    if [ ! -n "$nativeMessaging" ]; then
+
+        nativeMessaging__pass=true;
+
+    else
+
+        nativeMessaging__pass=false;
+        
+    fi
+
+    echo "$nativeMessaging__pass"
+
+    
+    # curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging
+    
+
+    # API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668
+    # The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies.
+
+    # Old solution:
+
+    # 1: Add this to license-reporter-dependencies:
+    #belgium-eid	eid-mw
+
+    # 2: Uncomment this code:
+    #    depends_on_external_program=false;
+    #    while IFS= read -r dependencies; do
+    #
+    #        dependency_slug=$(echo $dependencies | awk '{print $1}');
+    #        dependency=$(echo "$dependencies" | cut -f 2);
+    #
+    #        if [ "$dependency_slug" == "$slug" ]; then
+    #
+    #            depends_on_external_program=true;
+    #
+    #        fi
+    #
+    #    done < $RUN_PATH/license-reporter-dependencies
+
+
+
+
+    approved_program=false;
+    if [ "$has_eula" == "eulafree" ] && [ "$firefox_max_version__pass" = true ] && [ "$nativeMessaging__pass" = true ]; then
+
+        if [ "$1" == "--get-licenses" ]; then
+
+            approved_program=true;
+            
+        else
+
+            license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+
+
+
+            # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License".
+            # See https://addons.mozilla.org/en-US/developers/addon/<your add-on>/ownership for the current add-on list. Note that not all licenses are added to the list.
+            # DuckDuckGo Privacy Essentials is distributed under Apache License 2.0.
+            # Apache License, version 2.0 is not on the add-on license list - https://github.com/mozilla/addons-server/issues/8545
+            if
+                [ "$license" == "Apache License, Version 2.0" ] || # Custom license
+                    [ "$license" == "Apache License, version 2.0" ] || # Custom license
+                    [ "$license" == "BSD License" ] ||
+                    [ "$license" == "GNU General Public License, version 3.0" ] ||
+                    [ "$license" == "GNU General Public License, version 2.0" ] ||
+                    [ "$license" == "GNU Lesser General Public License, version 3.0" ] ||
+                    [ "$license" == "GNU Lesser General Public License, version 2.1" ] ||
+                    [ "$license" == "MIT/X11 License" ] ||
+                    [ "$license" == "Mozilla Public License, version 2.0" ] ||
+                    [ "$license" == "Mozilla Public License Version 1.1" ]
+            then
+
+                # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files.
+                freedom_status="free";
+
+            elif [ "$license" == "All Rights Reserved" ]; then
+
+                freedom_status="nonfree";
+
+            else
+
+                freedom_status="unknown";
+            fi
+            
+            [ "$freedom_status" == "free" ]
+            
+        fi
+
+    fi
+    
+}
+
 case "$1" in
 
     ""|-help)
@@ -110,8 +235,10 @@ OPTIONS
 	$0 --get-licenses
             The licenses are not added to the general JSON file. This option will download the neccesary license JSON files for all add-ons.
         $0 --download-licenses-json
-        $0 --download-free-webextensions-for-gnu-and-linux
+        $0 --download-free-webextensions
+            Downloads the latest version of the free WebExtensions.
         $0 --verify-license-copy
+            Search for license files in the root directory in the downloaded WebExtensions (.xpi files)
         $0  --merged-free
             Generates MERGED-ALL-freedom_status.txt
         $0  --make-wiki
@@ -367,78 +494,57 @@ s/,/\n/g;
         ;;
     --get-licenses)
 
+        # API: Please adopt SPDX - https://github.com/mozilla/addons-server/issues/8706
+        # Allow GNU [L]GPL x.x or later - https://github.com/mozilla/addons-server/issues/8707
+        
         cd build/ || exit
         mkdir -p json/current_versions
         cd json/current_versions || exit
 
         line="0"; # Should not be 1
-        
+        all_lines="1" # Should not be 0
+
+        # Begin template
+        unset file
         while IFS= read -r table; do
 
-            id=$(echo $table | awk '{print $2}');
-            file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}');
+            verify_variables "$1"
 
-            line_status
-            
-            # echo "$file $line $id"
-            
-            wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json"
-            
-            previous_file="$file";
-            
-        done < $RUN_PATH/build/MERGED-ALL.txt
-        
-        ;;
-    --merged-free)
-        
-        cd build/ || exit
-        cp -a MERGED-ALL.txt MERGED-ALL-freedom_status.txt
-        
-        line="1"; # Should not be "0"
-        while IFS= read -r table; do
+            if [ "$approved_program" = true ]; then
+                # End template
 
-            id=$(echo $table | awk '{print $2}');
-            license=$(jq .license.name "$RUN_PATH/build/json/current_versions/$id.json" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
-            
-            # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License".
-            # See https://addons.mozilla.org/en-US/developers/addon/<your add-on>/ownership for the current add-on list. Note that not all licenses are added to the list.
-            # DuckDuckGo Privacy Essentials is distributed under Apache License 2.0.
-            # Apache License, version 2.0 is not on the add-on license list - https://github.com/mozilla/addons-server/issues/8545
-            if
-                [ "$license" == "Apache License, Version 2.0" ] || # Custom license
-                    [ "$license" == "Apache License, version 2.0" ] || # Custom license
-                    [ "$license" == "BSD License" ] ||
-                    [ "$license" == "GNU General Public License, version 3.0" ] ||
-                    [ "$license" == "GNU General Public License, version 2.0" ] ||
-                    [ "$license" == "GNU Lesser General Public License, version 3.0" ] ||
-                    [ "$license" == "GNU Lesser General Public License, version 2.1" ] ||
-                    [ "$license" == "MIT/X11 License" ] ||
-                    [ "$license" == "Mozilla Public License, version 2.0" ] ||
-                    [ "$license" == "Mozilla Public License Version 1.1" ]
-            then
+                echo "ddd";
+                
+                id=$(echo $table | awk '{print $2}');
+                file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}');
+                has_eula=$(echo "$table" | cut -f 6);
+                
+                #                line_status
 
-                # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files.
-                sed -i "$line""s/$/\\tfree/" MERGED-ALL-freedom_status.txt
+                total=$(wc -l < $RUN_PATH/build/MERGED-ALL.txt)
+                echo "#######################################
+Entry: $all_lines (approved) / $total"
 
-            elif [ "$license" == "All Rights Reserved" ]; then
+
+                #                echo $(jq ".results[$line].slug" "$file");
+
+                echo "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/"
 
                 
-                sed -i "$line""s/$/\\tnonfree/" MERGED-ALL-freedom_status.txt
+                # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json"
                 
-            else
+                previous_file="$file";
+                ((all_lines++))
 
-                sed -i "$line""s/$/\\tunknown/" MERGED-ALL-freedom_status.txt
-                
+
+                # Begin template
             fi
 
-            # Add license for analysis
-            # don't use "/" as delimeters since some licenses are "MIT/X11 License"
-            sed -i "$line""s|$|\\t$license|" MERGED-ALL-freedom_status.txt
-            
             ((line++))
             
         done < $RUN_PATH/build/MERGED-ALL.txt
-
+        # End template
+        
         ;;
     --make-wiki)
 
@@ -453,44 +559,15 @@ s/,/\n/g;
 ! Rating
 ! Updated" > "wiki/IceCat WebExtensions (proposed).wiki"
 
-        
+
+        # Begin template
         unset file
         while IFS= read -r table; do
 
-            if [ -z ${file+x} ] || [ "$file" != "$(echo $table | awk '{print $3}')" ]; then
-
-                line="0"; # should not be 1
-
-            fi
-
-            average_daily_users=$(echo $table | awk '{print $1}');
-            id=$(echo $table | awk '{print $2}');
-            file=$(echo $table | awk '{print $3}');
-            name=$(echo "$table" | cut -f 5);
-            has_eula=$(echo "$table" | cut -f 6);
-            freedom_status=$(echo "$table" | cut -f 7);
-            
-            license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
-            firefox_max_version="$(jq .results[$line].current_version.compatibility.firefox.max $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|.\*||")";
-
-
-            # Values are either an integer or "*"
-            re='^[0-9]+$'
-            if [[ $firefox_max_version =~ $re ]] && [ "$firefox_max_version" -ge 60 ] || [ "$firefox_max_version" == '*' ]; then
-
-#                echo "$firefox_max_version"
-                firefox_max_version__pass=true;
-
-            else
-
-                unset firefox_max_version__pass
-                
-            fi
+            verify_variables "$1"
             
-            if [ "$freedom_status" == "free" ] && [ "$has_eula" == "eulafree" ] && [ "$firefox_max_version__pass" = true ]; then
-
-
-                # require min version 60 ddd
+            if [ "$approved_program" = true ]; then
+                # End template
                 
                 ####################################
                 # Build the core wiki pages
@@ -498,6 +575,7 @@ s/,/\n/g;
                 average_daily_users=$(echo $table | awk '{print $1}');
                 id=$(echo $table | awk '{print $2}');
                 file=$(echo $table | awk '{print $3}');
+                slug=$(echo $table | awk '{print $4}');
                 name=$(echo "$table" | cut -f 5);
                 has_eula=$(echo "$table" | cut -f 6);
                 freedom_status=$(echo "$table" | cut -f 7);
@@ -520,11 +598,24 @@ s/,/\n/g;
 
                 
 
-                short_description=$(jq .results[$line].summary $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|\\\n|\n|g; s|\\\||g");
+                short_description_raw=$(jq .results[$line].summary $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|\\\n|\n|g; s|\\\||g;
+s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues
+s|http|\nhttp|g;
+");
+                short_description_again=$(echo "$short_description_raw" | sed "/outgoing.prod.mozaws.net/d;");
 
-#                echo "$short_description"
+                short_description=$(echo "$short_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g');
                 
-                full_description=$(jq .results[$line].description $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+                full_description_raw=$(jq .results[$line].description $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;
+ s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues
+");
+                full_description_again=$(echo "$full_description_raw" | sed "/outgoing.prod.mozaws.net/d;");
+
+                full_description=$(echo "$full_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g');
+
+
+                
+
                 homepage_url=$(jq .results[$line].homepage $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
                 version=$(jq .results[$line].current_version.version $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;");
                 version_date=$(jq .results[$line].last_updated $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g");
@@ -556,7 +647,7 @@ s/,/\n/g;
                 # Do not list IceCatMobile
                 extension_of=$(echo "$set_seamonkey$set_icecat$set_thunderbird" | sed "s|,$||;");
 
-               
+                
 
                 github_true=$(echo "$homepage_url" | grep "github.com");
 
@@ -603,6 +694,7 @@ s/,/\n/g;
 |Is GNU=$is_gnu
 }}" >> wiki/$id.wiki
 
+                
 
 	        # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/
 	        echo "{{Project license
@@ -632,43 +724,96 @@ s/,/\n/g;
 
 
                 
-                
+                # Begin template
             fi
 
             ((line++))
             
-            
         done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt
-
+        # End template
+        
         echo "|}" >> "wiki/IceCat WebExtensions (proposed).wiki"
         
         ;;
 
-    --download-free-webextensions-for-gnu-and-linux)
+    --download-free-webextensions)
         
         cd build || exit
-        rm -fr free_webextensions_for_GNU_and_Linux
-        mkdir free_webextensions_for_GNU_and_Linux
-        cd free_webextensions_for_GNU_and_Linux || exit
+        rm -fr free_webextensions
+        mkdir free_webextensions
+        cd free_webextensions || exit
 
-        line="0";
-        
-	while IFS= read -r license; do
-            
-            ((line++))
 
-            # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough).
-            wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi"
 
-        done < ../license.name.enUS.txt
+        # Begin template
+        unset file
+        while IFS= read -r table; do
+
+            verify_variables "$1"
+
+            if [ "$approved_program" = true ]; then
+
+                # End template
+
+                id=$(echo $table | awk '{print $2}');
+                file=$(echo $table | awk '{print $3}');
+                slug=$(echo $table | awk '{print $4}');
+
+                # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough).
+                wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi"
+                
+                
 
+                # Begin template
+            fi
+
+            ((line++))
+            
+        done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt
+        # End template
 
         ;;
     --verify-license-copy)
 
         cd build || exit
         rm -fr VERIFY-LICENSE-COPY.txt
-        cd free_webextensions_for_GNU_and_Linux || exit
+        cd free_webextensions || exit
+
+
+
+        # Begin template
+        unset file
+        while IFS= read -r table; do
+
+            verify_variables "$1"
+
+            if [ "$approved_program" = true ]; then
+                # End template
+
+                id=$(echo $table | awk '{print $2}');
+                file=$(echo $table | awk '{print $3}');
+                slug=$(echo $table | awk '{print $4}');
+                
+                echo "--------------------------
+$name ($id): $license";
+                unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING"
+                unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.txt"
+                unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.md"
+                unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE"
+                unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.txt"
+                unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.md"
+                
+                # Begin template
+            fi
+
+            ((line++))
+            
+        done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt
+        # End template
+
+        
+        ;;
+    --verify-license-copy-old)
 
 
         while IFS= read -r id; do
@@ -782,7 +927,15 @@ $count_reports reports to submit";
 
     --make-xml)
         # Deprecated.
-        # sha1sum file.xml
+
+
+        # XML pages look like this:
+        #...
+        #      <sha1>tlybh1k9mnxz92kccg46hqv9jvyrkp0</sha1>
+        #    </revision>
+        #  </page>
+        #</mediawiki>
+        #
         # However. It's not possible to insert the checsum in <sha1> for the page since it will break the checksum itself.
 
 	# Get number of edits: