Updated FreeAMO

6 files changed, 345 insertions, 61 deletions

diff --git a/subprojects/freeamo/src/get-add-ons b/subprojects/freeamo/src/get-add-ons
new file mode 100644
index 0000000..6038bd6
--- /dev/null
+++ b/subprojects/freeamo/src/get-add-ons
@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+#    This file is part of FreeAMO
+#    Copyright (C) 2017 David Hedlund
+#
+#    FreeAMO is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    FreeAMO is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+cd "build/json/search-pages" || exit
+
+function get-add-ons--download {
+
+
+    output_file="$output_file_2";
+
+    if [ "$debug" = true ]; then
+
+        pv__size=$(cat "$output_file" | head -n 3 | wc -l)
+        
+    else
+        
+        pv__size="$(wc -l < "$output_file")"
+
+    fi
+
+    echo "Download $pv__size add-ons"
+    
+    while IFS= read -r table; do
+
+        ((line++))
+
+        file=$(echo "$table" | awk '{print $1}');
+        entry=$(echo "$table" | awk '{print $2}');
+
+        source "$run_path/src/main_functions"
+        cd "$run_path/build/json/search-pages" || exit
+        amo_variables
+
+        cd "$run_path/build/add-ons" || exit
+        # Add-ons not avalible for GNU/Linux will be ignored (used to be very few though).
+        wget -nc -q "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc -q "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi"
+
+        echo "foo" # Must be here to work with pv
+        
+    done < "$output_file" | pv__progress
+
+}
+
+
+function get-add-ons--filter {
+
+    rm -f "$run_path/build/index-search-pages--free-licenses--license-file-status.txt"
+    
+    output_file="$output_file_2";
+
+    if [ "$debug" = true ]; then
+
+        pv__size=$(cat "$output_file" | head -n 3 | wc -l)
+        
+    else
+        
+        pv__size="$(wc -l < "$output_file")"
+
+    fi
+
+    echo "Filter $pv__size add-ons "
+    
+    while IFS= read -r table; do
+
+        ((line++))
+
+        file=$(echo "$table" | awk '{print $1}');
+        entry=$(echo "$table" | awk '{print $2}');
+
+        source "$run_path/src/main_functions"
+        cd "$run_path/build/json/search-pages" || exit
+        amo_variables
+
+        cd "$run_path/build/add-ons" || exit
+        license_file_text=$(unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -E "^COPYING$|^COPYING.txt$|^COPYING.md$|^LICENSE$|^LICENSE.txt$|^LICENSE.md$" | sed ':a;N;$!ba;s/\n/\t/g' || echo "NOLICENSEFILEFOUND");
+        echo -e "$file\\t$entry\\t$slug\\taddon-$id-latest.xpi\\t$license_file_text" >> "$run_path/build/index-search-pages--free-licenses--license-file-status.txt"
+        
+        echo "foo" # Must be here to work with pv
+
+    done < "$output_file" | pv__progress
+
+}
diff --git a/subprojects/freeamo/src/main_functions b/subprojects/freeamo/src/main_functions
index 782f896..3eb673e 100644
--- a/subprojects/freeamo/src/main_functions
+++ b/subprojects/freeamo/src/main_functions
@@ -62,9 +62,22 @@ function line_status {
 # Begin locale code
 
 function locale_description {
+
+echo "####################"
     
     tmp__enUS=$(jq ".results[$entry].$locale_description" "$file" | sed "s|-||;" | jq .enUS | sed "s|^\"||; s|\"$||;" )
     tmp__enGB=$(jq ".results[$entry].$locale_description" "$file" | sed "s|-||;" | jq .enGB | sed "s|^\"||; s|\"$||;" )
+    
+    tmp=$(jq ".results[$entry].$locale_description[]" "$file" | head -n 1 | sed "s|^\"||; s|\"$||;"); # Use the first avalible if English is not avaliable
+
+# David will work on this later
+# if [ "$tmp" == "" ]; then
+#
+#        # Some entries doesn't provide any `description` field but only a `summary` field.
+#        tmp=$(jq ".results[$entry].summary[]" "$file" | head -n 1 | sed "s|-||;");
+#
+#fi
+    
     if [ "$tmp__enUS" != "null" ]; then
 
         if [ "$locale_description" == "name" ]; then
@@ -98,10 +111,8 @@ function locale_description {
             
         fi
         
-    else
+    elif [ "$tmp" != "null" ]; then
 
-        tmp=$(jq ".results[$entry].$locale_description[]" "$file" | sed "s|-||;" | jq . | sed "s|^\"||; s|\"$||;"); # Use the first avalible summary if English is not avaliable
-        
         if [ "$locale_description" == "name" ]; then
             
             name="$tmp";
@@ -115,9 +126,11 @@ function locale_description {
             description="$tmp";
             
         fi
-
         
     fi
+
+echo "$name
+.results[$entry].$locale_description[] $file"
     
 }
 
diff --git a/subprojects/freeamo/src/make-wiki b/subprojects/freeamo/src/make-wiki
index 531116d..93b768d 100644
--- a/subprojects/freeamo/src/make-wiki
+++ b/subprojects/freeamo/src/make-wiki
@@ -30,21 +30,21 @@ if [ "$minimum_average_daily_users" == "100000" ]; then
 
 fi
 
+output_file="$output_file_2";
 
 if [ "$debug" = true ]; then
-    
-    pv__size=$(find . -not -name . | head -n 3 | wc -l)
+
+    pv__size=$(cat "$output_file_2" | head -n 3 | wc -l)
     
 else
     
-    pv__size=$(find . -not -name . | wc -l)
+    pv__size=$(cat "$output_file_2" |  wc -l)
     
 fi
 
-output_file="$output_file_2";
 source "$run_path/src/main_functions"
 
-echo "Make $pv__size .wiki pages"
+echo "Generate $pv__size .wiki pages."
 
 while IFS= read -r table; do
 
@@ -213,7 +213,7 @@ at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/
     fi
 
     echo "foo" # Must be here to work with pv
-done < "$output_file_2" | pv__progress
+done < "$output_file_2" # | pv__progress
 # End template
 
 
diff --git a/subprojects/freeamo/src/make-wiki (copy) b/subprojects/freeamo/src/make-wiki (copy)
new file mode 100644
index 0000000..a122592
--- /dev/null
+++ b/subprojects/freeamo/src/make-wiki (copy)
@@ -0,0 +1,224 @@
+#!/usr/bin/env bash
+#    This file is part of FreeAMO
+#    Copyright (C) 2018 David Hedlund
+#
+#    FreeAMO is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    FreeAMO is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+cd "build/json/search-pages" || exit
+
+if [ "$minimum_average_daily_users" == "100000" ]; then
+
+    echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\"
+! WebExtension
+! Description
+! Claimed license
+! Users
+! Rating
+! Updated
+! Featured" > "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki"
+
+fi
+
+output_file="$output_file_2";
+
+if [ "$debug" = true ]; then
+
+    pv__size=$(cat "$output_file_2" | head -n 3 | wc -l)
+    
+else
+    
+    pv__size=$(cat "$output_file_2" |  wc -l)
+    
+fi
+
+source "$run_path/src/main_functions"
+
+echo "Generate $pv__size .wiki pages."
+
+while IFS= read -r table; do
+
+    ((++line))
+    
+    if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then
+        
+        file=$(echo "$table" | awk '{print $1}');
+        entry=$(echo "$table" | awk '{print $2}');
+
+        source "$run_path/src/main_functions"
+        amo_variables
+        amo_variables__licenses
+
+        if [ "$license__amo_full_name" == "BSD License" ]; then license__fsd_short_name="BSD 2Clause";
+        elif [ "$license__amo_full_name" == "GNU General Public License, version 3.0" ]; then license__fsd_short_name="GPLv3";
+        elif [ "$license__amo_full_name" == "GNU General Public License, version 2.0" ]; then license__fsd_short_name="GPLv2";
+        elif [ "$license__amo_full_name" == "GNU Lesser General Public License, version 3.0" ]; then license__fsd_short_name="LGPLv3";
+        elif [ "$license__amo_full_name" == "GNU Lesser General Public License, version 2.1" ]; then license__fsd_short_name="LGPLv2.1";
+        elif [ "$license__amo_full_name" == "MIT/X11 License" ]; then license__fsd_short_name="X11";
+        elif [ "$license__amo_full_name" == "Mozilla Public License, version 2.0" ]; then license__fsd_short_name="MPLv2.0";
+        elif [ "$license__amo_full_name" == "Mozilla Public License Version 1.1" ]; then license__fsd_short_name="MPLv1.1";
+        fi
+
+        # Start description code
+
+        source "$run_path/src/main_functions"
+        locale_description="summary"
+        locale_description
+        locale_description="description"
+        locale_description
+        short_description=$(echo "$summary" | format_description)
+        full_description=$(echo "$description" | format_description)
+        
+        # End description code
+
+        # The homepage field is mandatory so let's remove "null" from it because we don't need to evaluat it
+        homepage_url=$(jq .results[$entry].homepage "$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|null||");
+        license_copyright=$(jq .results[$entry].authors[].name "$file" | sed "s|^\"||; s|\"$||;");
+        bayesian_average=$(jq .results[$entry].ratings.bayesian_average "$file");
+        bayesian_average__simple=$(printf "%.1f" "${bayesian_average//./,}");
+
+        is_featured=$(jq .results[$entry].is_featured "$file");
+        
+        last_review_date=$(date +"%Y/%m/%d");
+        last_review_by="$mediawikibot__bot_password__Username";
+        submitted_by="$mediawikibot__bot_password__Username";
+
+        if [[ "$(jq .results[$entry].current_version.compatibility.seamonkey "$file")" != "null" ]]; then
+            set_seamonkey="Iceape,";
+        fi
+
+        if [[ "$(jq .results[$entry].current_version.compatibility.firefox "$file")" != "null" ]]; then
+            set_icecat="IceCat,";
+        fi
+
+        #	    if [[ "$(jq .results[$entry].current_version.compatibility.android $file)" != "null" ]] || [[ "$(jq .results[$entry].current_version.compatibility.mobile $file)" != "null" ]]; then
+        #		set_icecatmobile="IceCatMobile,";
+        #	    fi
+
+        if [[ "$(jq .results[$entry].current_version.compatibility.thunderbird "$file")" != "null" ]]; then
+            set_thunderbird="Icedove,";
+        fi
+
+        # Do not list IceCatMobile
+        extension_of=$(echo "$set_seamonkey$set_icecat$set_thunderbird" | sed "s|,$||;");
+
+
+
+        github_true=$(echo "$homepage_url" | grep "github.com");
+
+        if [[ $github_true != "" ]]; then
+
+            # Always use https, and remove anchors
+            homepage_url__for__vcs_checkout_command="https://github.com$(echo "$github_true" | sed "s|https://github.com||; s|http://github.com||; s|#|\\n|;" | head -n 1)";
+            # Remove trailing slash
+            homepage_url__for__vcs_checkout_command="${homepage_url__for__vcs_checkout_command%/}"
+            vcs_checkout_command="git clone $homepage_url__for__vcs_checkout_command.git";
+
+        fi
+
+        if [ "$name" == "GNU LibreJS" ]; then
+
+            is_gnu="Yes";
+
+        else
+
+            is_gnu="No";
+
+        fi
+
+        support_url=$(jq .results[$entry].support_url "$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+        last_updated=$(jq .results[$entry].last_updated $file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g");
+
+        # List JSON structure: cat json/$file | js '.'
+        # XML value "homepage" is not always set in AMO API, but the XML require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically.
+        echo "{{Entry" > "$run_path/build/wiki/$slug.wiki"
+
+        # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341
+        # Avoid specific versions since it's not compatible with all IceCat versions:
+        # |Version download=https://addons.mozilla.org/firefox/downloads/latest/$id/addon-$version_id-latest.xpi
+        # |Version identifier=$version
+        # |Version date=$version_date
+
+        
+        # Do add these fields since we refer to the download page in the `|Version download` field that lists compatible versions
+        # |Version identifier=52.6.0
+        # |Version date=2018/02/26
+        # |Version status=stable
+        
+        echo "|Name=$name
+|Short description=$short_description
+|Full description=$full_description
+|Homepage URL=$homepage_url
+|Extension of=$extension_of
+|VCS checkout command=$vcs_checkout_command
+|Documentation note=Extra data (https://addons.mozilla.org/api/v$api_version/addons/addon/$slug/):
+* Users: $average_daily_users
+* Rating: $bayesian_average__simple
+* Updated: $last_updated
+* Featured: $is_featured
+|Version download=https://addons.mozilla.org/en-US/firefox/addon/$slug/versions/
+|Last review by=$last_review_by
+|Last review date=$last_review_date
+|Submitted by=$submitted_by
+|Submitted date=
+|Is GNU=$is_gnu
+}}" >> "$run_path/build/wiki/$slug.wiki"
+
+
+
+        # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/
+        echo "{{Project license
+|License=$license__fsd_short_name
+|License copyright=$license_copyright
+|License note=\"License: $license__amo_full_name\" listed
+at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/
+}}" >> "$run_path/build/wiki/$slug.wiki"
+
+        if [ "$support_url" != "null" ];
+        then
+
+            echo "{{Resource
+|Resource audience=Users
+|Resource kind=Support
+|Resource URL=$support_url
+}}" >> "$run_path/build/wiki/$slug.wiki"
+
+        fi
+
+        if [ "$minimum_average_daily_users" == "100000" ]; then
+            
+        echo "|-
+| [[$slug|$name]]
+| $short_description
+| [[License:$license__fsd_short_name|$license__fsd_short_name]]
+| $average_daily_users
+| $bayesian_average__simple
+| $last_updated
+| $is_featured" >> "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki"
+
+        fi
+
+
+        
+    fi
+
+    echo "$name" # Must be here to work with pv
+done < "$output_file_2" # | pv__progress
+# End template
+
+
+if [ "$minimum_average_daily_users" == "100000" ]; then
+
+    echo "|}" >> "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki"
+
+fi
diff --git a/subprojects/freeamo/src/search-pages b/subprojects/freeamo/src/search-pages
index 45ed716..603c750 100644
--- a/subprojects/freeamo/src/search-pages
+++ b/subprojects/freeamo/src/search-pages
@@ -56,6 +56,7 @@ function search-pages--download() {
         
         pv__discard=true;
         echo "Cannot show a progress counter since minimum_average_daily_users is set to more than 0 in freeamo.conf."
+        extra_text="max"
         
     else
 
@@ -75,7 +76,7 @@ function search-pages--download() {
         if ( [ "$debug" = true ] && [ "$page" -le "3" ] || [ "$debug" = false ] ) && ( [ "$average_daily_users" -ge "$minimum_average_daily_users" ] ); then
 
             source "$run_path/src/main_functions"
-            echo "Downloading search page: $page of $page_count"
+            echo "Downloading search page: $page of $extra_text $page_count"
             wget_page
 
         else
diff --git a/subprojects/freeamo/src/unmaintained/download-free-webextensions b/subprojects/freeamo/src/unmaintained/download-free-webextensions
deleted file mode 100644
index 7084b65..0000000
--- a/subprojects/freeamo/src/unmaintained/download-free-webextensions
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-#    This file is part of FreeAMO
-#    Copyright (C) 2017 David Hedlund
-#
-#    FreeAMO is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    FreeAMO is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-cd build || exit
-rm -fr free_webextensions
-mkdir free_webextensions
-cd free_webextensions || exit
-
-
-
-# Begin template
-unset file
-while IFS= read -r table; do
-
-    verify_variables "$1"
-
-    if [ "$approved_program" = true ]; then
-
-        # End template
-
-        id=$(echo $table | awk '{print $2}');
-        file=$(echo $table | awk '{print $3}');
-        slug=$(echo $table | awk '{print $4}');
-
-        # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough).
-        wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi"
-        
-        
-
-        # Begin template
-    fi
-
-    ((line++))
-    
-done < $run_path/build/MERGED-ALL-freedom_status.txt
-# End template