FreeAMO. General bug fixes. Fixed the description fields for .wiki pages

8 files changed, 332 insertions, 152 deletions

diff --git a/subprojects/freeamo/default.conf b/subprojects/freeamo/default.conf
new file mode 100644
index 0000000..bd4d01d
--- /dev/null
+++ b/subprojects/freeamo/default.conf
@@ -0,0 +1,20 @@
+# Read the API reference documentation for parameter values:
+# https://addons-server.readthedocs.io/en/latest/topics/api/addons.html
+
+# Mozilla API
+api_version=
+
+# Mozilla API parameters: Add-ons -> Search
+app=""
+appversion=
+page_size=
+platform=""
+sort=""
+type=""
+
+# Mozilla API parameters: Add-ons -> Detail
+minimum_average_daily_users=
+
+# Extra
+notes=""
+mediawikibot__bot_password__Username="" # Not mandatory
diff --git a/subprojects/freeamo/freeamo b/subprojects/freeamo/freeamo
index dec2d5c..bd1df0e 100755
--- a/subprojects/freeamo/freeamo
+++ b/subprojects/freeamo/freeamo
@@ -15,20 +15,22 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-source freeamo.conf
-
 scriptsrc=$(readlink -f -- "${BASH_SOURCE[0]}")
 run_path=$(dirname "${scriptsrc}" || echo .)
 filename=$(basename "$0");
 
+source "$run_path/src/main_functions"
+source "$run_path/freeamo.conf"
+
+
+#source "$run_path/src/err"
+
 output_file_1="$run_path/build/index-search-pages.txt";
 output_file_2="$run_path/build/index-search-pages--free-licenses.txt"
 
-# source "$run_path/build/err"
-
 # Evaluate debug mode
 if [ "$1" == "--debug" ] || [ "$2" == "--debug" ]; then
-
+    
     debug=true
     debug_arg="--debug";
     
@@ -55,14 +57,31 @@ if [[ $exit == true ]]; then exit 1; fi
 
 case "$1" in
 
-    ""|--debug)
+    ""|--debug|--normal-mode)
 
-        $0  --disclaimer "$debug_arg"
-        $0  --remove-build "$debug_arg"
-        $0  --search-pages "$debug_arg"
-        $0  --get-licenses "$debug_arg"
-        $0  --make-directory.fsf.org-wiki "$debug_arg"
+        freeamo_command="$0  --conf-note $debug_arg :"
+        echo "$freeamo_command"; $freeamo_command
+        
+        freeamo_command="$0  --remove-build $debug_arg :"
+        echo "$freeamo_command"; $freeamo_command
 
+        freeamo_command="$0  --conf-copy $debug_arg :"
+        echo "$freeamo_command"; $freeamo_command
+        
+        freeamo_command="$0  --search-pages $debug_arg :"
+        echo "$freeamo_command"; $freeamo_command
+
+        freeamo_command="$0  --get-licenses $debug_arg :"
+        echo "$freeamo_command"; $freeamo_command
+
+        if [ ! "$1" == "--normal-mode" ]; then
+            
+            freeamo_command="$0  --make-directory.fsf.org-wiki $debug_arg :"
+            echo "$freeamo_command"; $freeamo_command
+            
+        fi
+        
+        
         ;;
     
     -help)
@@ -70,10 +89,13 @@ case "$1" in
 	[ "$1" = "" ] && echo "Usage: $filename [--option] [--debug]
 
 OPTIONS
-	$0 --disclaimer
+f.org-wiki (they are used by Free Software Directory developers)
+	$0 --conf-note
             Display which icecat version the add-on will be compatible with.
 	$0 --remove-build
             Remove the build directory if it exists.
+	$0 --conf-copy
+            Copy the conf file to the build directory.
 	$0 --search-pages
             Downloads the most popular WebExtensions from https://addons.mozilla.org/en-US/firefox/search/...
 	$0 --get-licenses
@@ -100,23 +122,35 @@ Disabled options due to limited development time
 
 	;;
 
-    --disclaimer)
-
-        echo "$notes"
+    --conf-note)
         
+        echo "$notes"
+
         ;;
-    
+
     --remove-build)
 
         rm -fr "$run_path/build"
 
         ;;
+    
+    --conf-copy)
+        
+        mkdir -p "$run_path/build/"
+        echo -e "# [This file is a copy used as a receipt to declare the settings that were used to generate the files.]\\n" > "$run_path/build/freeamo.conf"
+        cat "$run_path/freeamo.conf" >> "$run_path/build/freeamo.conf"
+        sed -i "/^appversion=/ s/$/ # $appversion at the time/" "$run_path/build/freeamo.conf"
+        
+        ;;
 
     --search-pages)
 
         # This option is named after named after https://addons.mozilla.org/en-US/firefox/search/
-        $0  --search-pages--download "$debug_arg"
-        $0  --search-pages--filter "$debug_arg"
+        freeamo_command="$0  --search-pages--download $debug_arg :"
+        echo "  ↳$freeamo_command"; $freeamo_command
+        
+        freeamo_command="$0  --search-pages--filter $debug_arg :"
+        echo "  ↳$freeamo_command"; $freeamo_command
         
         ;;
     
@@ -138,8 +172,11 @@ Disabled options due to limited development time
     
     --get-licenses)
 
-        $0  --get-licenses--download "$debug_arg"
-        $0  --get-licenses--filter "$debug_arg"
+        freeamo_command="$0  --get-licenses--download $debug_arg :"
+        echo "  ↳$freeamo_command"; $freeamo_command
+        
+        freeamo_command="$0  --get-licenses--filter $debug_arg :"
+        echo "  ↳$freeamo_command"; $freeamo_command
         
         ;;
 
diff --git a/subprojects/freeamo/freeamo-org.conf b/subprojects/freeamo/freeamo-org.conf
new file mode 100644
index 0000000..74ea564
--- /dev/null
+++ b/subprojects/freeamo/freeamo-org.conf
@@ -0,0 +1,19 @@
+# Read the API reference documentation for parameter values:
+# https://addons-server.readthedocs.io/en/latest/topics/api/addons.html
+
+# Mozilla API
+api_version=3
+
+# Mozilla API parameters: Add-ons -> Search
+app="firefox"
+appversion=$(debranded_package="icecat"; latest_debranded_trisquel_version);
+page_size=50
+platform="linux"
+sort="users"
+type="extension"
+
+# Mozilla API parameters: Add-ons -> Detail
+minimum_average_daily_users=0
+
+# Extra
+notes="Extensions for the latest Trisquel GNU/Linux icecat package version."
diff --git a/subprojects/freeamo/freeamo.conf b/subprojects/freeamo/freeamo.conf
index fd133e7..910871f 100644
--- a/subprojects/freeamo/freeamo.conf
+++ b/subprojects/freeamo/freeamo.conf
@@ -1,13 +1,20 @@
+# Read the API reference documentation for parameter values:
+# https://addons-server.readthedocs.io/en/latest/topics/api/addons.html
+
+# Mozilla API
+api_version=3
+
 # Mozilla API parameters: Add-ons -> Search
 app="firefox"
-appversion="$(wget -qO- "http://archive.trisquel.info/trisquel/pool/main/i/icecat/?C=N;O=D" | grep icecat_ | head -n 1 | sed "s|\"|\\n|g; s|-|\\n|g" | grep icecat_ | sed "s|icecat_||" | head -n 1)" # Latest icecat version for Trisquel
-page_size=50 # page_size=50 -- this is the maximum accepted number, it makes the download much faster: 20 seconds for 10 x page_size=50, 35 seconds for 20 x page_size=25 (default)
+appversion=60 # Earliest IceCat version (not released yet)
+page_size=50
 platform="linux"
-sort="users" # sort=users. sort=name not working yet - https://github.com/mozilla/addons-server/issues/7600
+sort="users"
 type="extension"
 
 # Mozilla API parameters: Add-ons -> Detail
-minimum_average_daily_users="10000"
+minimum_average_daily_users=100000
 
 # Extra
-notes="WebExtensions compatible with the latest icecat package version for the latest stable Trisquel GNU/Linux distribution. GNU IceCat is based on Firefox Extended Support Release (ESR)."
+notes="Extensions for the latest Trisquel GNU/Linux icecat package version."
+mediawikibot__bot_password__Username="wikisysbot"
diff --git a/subprojects/freeamo/src/get-licenses b/subprojects/freeamo/src/get-licenses
index c7e5559..79b6da4 100644
--- a/subprojects/freeamo/src/get-licenses
+++ b/subprojects/freeamo/src/get-licenses
@@ -18,12 +18,11 @@
 # API: Please adopt SPDX - https://github.com/mozilla/addons-server/issues/8706
 # Allow GNU [L]GPL x.x or later - https://github.com/mozilla/addons-server/issues/8707
 
-cd "build/json/search-pages"
+cd "build/json/search-pages" || exit
 
 function get-licenses--download {
 
-    echo -e "\\nDownloading license pages:"
-
+    # API: Add "last_license" to the search pages - https://github.com/mozilla/addons-server/issues/8945
     
     output_file="$output_file_1";
     pv__size="$(wc -l < "$output_file")"
@@ -35,8 +34,8 @@ function get-licenses--download {
         
         if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then
             
-            file=$(echo $table | awk '{print $1}');
-            entry=$(echo $table | awk '{print $2}');
+            file=$(echo "$table" | awk '{print $1}');
+            entry=$(echo "$table" | awk '{print $2}');
 
             source "$run_path/src/main_functions"
             amo_variables
@@ -52,14 +51,12 @@ function get-licenses--download {
 
 
         echo "foo" # Must be here to work with pv
-    done < "$output_file_1" | progress
+    done < "$output_file_1" | pv__progress
 
 }
 
 function get-licenses--filter {
 
-    echo -e "\\nFilter license pages:"
-
     output_file="$output_file_1";
     pv__size="$(wc -l < "$output_file")"
     source "$run_path/src/main_functions"
@@ -72,8 +69,8 @@ function get-licenses--filter {
         
         if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then
             
-            file=$(echo $table | awk '{print $1}');
-            entry=$(echo $table | awk '{print $2}');
+            file=$(echo "$table" | awk '{print $1}');
+            entry=$(echo "$table" | awk '{print $2}');
 
             source "$run_path/src/main_functions"
             amo_variables
@@ -101,7 +98,7 @@ function get-licenses--filter {
 
                 # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files.
                 #                echo "$name: $license__amo_full_name"
-                echo "$file $entry" >> "$output_file_2"
+                echo "$file $entry $slug" >> "$output_file_2"
                 
                 freedom_status="free";
 
@@ -120,6 +117,6 @@ function get-licenses--filter {
 
         echo "foo" # Must be here to work with pv
         
-    done < "$output_file_1" | progress
+    done < "$output_file_1" | pv__progress
 
 }
diff --git a/subprojects/freeamo/src/main_functions b/subprojects/freeamo/src/main_functions
index a96ada3..7a66cff 100644
--- a/subprojects/freeamo/src/main_functions
+++ b/subprojects/freeamo/src/main_functions
@@ -15,6 +15,13 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+function latest_debranded_trisquel_version {
+
+    # All debranded programs starts with "i" so all of them are in Trisquels "/i/" directory
+    wget -qO- "http://archive.trisquel.info/trisquel/pool/main/i/$debranded_package/?C=N;O=D" | sed "s|\"|\\n|g; s|-|\\n|g" | grep "$debranded_package"_ | head -n 1 | sed "s|"$debranded_package"_||"
+    
+}
+
 function check_files {
 
     if [ ! -f "$run_path/FreeAMO-$check_type" ]; then
@@ -26,7 +33,7 @@ function check_files {
 
 
 	# Make sure all links are using SSL
-	if grep -q "http://addons.mozilla.org/" $run_path/FreeAMO-$check_type; then
+	if grep -q "http://addons.mozilla.org/" "$run_path/FreeAMO-$check_type"; then
 	    
 	    echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)."
 	    echo "Exiting" && exit
@@ -52,22 +59,82 @@ function line_status {
 
 }
 
+# Begin locale code
+
+function locale_description {
+    
+    tmp__enUS=$(jq ".results[$entry].$locale_description" "$file" | sed "s|-||;" | jq .enUS | sed "s|^\"||; s|\"$||;" )
+    tmp__enGB=$(jq ".results[$entry].$locale_description" "$file" | sed "s|-||;" | jq .enGB | sed "s|^\"||; s|\"$||;" )
+    if [ "$tmp__enUS" != "null" ]; then
+
+        if [ "$locale_description" == "name" ]; then
+            
+            name="$tmp__enUS";
+
+        elif [ "$locale_description" == "summary" ]; then
+            
+            summary="$tmp__enUS";
+
+        elif [ "$locale_description" == "description" ]; then
+            
+            description="$tmp__enUS";
+            
+        fi
+
+
+    elif [ "$tmp__enGB" != "null" ]; then
+
+        if [ "$locale_description" == "name" ]; then
+            
+            name="$tmp__enGB";
+
+        elif [ "$locale_description" == "summary" ]; then
+            
+            summary="$tmp__enGB";
+
+        elif [ "$locale_description" == "description" ]; then
+            
+            description="$tmp__enGB";
+            
+        fi
+        
+    else
+
+        tmp=$(jq ".results[$entry].$locale_description[]" "$file" | sed "s|-||;" | jq . | sed "s|^\"||; s|\"$||;"); # Use the first avalible summary if English is not avaliable
+        
+        if [ "$locale_description" == "name" ]; then
+            
+            name="$tmp";
+
+        elif [ "$locale_description" == "summary" ]; then
+            
+            summary="$tmp";
+
+        elif [ "$locale_description" == "description" ]; then
+            
+            description="$tmp";
+            
+        fi
+
+        
+    fi
+    
+}
+
+# End locale code
+
+
 function amo_variables() {
     id="$(jq ".results[$entry].id" "$file")";
     average_daily_users="$(jq ".results[$entry].average_daily_users" "$file")";
     has_eula="$(jq ".results[$entry].has_eula" "$file")";
     slug="$(jq ".results[$entry].slug" "$file" | sed "s|^\"||; s|\"$||;")";
     current_version__id=$(jq ".results[$entry].current_version.id" "$file");
-    
-    # Names
-    name__enUS=$(jq ".results[$entry].name" "$file" | sed "s|-||;" | jq .enUS | sed "s|^\"||; s|\"$||;" )
-    name__enGB=$(jq ".results[$entry].name" "$file" | sed "s|-||;" | jq .enGB | sed "s|^\"||; s|\"$||;" )
-    if [ "$name__enUS" != "null" ]; then name="$name__enUS";
-    elif [ "$name__enGB" != "null" ]; then name="$name__enGB";
-    else name=$(jq ".results[$entry].name[]" "$file" | sed "s|-||;" | jq . | sed "s|^\"||; s|\"$||;"); # Use the first avalible name if English is not avaliable
-    fi
 
-    
+    source "$run_path/src/main_functions"
+    locale_description="name"
+    locale_description
+
     if jq .results[$entry].current_version.files[0].permissions "$file" | grep -q "nativeMessaging"; then
 
         nativeMessaging__pass=false;
@@ -102,60 +169,77 @@ function essential_addon {
 
 #function hard_to_evaluate_addon {
 
-        # curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging
-    
+# curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging
+
+
+# API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668
+# The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies.
+
+# Old solution:
 
-    # API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668
-    # The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies.
-
-    # Old solution:
-
-    # 1: Add this to FreeAMO-dependencies:
-    #belgium-eid	eid-mw
-
-    # 2: Uncomment this code:
-    #    depends_on_external_program=false;
-    #    while IFS= read -r dependencies; do
-    #
-    #        dependency_slug=$(echo $dependencies | awk '{print $1}');
-    #        dependency=$(echo "$dependencies" | cut -f 2);
-    #
-    #        if [ "$dependency_slug" == "$slug" ]; then
-    #
-    #            depends_on_external_program=true;
-    #
-    #        fi
-    #
-    #    done < $run_path/FreeAMO-dependencies
+# 1: Add this to FreeAMO-dependencies:
+#belgium-eid	eid-mw
+
+# 2: Uncomment this code:
+#    depends_on_external_program=false;
+#    while IFS= read -r dependencies; do
+#
+#        dependency_slug=$(echo $dependencies | awk '{print $1}');
+#        dependency=$(echo "$dependencies" | cut -f 2);
+#
+#        if [ "$dependency_slug" == "$slug" ]; then
+#
+#            depends_on_external_program=true;
+#
+#        fi
+#
+#    done < $run_path/FreeAMO-dependencies
 
 #}
 
-function progress {
-    
-    (>&2 echo -en "\r"; pv --progress --line-mode --size $pv__size --eta --timer) > /dev/null
-    
+function pv__progress {
+
+    if [ ! "$pv__discard" = true ]; then
+
+       (pv --progress --line-mode --size "$pv__size" --timer) > /dev/null
+
+    fi
+       
 }
 
 
-function html_to_mediawiki {
+function format_description__mixed_html_to_mediawiki {
 
     sed "
 # Remove Mozilla tracking links
-s|https://outgoing.prod.mozaws.net/v1/.\{65\}||g; s|%3A|:|g; s|%23|#|g
+s|https://outgoing.prod.mozaws.net/v1/.\\{65\\}||g; s|%3A|:|g; s|%23|#|g
 
-# Fix general links
-s|<a rel=\\\\\\\"nofollow\\\\\\\" href=\\\\\\\"|[|g
-s|\\\\\\\">| |g
-s|</a>|]|g
+# Remove reduntant backslashes. Pandoc will convert quotes regardless.
+s|\\\\\\\"|\\\"|g
 
-# Fix line breaks
-s|\\\\n|\\n|g
 
-# Fix quotes
-s|\\\\\\\"|\\\"|g
+"
+
+}
+
+function format_description__after_pandoc {
+
+    sed "
+
+### Fix formatting that cannot be fixed by Pandoc since it's not HTML
+
+# Fix line breaks. 
+s|\\\\n|\\n|g
 
-# Fix bullets, many developers failed to use <li> needed to make line break
-s|\\n- |\\n* |g
+# Remove \t
+s|\\\\t||g
 "
+    # Change local links to external links
+    
+}
+
+function format_description {
+
+    format_description__mixed_html_to_mediawiki | pandoc -f html -t mediawiki | format_description__after_pandoc
 
 }
diff --git a/subprojects/freeamo/src/make-wiki b/subprojects/freeamo/src/make-wiki
index 19e782f..118f257 100644
--- a/subprojects/freeamo/src/make-wiki
+++ b/subprojects/freeamo/src/make-wiki
@@ -15,20 +15,23 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-cd "build/json/search-pages"
+cd "build/json/search-pages" || exit
 
-echo -e "\\nGenerating .wiki files for https://directory.fsf.org/wiki/:"
+if [ "$minimum_average_daily_users" == "100000" ]; then
 
-echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\"
+    echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\"
 ! WebExtension
 ! Description
 ! Claimed license
 ! Users
 ! Rating
-! Updated" > "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki"
+! Updated
+! Featured" > "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki"
 
+fi
 
-pv__size=$(ls . | wc -l)
+
+pv__size=$(find . -not -name . | wc -l)
 output_file="$output_file_2";
 source "$run_path/src/main_functions"
 
@@ -38,8 +41,8 @@ while IFS= read -r table; do
     
     if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then
         
-        file=$(echo $table | awk '{print $1}');
-        entry=$(echo $table | awk '{print $2}');
+        file=$(echo "$table" | awk '{print $1}');
+        entry=$(echo "$table" | awk '{print $2}');
 
         source "$run_path/src/main_functions"
         amo_variables
@@ -55,27 +58,35 @@ while IFS= read -r table; do
         elif [ "$license__amo_full_name" == "Mozilla Public License Version 1.1" ]; then license__fsd_short_name="MPLv1.1";
         fi
 
+        # Start description code
+
         source "$run_path/src/main_functions"
-        full_description=$(jq .results[$entry].description $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | html_to_mediawiki)
-        short_description=$(jq .results[$entry].summary $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | html_to_mediawiki)
+        locale_description="summary"
+        locale_description
+        locale_description="description"
+        locale_description
+        short_description=$(echo "$summary" | format_description)
+        full_description=$(echo "$description" | format_description)
+        
+        # End description code
 
-        homepage_url=$(jq .results[$entry].homepage $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
-        version=$(jq .results[$entry].current_version.version $file | sed "s|^\"||; s|\"$||;");
-        version_date=$(jq .results[$entry].last_updated $file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g");
-        version_id=$(jq .results[$entry].current_version.id $file);
-        license_copyright=$(jq .results[$entry].authors[].name $file | sed "s|^\"||; s|\"$||;");
-        bayesian_average=$(jq .results[$entry].ratings.bayesian_average $file);
-        bayesian_average__simple=$(printf "%.1f" "$(echo "$bayesian_average" | sed "s|\.|,|")");
+        # The homepage field is mandatory so let's remove "null" from it because we don't need to evaluat it
+        homepage_url=$(jq .results[$entry].homepage "$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|null||");
+        license_copyright=$(jq .results[$entry].authors[].name "$file" | sed "s|^\"||; s|\"$||;");
+        bayesian_average=$(jq .results[$entry].ratings.bayesian_average "$file");
+        bayesian_average__simple=$(printf "%.1f" "${bayesian_average//./,}");
 
+        is_featured=$(jq .results[$entry].is_featured "$file");
+        
         last_review_date=$(date +"%Y/%m/%d");
-        last_review_by="wikisysbot";
-        submitted_by="wikisysbot";
+        last_review_by="$mediawikibot__bot_password__Username";
+        submitted_by="$mediawikibot__bot_password__Username";
 
-        if [[ "$(jq .results[$entry].current_version.compatibility.seamonkey $file)" != "null" ]]; then
+        if [[ "$(jq .results[$entry].current_version.compatibility.seamonkey "$file")" != "null" ]]; then
             set_seamonkey="Iceape,";
         fi
 
-        if [[ "$(jq .results[$entry].current_version.compatibility.firefox $file)" != "null" ]]; then
+        if [[ "$(jq .results[$entry].current_version.compatibility.firefox "$file")" != "null" ]]; then
             set_icecat="IceCat,";
         fi
 
@@ -83,7 +94,7 @@ while IFS= read -r table; do
         #		set_icecatmobile="IceCatMobile,";
         #	    fi
 
-        if [[ "$(jq .results[$entry].current_version.compatibility.thunderbird $file)" != "null" ]]; then
+        if [[ "$(jq .results[$entry].current_version.compatibility.thunderbird "$file")" != "null" ]]; then
             set_thunderbird="Icedove,";
         fi
 
@@ -97,7 +108,7 @@ while IFS= read -r table; do
         if [[ $github_true != "" ]]; then
 
             # Always use https, and remove anchors
-            homepage_url__for__vcs_checkout_command="https://github.com$(echo $github_true | sed "s|https://github.com||; s|http://github.com||; s|#|\n|;" | head -n 1)";
+            homepage_url__for__vcs_checkout_command="https://github.com$(echo "$github_true" | sed "s|https://github.com||; s|http://github.com||; s|#|\\n|;" | head -n 1)";
             # Remove trailing slash
             homepage_url__for__vcs_checkout_command="${homepage_url__for__vcs_checkout_command%/}"
             vcs_checkout_command="git clone $homepage_url__for__vcs_checkout_command.git";
@@ -114,11 +125,12 @@ while IFS= read -r table; do
 
         fi
 
-        support_url=$(jq .results[$entry].support_url $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+        support_url=$(jq .results[$entry].support_url "$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+        last_updated=$(jq .results[$entry].last_updated $file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g");
 
         # List JSON structure: cat json/$file | js '.'
         # XML value "homepage" is not always set in AMO API, but the XML require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically.
-        echo "{{Entry" > $run_path/build/wiki/$slug.wiki
+        echo "{{Entry" > "$run_path/build/wiki/$slug.wiki"
 
         # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341
         # Avoid specific versions since it's not compatible with all IceCat versions:
@@ -126,20 +138,30 @@ while IFS= read -r table; do
         # |Version identifier=$version
         # |Version date=$version_date
 
-
+        
+        # Do add these fields since we refer to the download page in the `|Version download` field that lists compatible versions
+        # |Version identifier=52.6.0
+        # |Version date=2018/02/26
+        # |Version status=stable
+        
         echo "|Name=$name
 |Short description=$short_description
 |Full description=$full_description
 |Homepage URL=$homepage_url
 |Extension of=$extension_of
 |VCS checkout command=$vcs_checkout_command
+|Documentation note=Extra data (https://addons.mozilla.org/api/v$api_version/addons/addon/$slug/):
+* Users: $average_daily_users
+* Rating: $bayesian_average__simple
+* Updated: $last_updated
+* Featured: $is_featured
 |Version download=https://addons.mozilla.org/en-US/firefox/addon/$slug/versions/
 |Last review by=$last_review_by
 |Last review date=$last_review_date
 |Submitted by=$submitted_by
-|Submitted date=2018/06/20
+|Submitted date=
 |Is GNU=$is_gnu
-}}" >> $run_path/build/wiki/$slug.wiki
+}}" >> "$run_path/build/wiki/$slug.wiki"
 
 
 
@@ -149,9 +171,9 @@ while IFS= read -r table; do
 |License copyright=$license_copyright
 |License note=\"License: $license__amo_full_name\" listed
 at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/
-}}" >> $run_path/build/wiki/$slug.wiki
+}}" >> "$run_path/build/wiki/$slug.wiki"
 
-        if [[ "$support_url" != "" ]];
+        if [ "$support_url" != "null" ];
         then
 
             echo "{{Resource
@@ -162,19 +184,30 @@ at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/
 
         fi
 
-
+        if [ "$minimum_average_daily_users" == "100000" ]; then
+            
         echo "|-
-| [[$name]]
+| [[$slug|$name]]
 | $short_description
 | [[License:$license__fsd_short_name|$license__fsd_short_name]]
 | $average_daily_users
 | $bayesian_average__simple
-| $version_date" >> "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki"
+| $last_updated
+| $is_featured" >> "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki"
+
+        fi
+
 
+        
     fi
 
-        echo "foo" # Must be here to work with pv
-done < "$output_file_2" | progress
+    echo "foo" # Must be here to work with pv
+done < "$output_file_2" | pv__progress
 # End template
 
-echo "|}" >> "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki"
+
+if [ "$minimum_average_daily_users" == "100000" ]; then
+
+    echo "|}" >> "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki"
+
+fi
diff --git a/subprojects/freeamo/src/search-pages b/subprojects/freeamo/src/search-pages
index a723601..3e894dc 100644
--- a/subprojects/freeamo/src/search-pages
+++ b/subprojects/freeamo/src/search-pages
@@ -17,22 +17,11 @@
 
 cd "$run_path/build/json/search-pages" || exit
 
-base_uri="https://addons.mozilla.org/api/v3/addons/search/?platform=$platform&sort=$sort&type=$type&app=$app&appversion=$appversion&page_size=$page_size&page=";
+base_uri="https://addons.mozilla.org/api/v$api_version/addons/search/?platform=$platform&sort=$sort&type=$type&app=$app&appversion=$appversion&page_size=$page_size&page=";
 
+echo "$base_uri"
 function search-pages--download() {
     
-    echo -e "\\nDownloading search pages from $base_uri :" | sed "s|page=|page=\$page|"
-
-    echo "This file is used to describe the settings that were used to generate the files.
-
-build/json/search-pages/
-$wget_uri" > "$run_path/build/declared_settings.txt"
-    # indicate that the page page variable are the page files
-    sed -i "s|page=1|page=\$page|g" "$run_path/build/declared_settings.txt"
-
-
-
-    #####################################################
 
     function wget_page {
         
@@ -57,12 +46,16 @@ $wget_uri" > "$run_path/build/declared_settings.txt"
 
         pv__size="2";
 
+    elif [ "$minimum_average_daily_users" -gt "0" ]; then
+        
+        pv__discard=true;
+        echo "Cannot show a progress counter since minimum_average_daily_users is set to more than 0 in freeamo.conf."
+        
     else
 
         pv__size="$page_count"
 
     fi
-
     
     while [ "$page" -lt "$page_count" ]; do
 
@@ -80,28 +73,17 @@ $wget_uri" > "$run_path/build/declared_settings.txt"
         fi
         
         echo "foo" # Must be here to work with pv
-    done | progress
+    done | pv__progress
     
 }
 
 function search-pages--filter() {
 
- 
-    echo -e "\\nFilter search pages -- no EULA, no native messaging, and minimum average daily users ($minimum_average_daily_users):"
-
-    #    source "$run_path/src/main_functions"
-    #    amo_variables
-    echo "
-build/index-search-pages.txt
-has_eula=false
-nativeMessaging__pass=true
-minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/declared_settings.txt"
-
-    pv__size=$(ls . | wc -l)
+    pv__size=$(find . -not -name . | wc -l)
     output_file="$output_file_1";
     source "$run_path/src/main_functions"
     
-    rm -f $output_file_1
+    rm -f "$output_file_1"
     
     # Sort by file modification time stamp for the downloaded JSON files
     find . -type f  | sort -n | while read -r file; do
@@ -111,6 +93,7 @@ minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/de
 
         entry="0"; # Should not be 1
 
+        # Should not be limited to 3 in --debug mode.
         for id in $(jq ".results[].average_daily_users" "$file"); do
 
             source "$run_path/src/main_functions"
@@ -119,7 +102,7 @@ minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/de
             essential_addon
             if [ "$essential_addon" = true ]; then
                 
-                echo -e "$filename\\t$entry" >> "$output_file_1"
+                echo -e "$filename\\t$entry\\t$slug" >> "$output_file_1"
                 
             fi
             
@@ -131,6 +114,6 @@ minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/de
 
 
         echo "foo" # Must be here to work with pv
-    done | progress
+    done | pv__progress
 
 }