aboutsummaryrefslogtreecommitdiff
path: root/subprojects/freeamo/src
diff options
context:
space:
mode:
Diffstat (limited to 'subprojects/freeamo/src')
-rw-r--r--subprojects/freeamo/src/get-licenses21
-rw-r--r--subprojects/freeamo/src/main_functions184
-rw-r--r--subprojects/freeamo/src/make-wiki103
-rw-r--r--subprojects/freeamo/src/search-pages43
4 files changed, 224 insertions, 127 deletions
diff --git a/subprojects/freeamo/src/get-licenses b/subprojects/freeamo/src/get-licenses
index c7e5559..79b6da4 100644
--- a/subprojects/freeamo/src/get-licenses
+++ b/subprojects/freeamo/src/get-licenses
@@ -18,12 +18,11 @@
# API: Please adopt SPDX - https://github.com/mozilla/addons-server/issues/8706
# Allow GNU [L]GPL x.x or later - https://github.com/mozilla/addons-server/issues/8707
-cd "build/json/search-pages"
+cd "build/json/search-pages" || exit
function get-licenses--download {
- echo -e "\\nDownloading license pages:"
-
+ # API: Add "last_license" to the search pages - https://github.com/mozilla/addons-server/issues/8945
output_file="$output_file_1";
pv__size="$(wc -l < "$output_file")"
@@ -35,8 +34,8 @@ function get-licenses--download {
if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then
- file=$(echo $table | awk '{print $1}');
- entry=$(echo $table | awk '{print $2}');
+ file=$(echo "$table" | awk '{print $1}');
+ entry=$(echo "$table" | awk '{print $2}');
source "$run_path/src/main_functions"
amo_variables
@@ -52,14 +51,12 @@ function get-licenses--download {
echo "foo" # Must be here to work with pv
- done < "$output_file_1" | progress
+ done < "$output_file_1" | pv__progress
}
function get-licenses--filter {
- echo -e "\\nFilter license pages:"
-
output_file="$output_file_1";
pv__size="$(wc -l < "$output_file")"
source "$run_path/src/main_functions"
@@ -72,8 +69,8 @@ function get-licenses--filter {
if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then
- file=$(echo $table | awk '{print $1}');
- entry=$(echo $table | awk '{print $2}');
+ file=$(echo "$table" | awk '{print $1}');
+ entry=$(echo "$table" | awk '{print $2}');
source "$run_path/src/main_functions"
amo_variables
@@ -101,7 +98,7 @@ function get-licenses--filter {
# The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files.
# echo "$name: $license__amo_full_name"
- echo "$file $entry" >> "$output_file_2"
+ echo "$file $entry $slug" >> "$output_file_2"
freedom_status="free";
@@ -120,6 +117,6 @@ function get-licenses--filter {
echo "foo" # Must be here to work with pv
- done < "$output_file_1" | progress
+ done < "$output_file_1" | pv__progress
}
diff --git a/subprojects/freeamo/src/main_functions b/subprojects/freeamo/src/main_functions
index a96ada3..7a66cff 100644
--- a/subprojects/freeamo/src/main_functions
+++ b/subprojects/freeamo/src/main_functions
@@ -15,6 +15,13 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+function latest_debranded_trisquel_version {
+
+ # All debranded programs starts with "i" so all of them are in Trisquels "/i/" directory
+ wget -qO- "http://archive.trisquel.info/trisquel/pool/main/i/$debranded_package/?C=N;O=D" | sed "s|\"|\\n|g; s|-|\\n|g" | grep "$debranded_package"_ | head -n 1 | sed "s|"$debranded_package"_||"
+
+}
+
function check_files {
if [ ! -f "$run_path/FreeAMO-$check_type" ]; then
@@ -26,7 +33,7 @@ function check_files {
# Make sure all links are using SSL
- if grep -q "http://addons.mozilla.org/" $run_path/FreeAMO-$check_type; then
+ if grep -q "http://addons.mozilla.org/" "$run_path/FreeAMO-$check_type"; then
echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)."
echo "Exiting" && exit
@@ -52,22 +59,82 @@ function line_status {
}
+# Begin locale code
+
+function locale_description {
+
+ tmp__enUS=$(jq ".results[$entry].$locale_description" "$file" | sed "s|-||;" | jq .enUS | sed "s|^\"||; s|\"$||;" )
+ tmp__enGB=$(jq ".results[$entry].$locale_description" "$file" | sed "s|-||;" | jq .enGB | sed "s|^\"||; s|\"$||;" )
+ if [ "$tmp__enUS" != "null" ]; then
+
+ if [ "$locale_description" == "name" ]; then
+
+ name="$tmp__enUS";
+
+ elif [ "$locale_description" == "summary" ]; then
+
+ summary="$tmp__enUS";
+
+ elif [ "$locale_description" == "description" ]; then
+
+ description="$tmp__enUS";
+
+ fi
+
+
+ elif [ "$tmp__enGB" != "null" ]; then
+
+ if [ "$locale_description" == "name" ]; then
+
+ name="$tmp__enGB";
+
+ elif [ "$locale_description" == "summary" ]; then
+
+ summary="$tmp__enGB";
+
+ elif [ "$locale_description" == "description" ]; then
+
+ description="$tmp__enGB";
+
+ fi
+
+ else
+
+ tmp=$(jq ".results[$entry].$locale_description[]" "$file" | sed "s|-||;" | jq . | sed "s|^\"||; s|\"$||;"); # Use the first avalible summary if English is not avaliable
+
+ if [ "$locale_description" == "name" ]; then
+
+ name="$tmp";
+
+ elif [ "$locale_description" == "summary" ]; then
+
+ summary="$tmp";
+
+ elif [ "$locale_description" == "description" ]; then
+
+ description="$tmp";
+
+ fi
+
+
+ fi
+
+}
+
+# End locale code
+
+
function amo_variables() {
id="$(jq ".results[$entry].id" "$file")";
average_daily_users="$(jq ".results[$entry].average_daily_users" "$file")";
has_eula="$(jq ".results[$entry].has_eula" "$file")";
slug="$(jq ".results[$entry].slug" "$file" | sed "s|^\"||; s|\"$||;")";
current_version__id=$(jq ".results[$entry].current_version.id" "$file");
-
- # Names
- name__enUS=$(jq ".results[$entry].name" "$file" | sed "s|-||;" | jq .enUS | sed "s|^\"||; s|\"$||;" )
- name__enGB=$(jq ".results[$entry].name" "$file" | sed "s|-||;" | jq .enGB | sed "s|^\"||; s|\"$||;" )
- if [ "$name__enUS" != "null" ]; then name="$name__enUS";
- elif [ "$name__enGB" != "null" ]; then name="$name__enGB";
- else name=$(jq ".results[$entry].name[]" "$file" | sed "s|-||;" | jq . | sed "s|^\"||; s|\"$||;"); # Use the first avalible name if English is not avaliable
- fi
-
+ source "$run_path/src/main_functions"
+ locale_description="name"
+ locale_description
+
if jq .results[$entry].current_version.files[0].permissions "$file" | grep -q "nativeMessaging"; then
nativeMessaging__pass=false;
@@ -102,60 +169,77 @@ function essential_addon {
#function hard_to_evaluate_addon {
- # curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging
-
+# curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging
+
+
+# API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668
+# The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies.
+
+# Old solution:
- # API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668
- # The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies.
-
- # Old solution:
-
- # 1: Add this to FreeAMO-dependencies:
- #belgium-eid eid-mw
-
- # 2: Uncomment this code:
- # depends_on_external_program=false;
- # while IFS= read -r dependencies; do
- #
- # dependency_slug=$(echo $dependencies | awk '{print $1}');
- # dependency=$(echo "$dependencies" | cut -f 2);
- #
- # if [ "$dependency_slug" == "$slug" ]; then
- #
- # depends_on_external_program=true;
- #
- # fi
- #
- # done < $run_path/FreeAMO-dependencies
+# 1: Add this to FreeAMO-dependencies:
+#belgium-eid eid-mw
+
+# 2: Uncomment this code:
+# depends_on_external_program=false;
+# while IFS= read -r dependencies; do
+#
+# dependency_slug=$(echo $dependencies | awk '{print $1}');
+# dependency=$(echo "$dependencies" | cut -f 2);
+#
+# if [ "$dependency_slug" == "$slug" ]; then
+#
+# depends_on_external_program=true;
+#
+# fi
+#
+# done < $run_path/FreeAMO-dependencies
#}
-function progress {
-
- (>&2 echo -en "\r"; pv --progress --line-mode --size $pv__size --eta --timer) > /dev/null
-
+function pv__progress {
+
+ if [ ! "$pv__discard" = true ]; then
+
+ (pv --progress --line-mode --size "$pv__size" --timer) > /dev/null
+
+ fi
+
}
-function html_to_mediawiki {
+function format_description__mixed_html_to_mediawiki {
sed "
# Remove Mozilla tracking links
-s|https://outgoing.prod.mozaws.net/v1/.\{65\}||g; s|%3A|:|g; s|%23|#|g
+s|https://outgoing.prod.mozaws.net/v1/.\\{65\\}||g; s|%3A|:|g; s|%23|#|g
-# Fix general links
-s|<a rel=\\\\\\\"nofollow\\\\\\\" href=\\\\\\\"|[|g
-s|\\\\\\\">| |g
-s|</a>|]|g
+# Remove reduntant backslashes. Pandoc will convert quotes regardless.
+s|\\\\\\\"|\\\"|g
-# Fix line breaks
-s|\\\\n|\\n|g
-# Fix quotes
-s|\\\\\\\"|\\\"|g
+"
+
+}
+
+function format_description__after_pandoc {
+
+ sed "
+
+### Fix formatting that cannot be fixed by Pandoc since it's not HTML
+
+# Fix line breaks.
+s|\\\\n|\\n|g
-# Fix bullets, many developers failed to use <li> needed to make line break
-s|\\n- |\\n* |g
+# Remove \t
+s|\\\\t||g
"
+ # Change local links to external links
+
+}
+
+function format_description {
+
+ format_description__mixed_html_to_mediawiki | pandoc -f html -t mediawiki | format_description__after_pandoc
}
diff --git a/subprojects/freeamo/src/make-wiki b/subprojects/freeamo/src/make-wiki
index 19e782f..118f257 100644
--- a/subprojects/freeamo/src/make-wiki
+++ b/subprojects/freeamo/src/make-wiki
@@ -15,20 +15,23 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-cd "build/json/search-pages"
+cd "build/json/search-pages" || exit
-echo -e "\\nGenerating .wiki files for https://directory.fsf.org/wiki/:"
+if [ "$minimum_average_daily_users" == "100000" ]; then
-echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\"
+ echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\"
! WebExtension
! Description
! Claimed license
! Users
! Rating
-! Updated" > "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki"
+! Updated
+! Featured" > "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki"
+fi
-pv__size=$(ls . | wc -l)
+
+pv__size=$(find . -not -name . | wc -l)
output_file="$output_file_2";
source "$run_path/src/main_functions"
@@ -38,8 +41,8 @@ while IFS= read -r table; do
if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then
- file=$(echo $table | awk '{print $1}');
- entry=$(echo $table | awk '{print $2}');
+ file=$(echo "$table" | awk '{print $1}');
+ entry=$(echo "$table" | awk '{print $2}');
source "$run_path/src/main_functions"
amo_variables
@@ -55,27 +58,35 @@ while IFS= read -r table; do
elif [ "$license__amo_full_name" == "Mozilla Public License Version 1.1" ]; then license__fsd_short_name="MPLv1.1";
fi
+ # Start description code
+
source "$run_path/src/main_functions"
- full_description=$(jq .results[$entry].description $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | html_to_mediawiki)
- short_description=$(jq .results[$entry].summary $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | html_to_mediawiki)
+ locale_description="summary"
+ locale_description
+ locale_description="description"
+ locale_description
+ short_description=$(echo "$summary" | format_description)
+ full_description=$(echo "$description" | format_description)
+
+ # End description code
- homepage_url=$(jq .results[$entry].homepage $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
- version=$(jq .results[$entry].current_version.version $file | sed "s|^\"||; s|\"$||;");
- version_date=$(jq .results[$entry].last_updated $file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g");
- version_id=$(jq .results[$entry].current_version.id $file);
- license_copyright=$(jq .results[$entry].authors[].name $file | sed "s|^\"||; s|\"$||;");
- bayesian_average=$(jq .results[$entry].ratings.bayesian_average $file);
- bayesian_average__simple=$(printf "%.1f" "$(echo "$bayesian_average" | sed "s|\.|,|")");
+ # The homepage field is mandatory so let's remove "null" from it because we don't need to evaluat it
+ homepage_url=$(jq .results[$entry].homepage "$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|null||");
+ license_copyright=$(jq .results[$entry].authors[].name "$file" | sed "s|^\"||; s|\"$||;");
+ bayesian_average=$(jq .results[$entry].ratings.bayesian_average "$file");
+ bayesian_average__simple=$(printf "%.1f" "${bayesian_average//./,}");
+ is_featured=$(jq .results[$entry].is_featured "$file");
+
last_review_date=$(date +"%Y/%m/%d");
- last_review_by="wikisysbot";
- submitted_by="wikisysbot";
+ last_review_by="$mediawikibot__bot_password__Username";
+ submitted_by="$mediawikibot__bot_password__Username";
- if [[ "$(jq .results[$entry].current_version.compatibility.seamonkey $file)" != "null" ]]; then
+ if [[ "$(jq .results[$entry].current_version.compatibility.seamonkey "$file")" != "null" ]]; then
set_seamonkey="Iceape,";
fi
- if [[ "$(jq .results[$entry].current_version.compatibility.firefox $file)" != "null" ]]; then
+ if [[ "$(jq .results[$entry].current_version.compatibility.firefox "$file")" != "null" ]]; then
set_icecat="IceCat,";
fi
@@ -83,7 +94,7 @@ while IFS= read -r table; do
# set_icecatmobile="IceCatMobile,";
# fi
- if [[ "$(jq .results[$entry].current_version.compatibility.thunderbird $file)" != "null" ]]; then
+ if [[ "$(jq .results[$entry].current_version.compatibility.thunderbird "$file")" != "null" ]]; then
set_thunderbird="Icedove,";
fi
@@ -97,7 +108,7 @@ while IFS= read -r table; do
if [[ $github_true != "" ]]; then
# Always use https, and remove anchors
- homepage_url__for__vcs_checkout_command="https://github.com$(echo $github_true | sed "s|https://github.com||; s|http://github.com||; s|#|\n|;" | head -n 1)";
+ homepage_url__for__vcs_checkout_command="https://github.com$(echo "$github_true" | sed "s|https://github.com||; s|http://github.com||; s|#|\\n|;" | head -n 1)";
# Remove trailing slash
homepage_url__for__vcs_checkout_command="${homepage_url__for__vcs_checkout_command%/}"
vcs_checkout_command="git clone $homepage_url__for__vcs_checkout_command.git";
@@ -114,11 +125,12 @@ while IFS= read -r table; do
fi
- support_url=$(jq .results[$entry].support_url $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+ support_url=$(jq .results[$entry].support_url "$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;");
+ last_updated=$(jq .results[$entry].last_updated $file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g");
# List JSON structure: cat json/$file | js '.'
# XML value "homepage" is not always set in AMO API, but the XML require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically.
- echo "{{Entry" > $run_path/build/wiki/$slug.wiki
+ echo "{{Entry" > "$run_path/build/wiki/$slug.wiki"
# Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341
# Avoid specific versions since it's not compatible with all IceCat versions:
@@ -126,20 +138,30 @@ while IFS= read -r table; do
# |Version identifier=$version
# |Version date=$version_date
-
+
+ # Do add these fields since we refer to the download page in the `|Version download` field that lists compatible versions
+ # |Version identifier=52.6.0
+ # |Version date=2018/02/26
+ # |Version status=stable
+
echo "|Name=$name
|Short description=$short_description
|Full description=$full_description
|Homepage URL=$homepage_url
|Extension of=$extension_of
|VCS checkout command=$vcs_checkout_command
+|Documentation note=Extra data (https://addons.mozilla.org/api/v$api_version/addons/addon/$slug/):
+* Users: $average_daily_users
+* Rating: $bayesian_average__simple
+* Updated: $last_updated
+* Featured: $is_featured
|Version download=https://addons.mozilla.org/en-US/firefox/addon/$slug/versions/
|Last review by=$last_review_by
|Last review date=$last_review_date
|Submitted by=$submitted_by
-|Submitted date=2018/06/20
+|Submitted date=
|Is GNU=$is_gnu
-}}" >> $run_path/build/wiki/$slug.wiki
+}}" >> "$run_path/build/wiki/$slug.wiki"
@@ -149,9 +171,9 @@ while IFS= read -r table; do
|License copyright=$license_copyright
|License note=\"License: $license__amo_full_name\" listed
at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/
-}}" >> $run_path/build/wiki/$slug.wiki
+}}" >> "$run_path/build/wiki/$slug.wiki"
- if [[ "$support_url" != "" ]];
+ if [ "$support_url" != "null" ];
then
echo "{{Resource
@@ -162,19 +184,30 @@ at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/
fi
-
+ if [ "$minimum_average_daily_users" == "100000" ]; then
+
echo "|-
-| [[$name]]
+| [[$slug|$name]]
| $short_description
| [[License:$license__fsd_short_name|$license__fsd_short_name]]
| $average_daily_users
| $bayesian_average__simple
-| $version_date" >> "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki"
+| $last_updated
+| $is_featured" >> "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki"
+
+ fi
+
+
fi
- echo "foo" # Must be here to work with pv
-done < "$output_file_2" | progress
+ echo "foo" # Must be here to work with pv
+done < "$output_file_2" | pv__progress
# End template
-echo "|}" >> "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki"
+
+if [ "$minimum_average_daily_users" == "100000" ]; then
+
+ echo "|}" >> "$run_path/build/IceCat WebExtensions - Most Users (proposed).wiki"
+
+fi
diff --git a/subprojects/freeamo/src/search-pages b/subprojects/freeamo/src/search-pages
index a723601..3e894dc 100644
--- a/subprojects/freeamo/src/search-pages
+++ b/subprojects/freeamo/src/search-pages
@@ -17,22 +17,11 @@
cd "$run_path/build/json/search-pages" || exit
-base_uri="https://addons.mozilla.org/api/v3/addons/search/?platform=$platform&sort=$sort&type=$type&app=$app&appversion=$appversion&page_size=$page_size&page=";
+base_uri="https://addons.mozilla.org/api/v$api_version/addons/search/?platform=$platform&sort=$sort&type=$type&app=$app&appversion=$appversion&page_size=$page_size&page=";
+echo "$base_uri"
function search-pages--download() {
- echo -e "\\nDownloading search pages from $base_uri :" | sed "s|page=|page=\$page|"
-
- echo "This file is used to describe the settings that were used to generate the files.
-
-build/json/search-pages/
-$wget_uri" > "$run_path/build/declared_settings.txt"
- # indicate that the page page variable are the page files
- sed -i "s|page=1|page=\$page|g" "$run_path/build/declared_settings.txt"
-
-
-
- #####################################################
function wget_page {
@@ -57,12 +46,16 @@ $wget_uri" > "$run_path/build/declared_settings.txt"
pv__size="2";
+ elif [ "$minimum_average_daily_users" -gt "0" ]; then
+
+ pv__discard=true;
+ echo "Cannot show a progress counter since minimum_average_daily_users is set to more than 0 in freeamo.conf."
+
else
pv__size="$page_count"
fi
-
while [ "$page" -lt "$page_count" ]; do
@@ -80,28 +73,17 @@ $wget_uri" > "$run_path/build/declared_settings.txt"
fi
echo "foo" # Must be here to work with pv
- done | progress
+ done | pv__progress
}
function search-pages--filter() {
-
- echo -e "\\nFilter search pages -- no EULA, no native messaging, and minimum average daily users ($minimum_average_daily_users):"
-
- # source "$run_path/src/main_functions"
- # amo_variables
- echo "
-build/index-search-pages.txt
-has_eula=false
-nativeMessaging__pass=true
-minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/declared_settings.txt"
-
- pv__size=$(ls . | wc -l)
+ pv__size=$(find . -not -name . | wc -l)
output_file="$output_file_1";
source "$run_path/src/main_functions"
- rm -f $output_file_1
+ rm -f "$output_file_1"
# Sort by file modification time stamp for the downloaded JSON files
find . -type f | sort -n | while read -r file; do
@@ -111,6 +93,7 @@ minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/de
entry="0"; # Should not be 1
+ # Should not be limited to 3 in --debug mode.
for id in $(jq ".results[].average_daily_users" "$file"); do
source "$run_path/src/main_functions"
@@ -119,7 +102,7 @@ minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/de
essential_addon
if [ "$essential_addon" = true ]; then
- echo -e "$filename\\t$entry" >> "$output_file_1"
+ echo -e "$filename\\t$entry\\t$slug" >> "$output_file_1"
fi
@@ -131,6 +114,6 @@ minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/de
echo "foo" # Must be here to work with pv
- done | progress
+ done | pv__progress
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-11-04 03:23:35 +0000