aboutsummaryrefslogtreecommitdiff
path: root/subprojects/freeamo/src/unmaintained/download-custom~
diff options
context:
space:
mode:
authordavid <public@beloved.name>2018-07-17 15:52:52 +0200
committerdavid <public@beloved.name>2018-07-17 15:52:52 +0200
commit1ae516a7bab1d5d9f02ab00c4ac13c219f43c1a1 (patch)
tree1733cb2fb3b2db132b4fc4fc433fc38a68062c12 /subprojects/freeamo/src/unmaintained/download-custom~
parent3aa358cf718f8fc15952757cd04489b26770ac74 (diff)
Renamed the AMO/FSD project to FreeAMO, and rewrote its architecture.
Diffstat (limited to 'subprojects/freeamo/src/unmaintained/download-custom~')
-rwxr-xr-xsubprojects/freeamo/src/unmaintained/download-custom~86
1 files changed, 86 insertions, 0 deletions
diff --git a/subprojects/freeamo/src/unmaintained/download-custom~ b/subprojects/freeamo/src/unmaintained/download-custom~
new file mode 100755
index 0000000..3724aba
--- /dev/null
+++ b/subprojects/freeamo/src/unmaintained/download-custom~
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+# This file is part of FreeAMO
+# Copyright (C) 2017 David Hedlund
+#
+# FreeAMO is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# FreeAMO is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+cd build || exit
+mkdir json
+
+cp -a "$run_path/FreeAMO-custom" custom_addons.txt
+
+# Sanitize a copy from empty lines, they use to occur at the end of the file
+sed -i '/^\s*$/d' FreeAMO-custom
+
+
+sed -i "s|https://addons.mozilla.org/en-US/firefox/addon/||g; s|/||" custom_addons.txt
+
+rm -fr \
+ custom_addons-json-source.txt \
+ custom_addons-average_daily_users.txt \
+ custom_addons-id.txt custom_addons-slug.txt \
+ custom_addons-merged.txt \
+ custom_addons-report-expired_links.txt \
+ custom_addons-merged-sorted.txt
+
+cd json || exit
+
+lines="1";
+
+while IFS= read -r slug; do
+
+
+ if [ "$debug" == false ] || [ "$debug" == true ] && [ "$lines" -le 3 ]; then
+
+ wget "https://addons.mozilla.org/api/v3/addons/addon/$slug/" -O "$slug.json";
+ [ -s "$slug.json" ] || rm -f "$slug.json" # Delete empty files saved by wget, tis is much faster than using wget options to first validate the links.
+
+ # Ignore expired URLs
+ if [ -f "$slug.json" ]; then
+
+ id="$(jq ".id" "$slug.json")";
+ mv "$slug.json" "$id.json"
+
+ if [ "$(jq ".average_daily_users" "$id.json")" -gt "$minimum_average_daily_users" ]; then
+
+ echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tAlready in the main repository\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$run_path/FreeAMO-log"
+
+ else
+
+ jq ".average_daily_users" "$id.json" >> ../custom_addons-average_daily_users.txt
+
+ # We need the JSON source because they are different for search indexed add-ons (they are store in index.html*)
+ jq ".id" "$id.json" >> ../custom_addons-id.txt
+ echo "$(jq ".id" "$id.json").json" >> ../custom_addons-json-source.txt
+
+ ((lines++))
+
+
+ fi
+
+
+ else
+
+ echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tLink rot\t\t\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$run_path/FreeAMO-log"
+
+ fi
+
+ fi
+
+done < ../custom_addons.txt
+
+# Sort by average_daily_users
+paste ../custom_addons-average_daily_users.txt ../custom_addons-id.txt ../custom_addons-json-source.txt > ../custom_addons-merged.txt
+sort -k1,1nr ../custom_addons-merged.txt > ../merged-custom.txt