aboutsummaryrefslogtreecommitdiff
path: root/addons.mozilla.org-fsd/sync-data/fsd-amo-bot
diff options
context:
space:
mode:
Diffstat (limited to 'addons.mozilla.org-fsd/sync-data/fsd-amo-bot')
-rwxr-xr-xaddons.mozilla.org-fsd/sync-data/fsd-amo-bot306
1 files changed, 306 insertions, 0 deletions
diff --git a/addons.mozilla.org-fsd/sync-data/fsd-amo-bot b/addons.mozilla.org-fsd/sync-data/fsd-amo-bot
new file mode 100755
index 0000000..14b008d
--- /dev/null
+++ b/addons.mozilla.org-fsd/sync-data/fsd-amo-bot
@@ -0,0 +1,306 @@
+#!/bin/bash
+
+# Copyright 2016, 2017 David Hedlund
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# https://addons-server.readthedocs.io/en/latest/topics/api/addons.html
+# Add to wiki and remove: echo "|Name=$(xmlstarlet sel -t -v "addon/name" test.xml)"
+
+SCRIPTSRC=$(readlink -f "$0" || echo "$0")
+RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .)
+
+if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi
+if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi
+
+if [[ $exit == true ]]; then exit 1; fi
+
+
+if [ ! -d "build" ]; then mkdir -p build; fi
+
+retrive () {
+ api="$(echo "$url" | sed "s|https://addons.mozilla.org/en-US/firefox/addon/|https://addons.mozilla.org/api/v3/addons/addon/|g;")";
+ slug="$(echo "$url" | sed "s|https://addons.mozilla.org/en-US/firefox/addon/||g; s|/||g")";
+}
+
+xpi () {
+ cd build/ || exit
+ echo "$download" > tmp.txt
+ sed -i "s|/addon-|\naddon-|" tmp.txt
+ xpi=$(grep ".xpi" tmp.txt);
+ fbname=$(basename "$xpi" | cut -d. -f1);
+}
+
+
+case "$1" in
+ --get-json)
+
+ if [ ! -d "build/amo/json" ]; then mkdir -p build/amo/json; fi
+ if [ ! -d "build/amo/json.versions" ]; then mkdir -p build/amo/json.versions; fi
+ cd build/amo/ || exit
+
+ while read -r url; do
+
+ retrive
+ wget "$api" -O json/$slug.json
+ wget $api\versions/$(cat json/$slug.json | jq '.current_version.id')/ -O json.versions/$slug.json
+
+ done < ../../amo-repo.txt
+
+ # List obsoleted projects
+ find json/ -size 0 -print
+
+ ;;
+ --get-xpi-url)
+
+ cd build/ || exit
+ rm -f amo_repo-latest_download.txt
+ for i in $(find amo/json/ -maxdepth 1 -type f); do echo "https://addons.mozilla.org/firefox/downloads/latest/$(basename $i | sed "s|.json||")/addon-$(cat $i | jq '.current_version.id').xpi" >> amo_repo-latest_download.txt; done
+
+ ;;
+
+ --get-xpi)
+
+ if [ ! -d "build/amo/xpi" ]; then mkdir -p build/amo/xpi; fi
+
+ while read download; do
+
+ xpi
+
+ if [ ! -f amo/xpi/$xpi ]; then
+
+ cd amo/xpi/ || exit
+ wget $download
+ cd ..
+
+ fi
+
+ cd ..
+
+
+ done < build/amo_repo-latest_download.txt
+
+ ;;
+
+ --xpi-extract)
+
+ if [ ! -d "build/amo/xpi.extracted" ]; then mkdir -p build/amo/xpi.extracted; fi
+
+ while read download; do
+
+ xpi
+
+ if [ ! -d amo/xpi.extracted/$fbname ]; then
+
+ mkdir -p amo/xpi.extracted/$fbname
+ cd amo/xpi.extracted/$fbname || exit
+ unzip ../../xpi/$xpi
+ cd ../../
+
+ fi
+
+ cd ..
+
+ done < build/amo_repo-latest_download.txt
+
+ ;;
+
+ --make-xml)
+
+ if [ ! -d "build/fsd" ]; then mkdir -p build/fsd; fi
+
+ # Get number of edits:
+ wget "https://directory.fsf.org/w/api.php?action=query&list=recentchanges&rclimit=1" -O fsdedits.xml
+ oldrevid="$(grep revid fsdedits.xml | sed "s|&quot;||g; s|revid|\nrevid|; s|old_revid|\n|" | grep revid | sed "s|revid=||")"
+ newrevid=$(expr 1 + $oldrevid)
+
+ cd build/ || exit
+
+ while read url; do
+
+ retrive
+ # List JSON structure: cat amo/json/$slug.json | js '.'
+ # XML value "homepage" is not always set in AMO API, but the FSD require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically.
+ echo "<mediawiki xmlns=\"http://www.mediawiki.org/xml/export-0.7/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.mediawiki.org/xml/export-0.7/ http://www.mediawiki.org/xml/export-0.7.xsd\" version=\"0.7\" xml:lang=\"en\">
+ <siteinfo>
+ <sitename>Free Software Directory</sitename>
+ <base>https://directory.fsf.org/wiki/Main_Page</base>
+ <generator>MediaWiki 1.20.2</generator>
+ <case>first-letter</case>
+ <namespaces>
+ <namespace key=\"-2\" case=\"first-letter\">Media</namespace>
+ <namespace key=\"-1\" case=\"first-letter\">Special</namespace>
+ <namespace key=\"0\" case=\"first-letter\" />
+ <namespace key=\"1\" case=\"first-letter\">Talk</namespace>
+ <namespace key=\"2\" case=\"first-letter\">User</namespace>
+ <namespace key=\"3\" case=\"first-letter\">User talk</namespace>
+ <namespace key=\"4\" case=\"first-letter\">Free Software Directory</namespace>
+ <namespace key=\"5\" case=\"first-letter\">Free Software Directory talk</namespace>
+ <namespace key=\"6\" case=\"first-letter\">File</namespace>
+ <namespace key=\"7\" case=\"first-letter\">File talk</namespace>
+ <namespace key=\"8\" case=\"first-letter\">MediaWiki</namespace>
+ <namespace key=\"9\" case=\"first-letter\">MediaWiki talk</namespace>
+ <namespace key=\"10\" case=\"first-letter\">Template</namespace>
+ <namespace key=\"11\" case=\"first-letter\">Template talk</namespace>
+ <namespace key=\"12\" case=\"first-letter\">Help</namespace>
+ <namespace key=\"13\" case=\"first-letter\">Help talk</namespace>
+ <namespace key=\"14\" case=\"first-letter\">Category</namespace>
+ <namespace key=\"15\" case=\"first-letter\">Category talk</namespace>
+ <namespace key=\"102\" case=\"first-letter\">Property</namespace>
+ <namespace key=\"103\" case=\"first-letter\">Property talk</namespace>
+ <namespace key=\"106\" case=\"first-letter\">Form</namespace>
+ <namespace key=\"107\" case=\"first-letter\">Form talk</namespace>
+ <namespace key=\"108\" case=\"first-letter\">Concept</namespace>
+ <namespace key=\"109\" case=\"first-letter\">Concept talk</namespace>
+ <namespace key=\"170\" case=\"first-letter\">Filter</namespace>
+ <namespace key=\"171\" case=\"first-letter\">Filter talk</namespace>
+ <namespace key=\"500\" case=\"first-letter\">Review</namespace>
+ <namespace key=\"501\" case=\"first-letter\">Review talk</namespace>
+ <namespace key=\"502\" case=\"first-letter\">License</namespace>
+ <namespace key=\"503\" case=\"first-letter\">License talk</namespace>
+ <namespace key=\"510\" case=\"first-letter\">Collection</namespace>
+ <namespace key=\"511\" case=\"first-letter\">Collection talk</namespace>
+ <namespace key=\"520\" case=\"first-letter\">Org</namespace>
+ <namespace key=\"521\" case=\"first-letter\">Org talk</namespace>
+ </namespaces>
+ </siteinfo>
+ <page>
+ <title>IceCat/$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.name.enUS' | sed "s|^\"||; s|\"$||;")</title>
+ <revision>
+ <id>$newrevid</id>
+ <timestamp>$(date -u +%Y-%m-%dT%H:%M:%SZ)</timestamp>
+ <contributor>
+ <username>amo-import</username>
+ <id>1501</id>
+ </contributor>
+ <sha1>kj1p9nd2p15bqr2tb7jkihshlxow11y</sha1>
+ <text xml:space=\"preserve\" bytes=\"2687\">{{Entry" > fsd/$slug.xml
+
+ # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341
+ echo "|Name=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.name.enUS' | sed "s|^\"||; s|\"$||;")
+|Short description=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.summary.enUS' | sed "s|^\"||; s|\"$||; s|\\\n|\n|g; s|\\\\\"|\"|g;")
+|Full description=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.description.enUS' | sed "s|^\"||; s|\"$||; s|\\\n|\n|g; s|\\\\\"|\"|g;")
+|Homepage URL=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | sed "s|^\"||; s|\"$||;")" >> fsd/$slug.xml
+
+
+ if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.seamonkey')" != "null" ]]; then
+ set_seamonkey="SeaMonkey,";
+ fi
+
+ if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.firefox')" != "null" ]]; then
+ set_icecat="IceCat,";
+ fi
+
+ if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.android')" != "null" ]]; then
+ set_icecatmobile="IceCatMobile,";
+ fi
+
+ if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.mobile')" != "null" ]]; then
+ set_icecatmobile="IceCatMobile,";
+ fi
+
+ if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.thunderbird')" != "null" ]]; then
+ set_thunderbird="Thunderbird,";
+ fi
+
+ echo "|Extension of=$(echo "$set_seamonkey$set_icecat$set_icecatmobile$set_thunderbird" | sed "s|,$||;")" >> fsd/$slug.xml
+
+
+ if [[ "$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | grep "github.com")" != "null" ]]; then
+ echo "|VCS checkout command=git clone $(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | sed "s|^\"||; s|\"$||;")" >> fsd/$slug.xml
+ fi
+
+ echo "|VCS checkout command=git clone $(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | sed "s|^\"||; s|\"$||;")";
+ echo "|Version identifier=$(cat amo/json/$slug.json | jq '.current_version.version' | sed "s|^\"||; s|\"$||;")
+|Version date=$(cat amo/json/$slug.json | jq '.last_updated' | sed "s|^\"||; s|\"$||;")
+|Version download=https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$(cat amo/json/$slug.json | jq '.current_version.id')-latest.xpi
+|Last review by=FSDbot
+|Last review date=$(date +"%Y/%m/%d")
+|Submitted by=FSDbot
+|Submitted date=2017/02/01
+|Is GNU=No
+}}" >> fsd/$slug.xml
+
+
+ # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/
+ echo "{{Project license
+
+# Stuck here (have to use old API until this have been fixed): https://github.com/mozilla/addons-server/issues/3936
+
+|License=$(cat amo/json.versions/$slug.json | sed "s|-||g;" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;")
+|License copyright=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.authors[].name' | sed "s|^\"||; s|\"$||;")
+}}" >> fsd/$slug.xml
+
+ # Homepage URL can be changed by the community so lets do a backup here
+ echo "{{Resource
+|Resource audience=amo-import
+|Resource kind=General
+|Resource URL=$url
+}}" >> fsd/$slug.xml
+
+
+ if [[ "$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.support_url[]' | sed "s|^\"||; s|\"$||;")" != "" ]];
+ then
+
+ echo "{{Resource
+|Resource audience=Users
+|Resource kind=Support
+|Resource URL=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.support_url[]' | sed "s|^\"||; s|\"$||;")
+}}" >> "fsd/$slug.xml"
+
+ fi
+
+ # Add "E-mail" here when "Support E-mail" have been added to the API: https://github.com/mozilla/addons-server/issues/3901
+
+
+ echo "{{Featured}}</text>
+ </revision>
+ </page>
+</mediawiki>" >> "fsd/$slug.xml"
+
+ cat "fsd/$slug.xml"
+
+
+
+ done < ../amo-repo.txt
+
+
+ ;;
+
+
+ --all)
+
+ $0 --get-json
+ $0 --get-xpi-url
+ $0 --get-xpi
+ $0 --xpi-extract
+ $0 --make-xml
+
+ ;;
+
+ *)
+
+ echo $"$0 [OPTION]
+
+OPTIONS
+ --all
+ --get-json
+ --get-xpi-url
+ --get-xpi
+ --xpi-extract
+ --make-xml"
+ exit 1
+
+esac
+
+exit 0
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-27 08:27:34 +0000