From 1ae516a7bab1d5d9f02ab00c4ac13c219f43c1a1 Mon Sep 17 00:00:00 2001 From: david Date: Tue, 17 Jul 2018 15:52:52 +0200 Subject: Renamed the AMO/FSD project to FreeAMO, and rewrote its architecture. --- .../src/unmaintained/sync-data/fsd-amo-bot~ | 306 +++++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100755 subprojects/freeamo/src/unmaintained/sync-data/fsd-amo-bot~ (limited to 'subprojects/freeamo/src/unmaintained/sync-data/fsd-amo-bot~') diff --git a/subprojects/freeamo/src/unmaintained/sync-data/fsd-amo-bot~ b/subprojects/freeamo/src/unmaintained/sync-data/fsd-amo-bot~ new file mode 100755 index 0000000..8f1d6f3 --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/sync-data/fsd-amo-bot~ @@ -0,0 +1,306 @@ +#!/bin/bash + +# Copyright 2016, 2017 David Hedlund +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# https://addons-server.readthedocs.io/en/latest/topics/api/addons.html +# Add to wiki and remove: echo "|Name=$(xmlstarlet sel -t -v "addon/name" test.xml)" + +SCRIPTSRC=$(readlink -f "$0" || echo "$0") +RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .) + +if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi +if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi + +if [[ $exit == true ]]; then exit 1; fi + + +if [ ! -d "build" ]; then mkdir -p build; fi + +retrive () { +api="$(echo $url | sed "s|https://addons.mozilla.org/en-US/firefox/addon/|https://addons.mozilla.org/api/v3/addons/addon/|g;")"; +slug="$(echo $url | sed "s|https://addons.mozilla.org/en-US/firefox/addon/||g; s|/||g")"; +} + +xpi () { + cd build/ + echo "$download" > tmp.txt + sed -i "s|/addon-|\naddon-|" tmp.txt + xpi=$(grep ".xpi" tmp.txt); + fbname=$(basename "$xpi" | cut -d. -f1); +} + + +case "$1" in + --get-json) + + if [ ! -d "build/amo/json" ]; then mkdir -p build/amo/json; fi + if [ ! -d "build/amo/json.versions" ]; then mkdir -p build/amo/json.versions; fi + cd build/amo/ + + while read url; do + + retrive + wget $api -O json/$slug.json + wget $api\versions/$(cat json/$slug.json | jq '.current_version.id')/ -O json.versions/$slug.json + + done < ../../amo-repo.txt + + # List obsoleted projects + find json/ -size 0 -print + + ;; + --get-xpi-url) + + cd build/ + rm -f amo_repo-latest_download.txt + for i in $(find amo/json/ -maxdepth 1 -type f); do echo "https://addons.mozilla.org/firefox/downloads/latest/$(basename $i | sed "s|.json||")/addon-$(cat $i | jq '.current_version.id').xpi" >> amo_repo-latest_download.txt; done + + ;; + + --get-xpi) + + if [ ! -d "build/amo/xpi" ]; then mkdir -p build/amo/xpi; fi + + while read download; do + + xpi + + if [ ! -f amo/xpi/$xpi ]; then + + cd amo/xpi/ + wget $download + cd .. + + fi + + cd .. + + + done < build/amo_repo-latest_download.txt + + ;; + + --xpi-extract) + + if [ ! -d "build/amo/xpi.extracted" ]; then mkdir -p build/amo/xpi.extracted; fi + + while read download; do + + xpi + + if [ ! -d amo/xpi.extracted/$fbname ]; then + + mkdir -p amo/xpi.extracted/$fbname + cd amo/xpi.extracted/$fbname + unzip ../../xpi/$xpi + cd ../../ + + fi + + cd .. + + done < build/amo_repo-latest_download.txt + + ;; + + --make-xml) + + if [ ! -d "build/fsd" ]; then mkdir -p build/fsd; fi + + # Get number of edits: + wget "https://directory.fsf.org/w/api.php?action=query&list=recentchanges&rclimit=1" -O fsdedits.xml + oldrevid="$(grep revid fsdedits.xml | sed "s|"||g; s|revid|\nrevid|; s|old_revid|\n|" | grep revid | sed "s|revid=||")" + newrevid=$(expr 1 + $oldrevid) + + cd build/ + + while read url; do + + retrive + # List JSON structure: cat amo/json/$slug.json | js '.' + # XML value "homepage" is not always set in AMO API, but the FSD require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically. + echo " + + Free Software Directory + https://directory.fsf.org/wiki/Main_Page + MediaWiki 1.20.2 + first-letter + + Media + Special + + Talk + User + User talk + Free Software Directory + Free Software Directory talk + File + File talk + MediaWiki + MediaWiki talk + Template + Template talk + Help + Help talk + Category + Category talk + Property + Property talk + Form + Form talk + Concept + Concept talk + Filter + Filter talk + Review + Review talk + License + License talk + Collection + Collection talk + Org + Org talk + + + + IceCat/$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.name.enUS' | sed "s|^\"||; s|\"$||;") + + $newrevid + $(date -u +%Y-%m-%dT%H:%M:%SZ) + + amo-import + 1501 + + kj1p9nd2p15bqr2tb7jkihshlxow11y + {{Entry" > fsd/$slug.xml + + # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341 + echo "|Name=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.name.enUS' | sed "s|^\"||; s|\"$||;") +|Short description=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.summary.enUS' | sed "s|^\"||; s|\"$||; s|\\\n|\n|g; s|\\\\\"|\"|g;") +|Full description=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.description.enUS' | sed "s|^\"||; s|\"$||; s|\\\n|\n|g; s|\\\\\"|\"|g;") +|Homepage URL=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | sed "s|^\"||; s|\"$||;")" >> fsd/$slug.xml + + + if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.seamonkey')" != "null" ]]; then + set_seamonkey="SeaMonkey,"; + fi + + if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.firefox')" != "null" ]]; then + set_icecat="IceCat,"; + fi + + if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.android')" != "null" ]]; then + set_icecatmobile="IceCatMobile,"; + fi + + if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.mobile')" != "null" ]]; then + set_icecatmobile="IceCatMobile,"; + fi + + if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.thunderbird')" != "null" ]]; then + set_thunderbird="Thunderbird,"; + fi + + echo "|Extension of=$(echo "$set_seamonkey$set_icecat$set_icecatmobile$set_thunderbird" | sed "s|,$||;")" >> fsd/$slug.xml + + + if [[ "$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | grep "github.com")" != "null" ]]; then + echo "|VCS checkout command=git clone $(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | sed "s|^\"||; s|\"$||;")" >> fsd/$slug.xml + fi + +echo "|VCS checkout command=git clone $(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | sed "s|^\"||; s|\"$||;")"; + echo "|Version identifier=$(cat amo/json/$slug.json | jq '.current_version.version' | sed "s|^\"||; s|\"$||;") +|Version date=$(cat amo/json/$slug.json | jq '.last_updated' | sed "s|^\"||; s|\"$||;") +|Version download=https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$(cat amo/json/$slug.json | jq '.current_version.id')-latest.xpi +|Last review by=FSDbot +|Last review date=$(date +"%Y/%m/%d") +|Submitted by=FSDbot +|Submitted date=2017/02/01 +|Is GNU=No +}}" >> fsd/$slug.xml + + + # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/ + echo "{{Project license + +# Stuck here (have to use old API until this have been fixed): https://github.com/mozilla/addons-server/issues/3936 + +|License=$(cat amo/json.versions/$slug.json | sed "s|-||g;" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;") +|License copyright=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.authors[].name' | sed "s|^\"||; s|\"$||;") +}}" >> fsd/$slug.xml + + # Homepage URL can be changed by the community so lets do a backup here + echo "{{Resource +|Resource audience=amo-import +|Resource kind=General +|Resource URL=$url +}}" >> fsd/$slug.xml + + + if [[ "$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.support_url[]' | sed "s|^\"||; s|\"$||;")" != "" ]]; + then + + echo "{{Resource +|Resource audience=Users +|Resource kind=Support +|Resource URL=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.support_url[]' | sed "s|^\"||; s|\"$||;") +}}" >> fsd/$slug.xml + + fi + + # Add "E-mail" here when "Support E-mail" have been added to the API: https://github.com/mozilla/addons-server/issues/3901 + + + echo "{{Featured}} + + +" >> fsd/$slug.xml + + cat fsd/$slug.xml + + + + done < ../amo-repo.txt + + + ;; + + + --all) + + $0 --get-json + $0 --get-xpi-url + $0 --get-xpi + $0 --xpi-extract + $0 --make-xml + + ;; + + *) + + echo $"$0 [OPTION] + +OPTIONS + --all + --get-json + --get-xpi-url + --get-xpi + --xpi-extract + --make-xml" + exit 1 + +esac + +exit 0 -- cgit v1.2.3