#!/bin/bash # Copyright 2016, 2017 David Hedlund # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # https://addons-server.readthedocs.io/en/latest/topics/api/addons.html # Add to wiki and remove: echo "|Name=$(xmlstarlet sel -t -v "addon/name" test.xml)" SCRIPTSRC=$(readlink -f "$0" || echo "$0") RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .) if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi if [[ $exit == true ]]; then exit 1; fi if [ ! -d "build" ]; then mkdir -p build; fi retrive () { api="$(echo $url | sed "s|https://addons.mozilla.org/en-US/firefox/addon/|https://addons.mozilla.org/api/v3/addons/addon/|g;")"; slug="$(echo $url | sed "s|https://addons.mozilla.org/en-US/firefox/addon/||g; s|/||g")"; } xpi () { cd build/ echo "$download" > tmp.txt sed -i "s|/addon-|\naddon-|" tmp.txt xpi=$(grep ".xpi" tmp.txt); fbname=$(basename "$xpi" | cut -d. -f1); } case "$1" in --get-json) if [ ! -d "build/amo/json" ]; then mkdir -p build/amo/json; fi if [ ! -d "build/amo/json.versions" ]; then mkdir -p build/amo/json.versions; fi cd build/amo/ while read url; do retrive wget $api -O json/$slug.json wget $api\versions/$(cat json/$slug.json | jq '.current_version.id')/ -O json.versions/$slug.json done < ../../amo-repo.txt # List obsoleted projects find json/ -size 0 -print ;; --get-xpi-url) cd build/ rm -f amo_repo-latest_download.txt for i in $(find amo/json/ -maxdepth 1 -type f); do echo "https://addons.mozilla.org/firefox/downloads/latest/$(basename $i | sed "s|.json||")/addon-$(cat $i | jq '.current_version.id').xpi" >> amo_repo-latest_download.txt; done ;; --get-xpi) if [ ! -d "build/amo/xpi" ]; then mkdir -p build/amo/xpi; fi while read download; do xpi if [ ! -f amo/xpi/$xpi ]; then cd amo/xpi/ wget $download cd .. fi cd .. done < build/amo_repo-latest_download.txt ;; --xpi-extract) if [ ! -d "build/amo/xpi.extracted" ]; then mkdir -p build/amo/xpi.extracted; fi while read download; do xpi if [ ! -d amo/xpi.extracted/$fbname ]; then mkdir -p amo/xpi.extracted/$fbname cd amo/xpi.extracted/$fbname unzip ../../xpi/$xpi cd ../../ fi cd .. done < build/amo_repo-latest_download.txt ;; --make-xml) if [ ! -d "build/fsd" ]; then mkdir -p build/fsd; fi # Get number of edits: wget "https://directory.fsf.org/w/api.php?action=query&list=recentchanges&rclimit=1" -O fsdedits.xml oldrevid="$(grep revid fsdedits.xml | sed "s|"||g; s|revid|\nrevid|; s|old_revid|\n|" | grep revid | sed "s|revid=||")" newrevid=$(expr 1 + $oldrevid) cd build/ while read url; do retrive # List JSON structure: cat amo/json/$slug.json | js '.' # XML value "homepage" is not always set in AMO API, but the FSD require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically. echo " Free Software Directory https://directory.fsf.org/wiki/Main_Page MediaWiki 1.20.2 first-letter Media Special Talk User User talk Free Software Directory Free Software Directory talk File File talk MediaWiki MediaWiki talk Template Template talk Help Help talk Category Category talk Property Property talk Form Form talk Concept Concept talk Filter Filter talk Review Review talk License License talk Collection Collection talk Org Org talk IceCat/$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.name.enUS' | sed "s|^\"||; s|\"$||;") $newrevid $(date -u +%Y-%m-%dT%H:%M:%SZ) amo-import 1501 kj1p9nd2p15bqr2tb7jkihshlxow11y {{Entry" > fsd/$slug.xml # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341 echo "|Name=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.name.enUS' | sed "s|^\"||; s|\"$||;") |Short description=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.summary.enUS' | sed "s|^\"||; s|\"$||; s|\\\n|\n|g; s|\\\\\"|\"|g;") |Full description=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.description.enUS' | sed "s|^\"||; s|\"$||; s|\\\n|\n|g; s|\\\\\"|\"|g;") |Homepage URL=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | sed "s|^\"||; s|\"$||;")" >> fsd/$slug.xml if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.seamonkey')" != "null" ]]; then set_seamonkey="SeaMonkey,"; fi if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.firefox')" != "null" ]]; then set_icecat="IceCat,"; fi if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.android')" != "null" ]]; then set_icecatmobile="IceCatMobile,"; fi if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.mobile')" != "null" ]]; then set_icecatmobile="IceCatMobile,"; fi if [[ "$(cat amo/json/$slug.json | jq '.current_version.compatibility.thunderbird')" != "null" ]]; then set_thunderbird="Thunderbird,"; fi echo "|Extension of=$(echo "$set_seamonkey$set_icecat$set_icecatmobile$set_thunderbird" | sed "s|,$||;")" >> fsd/$slug.xml if [[ "$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | grep "github.com")" != "null" ]]; then echo "|VCS checkout command=git clone $(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | sed "s|^\"||; s|\"$||;")" >> fsd/$slug.xml fi echo "|VCS checkout command=git clone $(cat amo/json/$slug.json | sed "s|-||g;" | jq '.homepage.enUS' | sed "s|^\"||; s|\"$||;")"; echo "|Version identifier=$(cat amo/json/$slug.json | jq '.current_version.version' | sed "s|^\"||; s|\"$||;") |Version date=$(cat amo/json/$slug.json | jq '.last_updated' | sed "s|^\"||; s|\"$||;") |Version download=https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$(cat amo/json/$slug.json | jq '.current_version.id')-latest.xpi |Last review by=FSDbot |Last review date=$(date +"%Y/%m/%d") |Submitted by=FSDbot |Submitted date=2017/02/01 |Is GNU=No }}" >> fsd/$slug.xml # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/ echo "{{Project license # Stuck here (have to use old API until this have been fixed): https://github.com/mozilla/addons-server/issues/3936 |License=$(cat amo/json.versions/$slug.json | sed "s|-||g;" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;") |License copyright=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.authors[].name' | sed "s|^\"||; s|\"$||;") }}" >> fsd/$slug.xml # Homepage URL can be changed by the community so lets do a backup here echo "{{Resource |Resource audience=amo-import |Resource kind=General |Resource URL=$url }}" >> fsd/$slug.xml if [[ "$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.support_url[]' | sed "s|^\"||; s|\"$||;")" != "" ]]; then echo "{{Resource |Resource audience=Users |Resource kind=Support |Resource URL=$(cat amo/json/$slug.json | sed "s|-||g;" | jq '.support_url[]' | sed "s|^\"||; s|\"$||;") }}" >> fsd/$slug.xml fi # Add "E-mail" here when "Support E-mail" have been added to the API: https://github.com/mozilla/addons-server/issues/3901 echo "{{Featured}} " >> fsd/$slug.xml cat fsd/$slug.xml done < ../amo-repo.txt ;; --all) $0 --get-json $0 --get-xpi-url $0 --get-xpi $0 --xpi-extract $0 --make-xml ;; *) echo $"$0 [OPTION] OPTIONS --all --get-json --get-xpi-url --get-xpi --xpi-extract --make-xml" exit 1 esac exit 0