From 014ec8c41a71bbc93092bbeabec39e4881201cbe Mon Sep 17 00:00:00 2001 From: david Date: Tue, 22 May 2018 20:47:41 +0200 Subject: I only marked addons.mozilla.org-fsd for commit --- .../get-data/src/download-custom | 86 ++++++++++++++++++++++ .../get-data/src/download-custom~ | 86 ++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100755 addons.mozilla.org-fsd/get-data/src/download-custom create mode 100755 addons.mozilla.org-fsd/get-data/src/download-custom~ (limited to 'addons.mozilla.org-fsd/get-data/src') diff --git a/addons.mozilla.org-fsd/get-data/src/download-custom b/addons.mozilla.org-fsd/get-data/src/download-custom new file mode 100755 index 0000000..b6c5def --- /dev/null +++ b/addons.mozilla.org-fsd/get-data/src/download-custom @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +# This file is part of license-reporter +# Copyright (C) 2017 David Hedlund +# +# license-reporter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# license-reporter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +cd build || exit +mkdir json + +cp -a "$RUN_PATH/license-reporter-custom" custom_addons.txt + +# Sanitize a copy from empty lines, they use to occur at the end of the file +sed -i '/^\s*$/d' license-reporter-custom + + +sed -i "s|https://addons.mozilla.org/en-US/firefox/addon/||g; s|/||" custom_addons.txt + +rm -fr \ + custom_addons-json-source.txt \ + custom_addons-average_daily_users.txt \ + custom_addons-id.txt custom_addons-slug.txt \ + custom_addons-merged.txt \ + custom_addons-report-expired_links.txt \ + custom_addons-merged-sorted.txt + +cd json || exit + +lines="1"; + +while IFS= read -r slug; do + + + if [ "$debug" == false ] || ( [ "$debug" == true ] && [ "$lines" -le 3 ] ); then + + wget "https://addons.mozilla.org/api/v3/addons/addon/$slug/" -O "$slug.json"; + [ -s "$slug.json" ] || rm -f "$slug.json" # Delete empty files saved by wget, tis is much faster than using wget options to first validate the links. + + # Ignore expired URLs + if [ -f "$slug.json" ]; then + + id="$(jq ".id" "$slug.json")"; + mv "$slug.json" "$id.json" + + if [ "$(jq ".average_daily_users" "$id.json")" -gt "$minimum_average_daily_users" ]; then + + echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tAlready in the main repository\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$RUN_PATH/license-reporter-log" + + else + + jq ".average_daily_users" "$id.json" >> ../custom_addons-average_daily_users.txt + + # We need the JSON source because they are different for search indexed add-ons (they are store in index.html*) + jq ".id" "$id.json" >> ../custom_addons-id.txt + echo "$(jq ".id" "$id.json").json" >> ../custom_addons-json-source.txt + + ((lines++)) + + + fi + + + else + + echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tLink rot\t\t\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$RUN_PATH/license-reporter-log" + + fi + + fi + +done < ../custom_addons.txt + +# Sort by average_daily_users +paste ../custom_addons-average_daily_users.txt ../custom_addons-id.txt ../custom_addons-json-source.txt > ../custom_addons-merged.txt +sort -k1,1nr ../custom_addons-merged.txt > ../merged-custom.txt diff --git a/addons.mozilla.org-fsd/get-data/src/download-custom~ b/addons.mozilla.org-fsd/get-data/src/download-custom~ new file mode 100755 index 0000000..4af8ce1 --- /dev/null +++ b/addons.mozilla.org-fsd/get-data/src/download-custom~ @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +# This file is part of license-reporter +# Copyright (C) 2017 David Hedlund +# +# license-reporter is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# license-reporter is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +cd build || exit +mkdir json + +cp -a "$RUN_PATH/license-reporter-custom" custom_addons.txt + +# Sanitize a copy from empty lines, they use to occur at the end of the file +sed -i '/^\s*$/d' license-reporter-custom + + +sed -i "s|https://addons.mozilla.org/en-US/firefox/addon/||g; s|/||" custom_addons.txt + +rm -fr \ + custom_addons-json-source.txt \ + custom_addons-average_daily_users.txt \ + custom_addons-id.txt custom_addons-slug.txt \ + custom_addons-merged.txt \ + custom_addons-report-expired_links.txt \ + custom_addons-merged-sorted.txt + +cd json || exit + +lines="1"; + +while IFS= read -r slug; do + + + if [ "$debug" == false ] || [ "$debug" == true ] && [ "$lines" -le 3 ]; then + + wget "https://addons.mozilla.org/api/v3/addons/addon/$slug/" -O "$slug.json"; + [ -s "$slug.json" ] || rm -f "$slug.json" # Delete empty files saved by wget, tis is much faster than using wget options to first validate the links. + + # Ignore expired URLs + if [ -f "$slug.json" ]; then + + id="$(jq ".id" "$slug.json")"; + mv "$slug.json" "$id.json" + + if [ "$(jq ".average_daily_users" "$id.json")" -gt "$minimum_average_daily_users" ]; then + + echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tAlready in the main repository\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$RUN_PATH/license-reporter-log" + + else + + jq ".average_daily_users" "$id.json" >> ../custom_addons-average_daily_users.txt + + # We need the JSON source because they are different for search indexed add-ons (they are store in index.html*) + jq ".id" "$id.json" >> ../custom_addons-id.txt + echo "$(jq ".id" "$id.json").json" >> ../custom_addons-json-source.txt + + ((lines++)) + + + fi + + + else + + echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tLink rot\t\t\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$RUN_PATH/license-reporter-log" + + fi + + fi + +done < ../custom_addons.txt + +# Sort by average_daily_users +paste ../custom_addons-average_daily_users.txt ../custom_addons-id.txt ../custom_addons-json-source.txt > ../custom_addons-merged.txt +sort -k1,1nr ../custom_addons-merged.txt > ../merged-custom.txt -- cgit v1.2.3