diff options
Diffstat (limited to 'addons.mozilla.org-fsd/get-data')
13 files changed, 0 insertions, 1389 deletions
diff --git a/addons.mozilla.org-fsd/get-data/TODO b/addons.mozilla.org-fsd/get-data/TODO deleted file mode 100644 index 52b5c0e..0000000 --- a/addons.mozilla.org-fsd/get-data/TODO +++ /dev/null @@ -1,17 +0,0 @@ -* Merged https://directory.fsf.org/wiki/Free_Software_Directory:IceCat_WebExtensions_(proposed)/About to the README - -The author of the package, licenseutils, has implemented an option per request that make it possible to use detect licenses, its evoked with: -$ licensing detect * - -I cannot install licenseutils for the moment because I held broken packages in my system. - - -############################### - -Look in GitHub repositories for LICENSE or COPYING. Sometime the add-on is delayed or have not been released to addons.mozilla.org." - -############################### -license-reporter-anti-features - -Work on license-reporter-anti-features - diff --git a/addons.mozilla.org-fsd/get-data/TODO~ b/addons.mozilla.org-fsd/get-data/TODO~ deleted file mode 100644 index 5bb2611..0000000 --- a/addons.mozilla.org-fsd/get-data/TODO~ +++ /dev/null @@ -1,5 +0,0 @@ -The author of the package, licenseutils, has implemented an option per request that make it possible to use detect licenses, its evoked with: -$ licensing detect * - -I cannot install licenseutils for the moment because I held broken packages in my system. - diff --git a/addons.mozilla.org-fsd/get-data/err b/addons.mozilla.org-fsd/get-data/err deleted file mode 100755 index 97b3cc3..0000000 --- a/addons.mozilla.org-fsd/get-data/err +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -# Copyright 2018 Ian Kelling - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Commentary: Bash stack trace and error handling functions. This file -# is meant to be sourced. It loads some functions which you may want to -# call manually (see the comments at the start of each one), and then -# runs err-catch. See the README file for a slightly longer explanation. - -err-allow() { - # help: turn off exit and stack trace on error. undoes err-catch - set +E +o pipefail; trap ERR -} - -err-bash-trace() { - # help: print stack trace - # - # Note: It does not show function args unless you first run: - # shopt -s extdebug - # err-catch runs this for you. - - local -i argc_index=0 frame i start=${1:-1} max_indent=8 indent - local source - local extdebug=false - if [[ $(shopt -p extdebug) == *-s* ]]; then - extdebug=true - fi - for ((frame=0; frame < ${#FUNCNAME[@]}-1; frame++)); do - argc=${BASH_ARGC[frame]} - argc_index+=$argc - ((frame < start)) && continue - if (( ${#BASH_SOURCE[@]} > 1 )); then - source="${BASH_SOURCE[frame+1]}:${BASH_LINENO[frame]}:" - fi - indent=$((frame-start+1)) - indent=$((indent < max_indent ? indent : max_indent)) - printf "%${indent}s↳%sin \`%s" '' "$source" "${FUNCNAME[frame]}" - if $extdebug; then - for ((i=argc_index-1; i >= argc_index-argc; i--)); do - printf " %s" "${BASH_ARGV[i]}" - done - fi - echo \' - done -} - -err-catch() { - # help: on errors: print stack trace and exit - # - # You can set "${_errcatch_cleanup[@]}" to a command and it will run before exiting. - # This function depends on err-bash-trace. - - set -E; shopt -s extdebug - _err-trap() { - err=$? - exec >&2 - set +x - echo "${BASH_SOURCE[1]}:${BASH_LINENO[0]}:in \`$BASH_COMMAND' returned $err" - # err trap does not work within an error trap, the following line: - err-bash-trace 2; set -e - "${_errcatch_cleanup[@]:-}" # note :- is to be compatible with set -u - echo "$0: exiting with code $err" - exit $err - } - trap _err-trap ERR - set -o pipefail -} - -err-exit() { - # usage: err-exit [EXIT_CODE] [MESSAGE] - # help: exit and print stack trace. - # - # Use this instead of the exit command to be more informative. default - # EXIT_CODE is 1. If only one of EXIT_CODE and MESSAGE is given, - # we consider it to be an exit code if it is a number. - # This function depends on err-bash-trace. - - exec >&2 - code=1 - if [[ "$*" ]]; then - if [[ ${1/[^0-9]/} == "$1" ]]; then - code=$1 - if [[ $2 ]]; then - printf '%s\n' "$2" - fi - else - printf '%s\n' "$0: $1" - fi - fi - echo "${BASH_SOURCE[1]}:${BASH_LINENO[0]}" - err-bash-trace 2 - echo "$0: exiting with code $code" - exit $err -} - -err-catch diff --git a/addons.mozilla.org-fsd/get-data/license-reporter b/addons.mozilla.org-fsd/get-data/license-reporter deleted file mode 100755 index 3e01334..0000000 --- a/addons.mozilla.org-fsd/get-data/license-reporter +++ /dev/null @@ -1,964 +0,0 @@ -#!/usr/bin/env bash -# This file is part of license-reporter -# Copyright (C) 2017, 2018 David Hedlund -# -# license-reporter is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# license-reporter is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -source err - -#if [ "$data" == "search" ]; then search_page_variable='.results[]'; fi - - -# Specifically for titles: en-US is sometimes unused, en-GB used to be used then. -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.name.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-GB|enGB|g" "$json_file" | jq "$search_page_variable.name.enGB" | sed "s|^\"||; s|\"$||;" - -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.homepage.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_email.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_url.enUS" | sed "s|^\"||; s|\"$||;" - - -# Licenses -# sed "s|en-US|enUS|g" "$slug.json" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;" - -if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi - -minimum_average_daily_users="10000"; - -scriptsrc=$(readlink -f -- "${BASH_SOURCE[0]}") -SCRIPTSRC=$(readlink -f "$0" || echo "$0") -RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .) -filename=$(basename "$0"); -export -p SCRIPTSRC RUN_PATH minimum_average_daily_users debug - -if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi -if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi - -if [[ $exit == true ]]; then exit 1; fi - - -function check_files { - - if [ ! -f "$RUN_PATH/license-reporter-$check_type" ]; then - - echo "$RUN_PATH/license-reporter-$check_type doesn't exist."; - echo "Exiting" && exit - - else - - - # Make sure all links are using SSL - if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-$check_type; then - - echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)." - echo "Exiting" && exit - - fi - - fi - -} - -function line_status { - - - if [ "$file" == "$previous_file" ]; then - - ((line++)) - - else - - line="0"; # Should not be 1 - - fi - -} - - -function verify_variables { - - if [ -z ${file+x} ] || [ "$file" != "$(echo $table | awk '{print $3}')" ]; then - - line="0"; # should not be 1 - - fi - - average_daily_users=$(echo $table | awk '{print $1}'); - id=$(echo $table | awk '{print $2}'); - file=$(echo $table | awk '{print $3}'); - slug=$(echo $table | awk '{print $4}'); - name=$(echo "$table" | cut -f 5); - has_eula=$(echo "$table" | cut -f 6); - - - - firefox_max_version="$(jq .results[$line].current_version.compatibility.firefox.max $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|.\*||")"; - nativeMessaging="$(jq .results[$line].current_version.files[0].permissions $RUN_PATH/build/json/$file | grep "nativeMessaging")"; - - # Values are either an integer or "*" - re='^[0-9]+$' - if [[ $firefox_max_version =~ $re ]] && [ "$firefox_max_version" -ge 60 ] || [ "$firefox_max_version" == '*' ]; then - - # echo "$firefox_max_version" - firefox_max_version__pass=true; - - else - - unset firefox_max_version__pass - - fi - - if [ ! -n "$nativeMessaging" ]; then - - nativeMessaging__pass=true; - - else - - nativeMessaging__pass=false; - - fi - - echo "$nativeMessaging__pass" - - - # curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging - - - # API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668 - # The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies. - - # Old solution: - - # 1: Add this to license-reporter-dependencies: - #belgium-eid eid-mw - - # 2: Uncomment this code: - # depends_on_external_program=false; - # while IFS= read -r dependencies; do - # - # dependency_slug=$(echo $dependencies | awk '{print $1}'); - # dependency=$(echo "$dependencies" | cut -f 2); - # - # if [ "$dependency_slug" == "$slug" ]; then - # - # depends_on_external_program=true; - # - # fi - # - # done < $RUN_PATH/license-reporter-dependencies - - - - - approved_program=false; - if [ "$has_eula" == "eulafree" ] && [ "$firefox_max_version__pass" = true ] && [ "$nativeMessaging__pass" = true ]; then - - if [ "$1" == "--get-licenses" ]; then - - approved_program=true; - - else - - license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - - - - # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". - # See https://addons.mozilla.org/en-US/developers/addon/<your add-on>/ownership for the current add-on list. Note that not all licenses are added to the list. - # DuckDuckGo Privacy Essentials is distributed under Apache License 2.0. - # Apache License, version 2.0 is not on the add-on license list - https://github.com/mozilla/addons-server/issues/8545 - if - [ "$license" == "Apache License, Version 2.0" ] || # Custom license - [ "$license" == "Apache License, version 2.0" ] || # Custom license - [ "$license" == "BSD License" ] || - [ "$license" == "GNU General Public License, version 3.0" ] || - [ "$license" == "GNU General Public License, version 2.0" ] || - [ "$license" == "GNU Lesser General Public License, version 3.0" ] || - [ "$license" == "GNU Lesser General Public License, version 2.1" ] || - [ "$license" == "MIT/X11 License" ] || - [ "$license" == "Mozilla Public License, version 2.0" ] || - [ "$license" == "Mozilla Public License Version 1.1" ] - then - - # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files. - freedom_status="free"; - - elif [ "$license" == "All Rights Reserved" ]; then - - freedom_status="nonfree"; - - else - - freedom_status="unknown"; - fi - - [ "$freedom_status" == "free" ] - - fi - - fi - -} - -case "$1" in - - ""|-help) - - [ "$1" = "" ] && echo "Usage: $filename [--option] [--debug] - -OPTIONS - $0 --fresh-build - Remove the build directory and create a new one. - $0 --make-search-list - Downloads the most popular WebExtensions from https://addons.mozilla.org/en-US/firefox/search/... - $0 --make-repository-list - Required file: license-reporter-collections - File content syntax: https://addons.mozilla.org/en-US/firefox/collections/<user>/ - $0 --make-collection-list - Required file: license-reporter-collections - File content syntax: https://addons.mozilla.org/en-US/firefox/collections/<user>/<collection name> - $0 --make-custom-list - Required file: license-reporter-custom - File content syntax: https://addons.mozilla.org/en-US/firefox/addon/<add-on name>/ - $0 --merge-lists - Merge the lists to MERGED-ALL.txt - $0 --get-licenses - The licenses are not added to the general JSON file. This option will download the neccesary license JSON files for all add-ons. - $0 --download-licenses-json - $0 --download-free-webextensions - Downloads the latest version of the free WebExtensions. - $0 --verify-license-copy - Search for license files in the root directory in the downloaded WebExtensions (.xpi files) - $0 --merged-free - Generates MERGED-ALL-freedom_status.txt - $0 --make-wiki - Generates .wiki files - - --all-recommended - $0 --fresh-build - $0 --make-search-list - $0 --merge-lists - $0 --get-licenses - $0 --merged-free - $0 --make-wiki - -" && exit 1 - - ;; - - --fresh-build) - - if [ -d "build" ]; then - - rm -fr $RUN_PATH/build - - fi - - mkdir -p $RUN_PATH/build - - ;; - - --make-search-list) - - - cd build/ || exit - mkdir json - cd json || exit - - # API documentation: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html - - - page="1" - - function foo() { - - # Do not evaluate the number of extensions seen in https://addons.mozilla.org/en-US/firefox/search/ in a Quantum based browser since that will hide legacy add-ons. - uri_query="page=$page&platform=linux&sort=users&type=extension"; - - - # Get the most popular WebExtensions - wget "https://addons.mozilla.org/api/v3/addons/search/?$uri_query" -O "index.html?$uri_query.json" - - ((page++)) - - } - - - foo - - - - if [ "$debug" == false ]; then - - while [ "$(jq ".results[].average_daily_users" "index.html?$uri_query.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do - - foo - - done - - fi - - - # Sort by file modification time stamp for the downloaded JSON files - find index*.json -type f -printf "%Tc %p\n" | sort -n | awk '{print $NF}' | while read -r file; do - - - - - line="0"; # Should not be 1 - - for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do - - - if [ "$(jq ".results[$line].has_eula" "$file")" == "false" ]; then - - has_eula="eulafree"; - - else - - has_eula="eula"; - - fi - - - # Clean titles - - name=$(jq .results[$line].name $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - - if [ "$name" == "null" ]; then - - name=$(jq .results[$line].name $RUN_PATH/build/json/$file | sed "s|en-GB|enGB|;" | jq .enGB | sed "s|^\"||; s|\"$||;"); - - fi - - - # Clean titles. Example of bad titles: "Ghostery – Privacy Ad Blocker" - - # - - # Hard to filter out "-" since many add-ons use it, see for example https://addons.mozilla.org/en-US/firefox/search/?platform=linux&q=1-click - # Flash Video Downloader - YouTube HD Download [4K] - - name=$(echo $name | sed ' -s/ — /\n/g; -s/ – /\n/g; -s/ - /\n/g; # looks identical to the prevoius line but is unique -s/: /\n/g; -s/(/\n/g; -s/,/\n/g; -' | head -n 1 | sed 's/ $//'); - - - - - - - # Exclude add-ons with to low average daily users from index.html* - if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then - - echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file\t$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")\t$name\t$has_eula" >> ../merged-search.txt - - - fi - - ((line++)) - - done - - - done - - ;; - --make-repository-list) - - # It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs. - # This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync - # your addons.mozilla.org collection list with a separate text file. - - # https://github.com/mozilla/addons/issues/722 - # Cannot list collections in desired formats - # Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list). - # So we have to parse it from HTML. - - - # Get the links to the collections - - function get_repositories { - - check_type="repositories"; - check_files - - cd build/ || exit - - for i in $(cat ../license-reporter-repositories); do - - user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;"); - rm -f repository-$user - - - function wget_repository { - - match="^ <a href=\"/en-US/firefox/collections/" - wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user - - } - - # Figure out if there are several pages - number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||"); - - if [ -z "$number_of_pages" ]; then - - echo "$i doesn't have any sub pages"; - wget_repository - - else - - echo "$i has sub pages"; - - page_number=1; - - while [ $page_number -le $number_of_pages ] - do - - wget_repository__page_extension="?page=$page_number"; - echo "$wget_repository__page_extension" - wget_repository - page_number=$(( $page_number + 1 )) - - done - - fi - - done - } - get_repositories - - - # Download the JSON files from the collections - - cd .. - for x in $(find build/ -name repository* | sort); do - - for i in $(cat $x); do - - collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); - wget $i"format%3Ajson" -O "build/collection-$collection_name.json" - - done - - done - - ;; - - --make-collection-list) - - check_type="collections"; - check_files - - cd build/ || exit - for i in $(cat ../license-reporter-collections); do - - # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson - collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); - wget $i"format%3Ajson" -O "collection-$collection_name.json" - - done - - ;; - - --make-custom-list) - - check_type="custom"; - check_files - - src/download-custom - - ;; - - --merge-lists) - - cd build || exit - - cat merged-search.txt >> MERGED-ALL.txt - cat merged-custom.txt >> MERGED-ALL.txt - - ;; - --get-licenses) - - # API: Please adopt SPDX - https://github.com/mozilla/addons-server/issues/8706 - # Allow GNU [L]GPL x.x or later - https://github.com/mozilla/addons-server/issues/8707 - - cd build/ || exit - mkdir -p json/current_versions - cd json/current_versions || exit - - line="0"; # Should not be 1 - all_lines="1" # Should not be 0 - - # Begin template - unset file - while IFS= read -r table; do - - verify_variables "$1" - - if [ "$approved_program" = true ]; then - # End template - - echo "ddd"; - - id=$(echo $table | awk '{print $2}'); - file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}'); - has_eula=$(echo "$table" | cut -f 6); - - # line_status - - total=$(wc -l < $RUN_PATH/build/MERGED-ALL.txt) - echo "####################################### -Entry: $all_lines (approved) / $total" - - - # echo $(jq ".results[$line].slug" "$file"); - - echo "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" - - - # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json" - - previous_file="$file"; - ((all_lines++)) - - - # Begin template - fi - - ((line++)) - - done < $RUN_PATH/build/MERGED-ALL.txt - # End template - - ;; - --make-wiki) - - cd build/ || exit - if [ ! -d "wiki" ]; then mkdir wiki; fi - - echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\" -! WebExtension -! Description -! Claimed license -! Users -! Rating -! Updated" > "wiki/IceCat WebExtensions (proposed).wiki" - - - # Begin template - unset file - while IFS= read -r table; do - - verify_variables "$1" - - if [ "$approved_program" = true ]; then - # End template - - #################################### - # Build the core wiki pages - - average_daily_users=$(echo $table | awk '{print $1}'); - id=$(echo $table | awk '{print $2}'); - file=$(echo $table | awk '{print $3}'); - slug=$(echo $table | awk '{print $4}'); - name=$(echo "$table" | cut -f 5); - has_eula=$(echo "$table" | cut -f 6); - freedom_status=$(echo "$table" | cut -f 7); - - license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - - - if [ "$license" == "Apache License, Version 2.0" ]; then license="Apache2.0"; - elif [ "$license" == "Apache License, version 2.0" ]; then license="Apache2.0"; - elif [ "$license" == "BSD License" ]; then license="BSD 2Clause"; - elif [ "$license" == "GNU General Public License, version 3.0" ]; then license="GPLv3"; - elif [ "$license" == "GNU General Public License, version 2.0" ]; then license="GPLv2"; - elif [ "$license" == "GNU Lesser General Public License, version 3.0" ]; then license="LGPLv3"; - elif [ "$license" == "GNU Lesser General Public License, version 2.1" ]; then license="LGPLv2.1"; - elif [ "$license" == "MIT/X11 License" ]; then license="X11"; - elif [ "$license" == "Mozilla Public License, version 2.0" ]; then license="MPLv2.0"; - elif [ "$license" == "Mozilla Public License Version 1.1" ]; then license="MPLv1.1"; - fi - - - - - short_description_raw=$(jq .results[$line].summary $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|\\\n|\n|g; s|\\\||g; -s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues -s|http|\nhttp|g; -"); - short_description_again=$(echo "$short_description_raw" | sed "/outgoing.prod.mozaws.net/d;"); - - short_description=$(echo "$short_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g'); - - full_description_raw=$(jq .results[$line].description $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||; - s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues -"); - full_description_again=$(echo "$full_description_raw" | sed "/outgoing.prod.mozaws.net/d;"); - - full_description=$(echo "$full_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g'); - - - - - homepage_url=$(jq .results[$line].homepage $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - version=$(jq .results[$line].current_version.version $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;"); - version_date=$(jq .results[$line].last_updated $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g"); - version_id=$(jq .results[$line].current_version.id $RUN_PATH/build/json/$file); - license_copyright=$(jq .results[$line].authors[].name $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;"); - bayesian_average=$(jq .results[$line].ratings.bayesian_average $RUN_PATH/build/json/$file); - bayesian_average__simple=$(printf "%.1f" "$(echo "$bayesian_average" | sed "s|\.|,|")"); - - last_review_date=$(date +"%Y/%m/%d"); - last_review_by="wikisysbot"; - submitted_by="wikisysbot"; - - if [[ "$(jq .results[$line].current_version.compatibility.seamonkey $RUN_PATH/build/json/$file)" != "null" ]]; then - set_seamonkey="Iceape,"; - fi - - if [[ "$(jq .results[$line].current_version.compatibility.firefox $RUN_PATH/build/json/$file)" != "null" ]]; then - set_icecat="IceCat,"; - fi - - # if [[ "$(jq .results[$line].current_version.compatibility.android $RUN_PATH/build/json/$file)" != "null" ]] || [[ "$(jq .results[$line].current_version.compatibility.mobile $RUN_PATH/build/json/$file)" != "null" ]]; then - # set_icecatmobile="IceCatMobile,"; - # fi - - if [[ "$(jq .results[$line].current_version.compatibility.thunderbird $RUN_PATH/build/json/$file)" != "null" ]]; then - set_thunderbird="Icedove,"; - fi - - # Do not list IceCatMobile - extension_of=$(echo "$set_seamonkey$set_icecat$set_thunderbird" | sed "s|,$||;"); - - - - github_true=$(echo "$homepage_url" | grep "github.com"); - - if [[ $github_true != "" ]]; then - - # Always use https, and remove anchors - homepage_url__for__vcs_checkout_command="https://github.com$(echo $github_true | sed "s|https://github.com||; s|http://github.com||; s|#|\n|;" | head -n 1)"; - # Remove trailing slash - homepage_url__for__vcs_checkout_command="${homepage_url__for__vcs_checkout_command%/}" - vcs_checkout_command="git clone $homepage_url__for__vcs_checkout_command.git"; - - fi - - if [ "$name" == "GNU LibreJS" ]; then - - is_gnu="Yes"; - - else - - is_gnu="No"; - - fi - - support_url=$(jq .results[$line].support_url $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - - # List JSON structure: cat json/$file | js '.' - # XML value "homepage" is not always set in AMO API, but the XML require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically. - echo "{{Entry" > wiki/$id.wiki - - # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341 - # Avoid specific versions since it's not compatible with all IceCat versions: - # |Version download=https://addons.mozilla.org/firefox/downloads/latest/$id/addon-$version_id-latest.xpi - # |Version identifier=$version - # |Version date=$version_date - - - echo "|Name=$name -|Short description=$short_description -|Full description=$full_description -|Homepage URL=$homepage_url -|Extension of=$extension_of -|VCS checkout command=$vcs_checkout_command -|Version download=https://addons.mozilla.org/en-US/firefox/addon/$slug/versions/ -|Last review by=$last_review_by -|Last review date=$last_review_date -|Submitted by=$submitted_by -|Submitted date=2018/06/20 -|Is GNU=$is_gnu -}}" >> wiki/$id.wiki - - - - # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/ - echo "{{Project license -|License=$license -|License copyright=$license_copyright -|License note=\"License: $original_license\" listed -at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/ -}}" >> wiki/$id.wiki - - if [[ "$support_url" != "" ]]; - then - - echo "{{Resource -|Resource audience=Users -|Resource kind=Support -|Resource URL=$support_url -}}" >> "wiki/$id.wiki" - - fi - - - echo "|- -| [[$name]] -| $short_description -| [[License:$license|$license]] -| $average_daily_users -| $bayesian_average__simple -| $version_date" >> "wiki/IceCat WebExtensions (proposed).wiki" - - - - # Begin template - fi - - ((line++)) - - done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt - # End template - - echo "|}" >> "wiki/IceCat WebExtensions (proposed).wiki" - - ;; - - --download-free-webextensions) - - cd build || exit - rm -fr free_webextensions - mkdir free_webextensions - cd free_webextensions || exit - - - - # Begin template - unset file - while IFS= read -r table; do - - verify_variables "$1" - - if [ "$approved_program" = true ]; then - - # End template - - id=$(echo $table | awk '{print $2}'); - file=$(echo $table | awk '{print $3}'); - slug=$(echo $table | awk '{print $4}'); - - # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). - wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" - - - - # Begin template - fi - - ((line++)) - - done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt - # End template - - ;; - --verify-license-copy) - - cd build || exit - rm -fr VERIFY-LICENSE-COPY.txt - cd free_webextensions || exit - - - - # Begin template - unset file - while IFS= read -r table; do - - verify_variables "$1" - - if [ "$approved_program" = true ]; then - # End template - - id=$(echo $table | awk '{print $2}'); - file=$(echo $table | awk '{print $3}'); - slug=$(echo $table | awk '{print $4}'); - - echo "-------------------------- -$name ($id): $license"; - unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING" - unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.txt" - unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.md" - unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE" - unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.txt" - unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.md" - - # Begin template - fi - - ((line++)) - - done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt - # End template - - - ;; - --verify-license-copy-old) - - - while IFS= read -r id; do - - - - - if [ ! -f "addon-$id-latest.xpi" ]; then - - echo "Nonfree" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE$"; then - - echo "LICENSE" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.txt$"; then - - echo "LICENSE.txt" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.md$"; then - - echo "LICENSE.md" >> ../VERIFY-LICENSE-COPY.txt - - - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING$"; then - - echo "COPYING" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.txt$"; then - - echo "COPYING.txt" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.md$"; then - - echo "COPYING.md" >> ../VERIFY-LICENSE-COPY.txt - - else - - echo "Request" >> ../VERIFY-LICENSE-COPY.txt - - fi - - done < ../id.txt - - - - line="0"; - - while IFS= read -r status; do - - - ((line++)) - - average_daily_users="$(sed -n ${line}p ../average_daily_users.txt)"; - id="$(sed -n ${line}p ../id.txt)"; - slug="$(sed -n ${line}p ../slug.txt)"; - license_name_en_US="$(sed -n ${line}p ../license.name.enUS.txt)"; - homepage_en_US="$(sed -n ${line}p ../homepage.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; - support_email_en_US="$(sed -n ${line}p ../support_email.en-US.txt)"; - support_url_en_US="$(sed -n ${line}p ../support_url.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; - - - - if [ "$license_name_en_US" == "GNU General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/gpl-3.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU General Public License, version 2.0" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/lgpl-3.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 2.1" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "Mozilla Public License, version 2.0" ]; then license_txt="https://www.mozilla.org/media/MPL/2.0/index.txt"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "Mozilla Public License Version 1.1" ]; then license_txt="https://www.mozilla.org/media/MPL/1.1/index.txt"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "BSD License" ]; then license_txt="https://directory.fsf.org/wiki?title=License:FreeBSD"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "MIT/X11 License" ]; then license_txt="https://directory.fsf.org/wiki/License:X11"; license_file_name="LICENSE"; - fi - - - if [ "$(sed -n ${line}p ../name.en-US.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-US.txt)"; - elif [ "$(sed -n ${line}p ../name.en-GB.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-GB.txt)"; - else name="$slug"; - fi - - - if [ "$status" = "Request" ]; then - - ((count_reports++)) - - # Authors that don't provide contact information will be ignored, its impossible to track conversations in Reviews anyway, because they will be deleted. - - if [ "$homepage_en_US" != "null" ] || [ "$support_url_en_US" != "null" ] || [ "$support_email_en_US" != "null" ]; then - - decimals="$(echo "$average_daily_users" | sed -r ':L;s=\b([0-9]+)([0-9]{3})\b=\1,\2=g;t L')"; - - echo " -$decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/addon/$slug/ - Request: \"You have distributed $name under $license_name_en_US on https://addons.mozilla.org/en-US/firefox/addon/$slug/, but you have not added a copy of the license. can you please download the text of the license from $license_txt and add save it to file name $license_file_name and put it in the root directory of you program?\" by contacting:"; - - if [ "$homepage_en_US" != "null" ]; then echo " * Homepage (please submit Pull Request for $license_file_name if possible): $homepage_en_US"; fi - if [ "$support_url_en_US" != "null" ]; then echo " * Support URL (please submit Pull Request for $license_file_name if possible): $support_url_en_US"; fi - if [ "$support_email_en_US" != "null" ]; then echo " * Support E-mail: $support_email_en_US"; fi - - fi - - fi - - done < ../VERIFY-LICENSE-COPY.txt - - echo " -######################### -$count_reports reports to submit"; - - ;; - - --make-xml) - # Deprecated. - - - # XML pages look like this: - #... - # <sha1>tlybh1k9mnxz92kccg46hqv9jvyrkp0</sha1> - # </revision> - # </page> - #</mediawiki> - # - # However. It's not possible to insert the checsum in <sha1> for the page since it will break the checksum itself. - - # Get number of edits: - revid="$(wget -qO- "https://directory.fsf.org/w/api.php?action=query&list=recentchanges&rclimit=1&format=json" | jq .query.recentchanges[0].revid)"; - xml_header="$(wget -qO- "https://directory.fsf.org/wiki/Special:Export/Main_Page" | head -n 42)"; - - ;; - - --all-recommended) - $0 --fresh-build "$2" - $0 --make-search-list "$2" - $0 --merge-lists "$2" - $0 --get-licenses "$2" - $0 --merged-free "$2" - $0 --make-wiki "$2" - ;; - - -esac diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-anti-features b/addons.mozilla.org-fsd/get-data/license-reporter-anti-features deleted file mode 100644 index 30e8a41..0000000 --- a/addons.mozilla.org-fsd/get-data/license-reporter-anti-features +++ /dev/null @@ -1,8 +0,0 @@ -https://directory.fsf.org/wiki/Free_Software_Directory:Antifeatures#Adware -Adblock Plus - -https://directory.fsf.org/wiki/Free_Software_Directory:Antifeatures#Blacklisted -Blacklist/Click&Clean Blacklist/Click&Clean -Blacklist/Looking Glass -Blacklist/Web of Trust - WOT - diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-collections b/addons.mozilla.org-fsd/get-data/license-reporter-collections deleted file mode 100644 index 1d58d53..0000000 --- a/addons.mozilla.org-fsd/get-data/license-reporter-collections +++ /dev/null @@ -1,2 +0,0 @@ -https://addons.mozilla.org/en-US/firefox/collections/NateN1222/librejs-compatibility-add-ons/ -https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/ diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-custom b/addons.mozilla.org-fsd/get-data/license-reporter-custom deleted file mode 100644 index dd185b0..0000000 --- a/addons.mozilla.org-fsd/get-data/license-reporter-custom +++ /dev/null @@ -1,51 +0,0 @@ -https://addons.mozilla.org/en-US/firefox/addon/bookmarks-tab/ -https://addons.mozilla.org/en-US/firefox/addon/capture-reverse-image-search/ -https://addons.mozilla.org/en-US/firefox/addon/country-flags-ip-whois/ -https://addons.mozilla.org/en-US/firefox/addon/dynamichistory/ -https://addons.mozilla.org/en-US/firefox/addon/foxytab/ -https://addons.mozilla.org/en-US/firefox/addon/fraudscore/ -https://addons.mozilla.org/en-US/firefox/addon/ghosttext/ -https://addons.mozilla.org/en-US/firefox/addon/laboratory-by-mozilla/ -https://addons.mozilla.org/en-US/firefox/addon/lesspass/ -https://addons.mozilla.org/en-US/firefox/addon/mute-all-inactive-tabs/ -https://addons.mozilla.org/en-US/firefox/addon/nemid-nøglefilsprogram/ -https://addons.mozilla.org/en-US/firefox/addon/noflash/ -https://addons.mozilla.org/en-US/firefox/addon/org-capture/ -https://addons.mozilla.org/en-US/firefox/addon/porn-site-blocker/ -https://addons.mozilla.org/en-US/firefox/addon/print-edit-we/ -https://addons.mozilla.org/en-US/firefox/addon/save-all-images-webextension/ -https://addons.mozilla.org/en-US/firefox/addon/sky-timer/ -https://addons.mozilla.org/en-US/firefox/addon/smart-https-revived/ -https://addons.mozilla.org/en-US/firefox/addon/speed-dial-lite/ -https://addons.mozilla.org/en-US/firefox/addon/stop-autoplay-next-for-youtube/ -https://addons.mozilla.org/en-US/firefox/addon/tab-notifier/ -https://addons.mozilla.org/en-US/firefox/addon/tab-reloader/ -https://addons.mozilla.org/en-US/firefox/addon/textern/ -https://addons.mozilla.org/en-US/firefox/addon/tile-tabs-we/ -https://addons.mozilla.org/en-US/firefox/addon/tomato-clock/ -https://addons.mozilla.org/en-US/firefox/addon/ubo-scope/ -https://addons.mozilla.org/en-US/firefox/addon/user-agent-switcher-revived/ -https://addons.mozilla.org/en-US/firefox/addon/violentmonkey/ -https://addons.mozilla.org/en-US/firefox/addon/webrtc-control/ -https://addons.mozilla.org/en-US/firefox/addon/wikipedia-peek/ -https://addons.mozilla.org/en-US/firefox/addon/withexeditor/ -https://addons.mozilla.org/en-US/firefox/addon/yass-we/ -https://addons.mozilla.org/en-US/firefox/addon/youtube-dark-mode/ -https://addons.mozilla.org/en-US/firefox/addon/youtube-hd-1/ -https://addons.mozilla.org/en-US/firefox/addon/zoom-page-we/ -https://addons.mozilla.org/en-US/firefox/addon/canvasblocker/ -https://addons.mozilla.org/en-US/firefox/addon/a-cookie-manager/ -https://addons.mozilla.org/en-US/firefox/addon/javascript-warning/ -https://addons.mozilla.org/en-US/firefox/addon/modify-header-value/ -https://addons.mozilla.org/en-US/firefox/addon/webrtc-control/ -https://addons.mozilla.org/en-US/firefox/addon/flash-debugger/ -https://addons.mozilla.org/en-US/firefox/addon/real-nano-defender/ -https://addons.mozilla.org/en-US/firefox/addon/bookmarks-manager-and-viewer/ -https://addons.mozilla.org/en-US/firefox/addon/simple-tab-groups/ -https://addons.mozilla.org/en-US/firefox/addon/try-xpath/ -https://addons.mozilla.org/en-US/firefox/addon/save-image-2-downloads/ -https://addons.mozilla.org/en-US/firefox/addon/history-cleaner/ -https://addons.mozilla.org/en-US/firefox/addon/copy-all-links/ -https://addons.mozilla.org/en-US/firefox/addon/new-hackbar/ -https://addons.mozilla.org/en-US/firefox/addon/new-tab-from-location-bar/ -https://addons.mozilla.org/en-US/firefox/addon/simple-form-fill/ diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-custom~ b/addons.mozilla.org-fsd/get-data/license-reporter-custom~ deleted file mode 100644 index 6927339..0000000 --- a/addons.mozilla.org-fsd/get-data/license-reporter-custom~ +++ /dev/null @@ -1,53 +0,0 @@ -https://addons.mozilla.org/en-US/firefox/addon/bookmarks-tab/ -https://addons.mozilla.org/en-US/firefox/addon/capture-reverse-image-search/ -https://addons.mozilla.org/en-US/firefox/addon/country-flags-ip-whois/ -https://addons.mozilla.org/en-US/firefox/addon/dynamichistory/ -https://addons.mozilla.org/en-US/firefox/addon/foxytab/ -https://addons.mozilla.org/en-US/firefox/addon/fraudscore/ -https://addons.mozilla.org/en-US/firefox/addon/ghosttext/ -https://addons.mozilla.org/en-US/firefox/addon/laboratory-by-mozilla/ -https://addons.mozilla.org/en-US/firefox/addon/lesspass/ -https://addons.mozilla.org/en-US/firefox/addon/mute-all-inactive-tabs/ -https://addons.mozilla.org/en-US/firefox/addon/nemid-nøglefilsprogram/ -https://addons.mozilla.org/en-US/firefox/addon/noflash/ -https://addons.mozilla.org/en-US/firefox/addon/org-capture/ -https://addons.mozilla.org/en-US/firefox/addon/porn-site-blocker/ -https://addons.mozilla.org/en-US/firefox/addon/print-edit-we/ -https://addons.mozilla.org/en-US/firefox/addon/save-all-images-webextension/ -https://addons.mozilla.org/en-US/firefox/addon/sky-timer/ -https://addons.mozilla.org/en-US/firefox/addon/smart-https-revived/ -https://addons.mozilla.org/en-US/firefox/addon/speed-dial-lite/ -https://addons.mozilla.org/en-US/firefox/addon/stop-autoplay-next-for-youtube/ -https://addons.mozilla.org/en-US/firefox/addon/tab-notifier/ -https://addons.mozilla.org/en-US/firefox/addon/tab-reloader/ -https://addons.mozilla.org/en-US/firefox/addon/textern/ -https://addons.mozilla.org/en-US/firefox/addon/tile-tabs-we/ -https://addons.mozilla.org/en-US/firefox/addon/tomato-clock/ -https://addons.mozilla.org/en-US/firefox/addon/ubo-scope/ -https://addons.mozilla.org/en-US/firefox/addon/user-agent-switcher-revived/ -https://addons.mozilla.org/en-US/firefox/addon/violentmonkey/ -https://addons.mozilla.org/en-US/firefox/addon/webrtc-control/ -https://addons.mozilla.org/en-US/firefox/addon/wikipedia-peek/ -https://addons.mozilla.org/en-US/firefox/addon/withexeditor/ -https://addons.mozilla.org/en-US/firefox/addon/yass-we/ -https://addons.mozilla.org/en-US/firefox/addon/youtube-dark-mode/ -https://addons.mozilla.org/en-US/firefox/addon/youtube-hd-1/ -https://addons.mozilla.org/en-US/firefox/addon/zoom-page-we/ - - -https://addons.mozilla.org/en-US/firefox/addon/canvasblocker/ -https://addons.mozilla.org/en-US/firefox/addon/a-cookie-manager/ -https://addons.mozilla.org/en-US/firefox/addon/javascript-warning/ -https://addons.mozilla.org/en-US/firefox/addon/modify-header-value/ -https://addons.mozilla.org/en-US/firefox/addon/webrtc-control/ -https://addons.mozilla.org/en-US/firefox/addon/flash-debugger/ -https://addons.mozilla.org/en-US/firefox/addon/real-nano-defender/ -https://addons.mozilla.org/en-US/firefox/addon/bookmarks-manager-and-viewer/ -https://addons.mozilla.org/en-US/firefox/addon/simple-tab-groups/ -https://addons.mozilla.org/en-US/firefox/addon/try-xpath/ -https://addons.mozilla.org/en-US/firefox/addon/save-image-2-downloads/ -https://addons.mozilla.org/en-US/firefox/addon/history-cleaner/ -https://addons.mozilla.org/en-US/firefox/addon/copy-all-links/ -https://addons.mozilla.org/en-US/firefox/addon/new-hackbar/ -https://addons.mozilla.org/en-US/firefox/addon/new-tab-from-location-bar/ - diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-log b/addons.mozilla.org-fsd/get-data/license-reporter-log deleted file mode 100644 index f8aada4..0000000 --- a/addons.mozilla.org-fsd/get-data/license-reporter-log +++ /dev/null @@ -1,5 +0,0 @@ -2017-09-19 04:00 Custom list Already in the main repository https://addons.mozilla.org/en-US/firefox/addon/ublock-origin/ -2017-09-19 04:00 Custom list Already in the main repository https://addons.mozilla.org/en-US/firefox/addon/search-image/ -2017-10-14 07:55 Custom list Already in the main repository https://addons.mozilla.org/en-US/firefox/addon/print-edit-we/ -2017-10-14 07:55 Custom list Already in the main repository https://addons.mozilla.org/en-US/firefox/addon/tile-tabs-we/ -2017-10-14 07:55 Custom list Already in the main repository https://addons.mozilla.org/en-US/firefox/addon/zoom-page-we/ diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-log~ b/addons.mozilla.org-fsd/get-data/license-reporter-log~ deleted file mode 100644 index 7c65a97..0000000 --- a/addons.mozilla.org-fsd/get-data/license-reporter-log~ +++ /dev/null @@ -1,2 +0,0 @@ -2017-09-19 03:54 Custom list Link rot https://addons.mozilla.org/en-US/firefox/addon/dynamichistory/ -2017-09-19 03:56 Custom list Already in the main repository https://addons.mozilla.org/en-US/firefox/addon// diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-repositories b/addons.mozilla.org-fsd/get-data/license-reporter-repositories deleted file mode 100644 index 9e75120..0000000 --- a/addons.mozilla.org-fsd/get-data/license-reporter-repositories +++ /dev/null @@ -1,2 +0,0 @@ -https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/ -https://addons.mozilla.org/en-US/firefox/collections/mozilla/ diff --git a/addons.mozilla.org-fsd/get-data/src/download-custom b/addons.mozilla.org-fsd/get-data/src/download-custom deleted file mode 100755 index b6c5def..0000000 --- a/addons.mozilla.org-fsd/get-data/src/download-custom +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -# This file is part of license-reporter -# Copyright (C) 2017 David Hedlund -# -# license-reporter is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# license-reporter is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -cd build || exit -mkdir json - -cp -a "$RUN_PATH/license-reporter-custom" custom_addons.txt - -# Sanitize a copy from empty lines, they use to occur at the end of the file -sed -i '/^\s*$/d' license-reporter-custom - - -sed -i "s|https://addons.mozilla.org/en-US/firefox/addon/||g; s|/||" custom_addons.txt - -rm -fr \ - custom_addons-json-source.txt \ - custom_addons-average_daily_users.txt \ - custom_addons-id.txt custom_addons-slug.txt \ - custom_addons-merged.txt \ - custom_addons-report-expired_links.txt \ - custom_addons-merged-sorted.txt - -cd json || exit - -lines="1"; - -while IFS= read -r slug; do - - - if [ "$debug" == false ] || ( [ "$debug" == true ] && [ "$lines" -le 3 ] ); then - - wget "https://addons.mozilla.org/api/v3/addons/addon/$slug/" -O "$slug.json"; - [ -s "$slug.json" ] || rm -f "$slug.json" # Delete empty files saved by wget, tis is much faster than using wget options to first validate the links. - - # Ignore expired URLs - if [ -f "$slug.json" ]; then - - id="$(jq ".id" "$slug.json")"; - mv "$slug.json" "$id.json" - - if [ "$(jq ".average_daily_users" "$id.json")" -gt "$minimum_average_daily_users" ]; then - - echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tAlready in the main repository\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$RUN_PATH/license-reporter-log" - - else - - jq ".average_daily_users" "$id.json" >> ../custom_addons-average_daily_users.txt - - # We need the JSON source because they are different for search indexed add-ons (they are store in index.html*) - jq ".id" "$id.json" >> ../custom_addons-id.txt - echo "$(jq ".id" "$id.json").json" >> ../custom_addons-json-source.txt - - ((lines++)) - - - fi - - - else - - echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tLink rot\t\t\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$RUN_PATH/license-reporter-log" - - fi - - fi - -done < ../custom_addons.txt - -# Sort by average_daily_users -paste ../custom_addons-average_daily_users.txt ../custom_addons-id.txt ../custom_addons-json-source.txt > ../custom_addons-merged.txt -sort -k1,1nr ../custom_addons-merged.txt > ../merged-custom.txt diff --git a/addons.mozilla.org-fsd/get-data/src/download-custom~ b/addons.mozilla.org-fsd/get-data/src/download-custom~ deleted file mode 100755 index 4af8ce1..0000000 --- a/addons.mozilla.org-fsd/get-data/src/download-custom~ +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -# This file is part of license-reporter -# Copyright (C) 2017 David Hedlund -# -# license-reporter is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# license-reporter is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -cd build || exit -mkdir json - -cp -a "$RUN_PATH/license-reporter-custom" custom_addons.txt - -# Sanitize a copy from empty lines, they use to occur at the end of the file -sed -i '/^\s*$/d' license-reporter-custom - - -sed -i "s|https://addons.mozilla.org/en-US/firefox/addon/||g; s|/||" custom_addons.txt - -rm -fr \ - custom_addons-json-source.txt \ - custom_addons-average_daily_users.txt \ - custom_addons-id.txt custom_addons-slug.txt \ - custom_addons-merged.txt \ - custom_addons-report-expired_links.txt \ - custom_addons-merged-sorted.txt - -cd json || exit - -lines="1"; - -while IFS= read -r slug; do - - - if [ "$debug" == false ] || [ "$debug" == true ] && [ "$lines" -le 3 ]; then - - wget "https://addons.mozilla.org/api/v3/addons/addon/$slug/" -O "$slug.json"; - [ -s "$slug.json" ] || rm -f "$slug.json" # Delete empty files saved by wget, tis is much faster than using wget options to first validate the links. - - # Ignore expired URLs - if [ -f "$slug.json" ]; then - - id="$(jq ".id" "$slug.json")"; - mv "$slug.json" "$id.json" - - if [ "$(jq ".average_daily_users" "$id.json")" -gt "$minimum_average_daily_users" ]; then - - echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tAlready in the main repository\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$RUN_PATH/license-reporter-log" - - else - - jq ".average_daily_users" "$id.json" >> ../custom_addons-average_daily_users.txt - - # We need the JSON source because they are different for search indexed add-ons (they are store in index.html*) - jq ".id" "$id.json" >> ../custom_addons-id.txt - echo "$(jq ".id" "$id.json").json" >> ../custom_addons-json-source.txt - - ((lines++)) - - - fi - - - else - - echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tLink rot\t\t\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$RUN_PATH/license-reporter-log" - - fi - - fi - -done < ../custom_addons.txt - -# Sort by average_daily_users -paste ../custom_addons-average_daily_users.txt ../custom_addons-id.txt ../custom_addons-json-source.txt > ../custom_addons-merged.txt -sort -k1,1nr ../custom_addons-merged.txt > ../merged-custom.txt |