diff options
57 files changed, 2976 insertions, 987 deletions
diff --git a/addons.mozilla.org-fsd/get-data/license-reporter b/addons.mozilla.org-fsd/get-data/license-reporter deleted file mode 100755 index 3e01334..0000000 --- a/addons.mozilla.org-fsd/get-data/license-reporter +++ /dev/null @@ -1,964 +0,0 @@ -#!/usr/bin/env bash -# This file is part of license-reporter -# Copyright (C) 2017, 2018 David Hedlund -# -# license-reporter is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# license-reporter is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -source err - -#if [ "$data" == "search" ]; then search_page_variable='.results[]'; fi - - -# Specifically for titles: en-US is sometimes unused, en-GB used to be used then. -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.name.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-GB|enGB|g" "$json_file" | jq "$search_page_variable.name.enGB" | sed "s|^\"||; s|\"$||;" - -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.homepage.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_email.enUS" | sed "s|^\"||; s|\"$||;" -# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_url.enUS" | sed "s|^\"||; s|\"$||;" - - -# Licenses -# sed "s|en-US|enUS|g" "$slug.json" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;" - -if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi - -minimum_average_daily_users="10000"; - -scriptsrc=$(readlink -f -- "${BASH_SOURCE[0]}") -SCRIPTSRC=$(readlink -f "$0" || echo "$0") -RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .) -filename=$(basename "$0"); -export -p SCRIPTSRC RUN_PATH minimum_average_daily_users debug - -if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi -if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi - -if [[ $exit == true ]]; then exit 1; fi - - -function check_files { - - if [ ! -f "$RUN_PATH/license-reporter-$check_type" ]; then - - echo "$RUN_PATH/license-reporter-$check_type doesn't exist."; - echo "Exiting" && exit - - else - - - # Make sure all links are using SSL - if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-$check_type; then - - echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)." - echo "Exiting" && exit - - fi - - fi - -} - -function line_status { - - - if [ "$file" == "$previous_file" ]; then - - ((line++)) - - else - - line="0"; # Should not be 1 - - fi - -} - - -function verify_variables { - - if [ -z ${file+x} ] || [ "$file" != "$(echo $table | awk '{print $3}')" ]; then - - line="0"; # should not be 1 - - fi - - average_daily_users=$(echo $table | awk '{print $1}'); - id=$(echo $table | awk '{print $2}'); - file=$(echo $table | awk '{print $3}'); - slug=$(echo $table | awk '{print $4}'); - name=$(echo "$table" | cut -f 5); - has_eula=$(echo "$table" | cut -f 6); - - - - firefox_max_version="$(jq .results[$line].current_version.compatibility.firefox.max $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|.\*||")"; - nativeMessaging="$(jq .results[$line].current_version.files[0].permissions $RUN_PATH/build/json/$file | grep "nativeMessaging")"; - - # Values are either an integer or "*" - re='^[0-9]+$' - if [[ $firefox_max_version =~ $re ]] && [ "$firefox_max_version" -ge 60 ] || [ "$firefox_max_version" == '*' ]; then - - # echo "$firefox_max_version" - firefox_max_version__pass=true; - - else - - unset firefox_max_version__pass - - fi - - if [ ! -n "$nativeMessaging" ]; then - - nativeMessaging__pass=true; - - else - - nativeMessaging__pass=false; - - fi - - echo "$nativeMessaging__pass" - - - # curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging - - - # API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668 - # The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies. - - # Old solution: - - # 1: Add this to license-reporter-dependencies: - #belgium-eid eid-mw - - # 2: Uncomment this code: - # depends_on_external_program=false; - # while IFS= read -r dependencies; do - # - # dependency_slug=$(echo $dependencies | awk '{print $1}'); - # dependency=$(echo "$dependencies" | cut -f 2); - # - # if [ "$dependency_slug" == "$slug" ]; then - # - # depends_on_external_program=true; - # - # fi - # - # done < $RUN_PATH/license-reporter-dependencies - - - - - approved_program=false; - if [ "$has_eula" == "eulafree" ] && [ "$firefox_max_version__pass" = true ] && [ "$nativeMessaging__pass" = true ]; then - - if [ "$1" == "--get-licenses" ]; then - - approved_program=true; - - else - - license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - - - - # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". - # See https://addons.mozilla.org/en-US/developers/addon/<your add-on>/ownership for the current add-on list. Note that not all licenses are added to the list. - # DuckDuckGo Privacy Essentials is distributed under Apache License 2.0. - # Apache License, version 2.0 is not on the add-on license list - https://github.com/mozilla/addons-server/issues/8545 - if - [ "$license" == "Apache License, Version 2.0" ] || # Custom license - [ "$license" == "Apache License, version 2.0" ] || # Custom license - [ "$license" == "BSD License" ] || - [ "$license" == "GNU General Public License, version 3.0" ] || - [ "$license" == "GNU General Public License, version 2.0" ] || - [ "$license" == "GNU Lesser General Public License, version 3.0" ] || - [ "$license" == "GNU Lesser General Public License, version 2.1" ] || - [ "$license" == "MIT/X11 License" ] || - [ "$license" == "Mozilla Public License, version 2.0" ] || - [ "$license" == "Mozilla Public License Version 1.1" ] - then - - # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files. - freedom_status="free"; - - elif [ "$license" == "All Rights Reserved" ]; then - - freedom_status="nonfree"; - - else - - freedom_status="unknown"; - fi - - [ "$freedom_status" == "free" ] - - fi - - fi - -} - -case "$1" in - - ""|-help) - - [ "$1" = "" ] && echo "Usage: $filename [--option] [--debug] - -OPTIONS - $0 --fresh-build - Remove the build directory and create a new one. - $0 --make-search-list - Downloads the most popular WebExtensions from https://addons.mozilla.org/en-US/firefox/search/... - $0 --make-repository-list - Required file: license-reporter-collections - File content syntax: https://addons.mozilla.org/en-US/firefox/collections/<user>/ - $0 --make-collection-list - Required file: license-reporter-collections - File content syntax: https://addons.mozilla.org/en-US/firefox/collections/<user>/<collection name> - $0 --make-custom-list - Required file: license-reporter-custom - File content syntax: https://addons.mozilla.org/en-US/firefox/addon/<add-on name>/ - $0 --merge-lists - Merge the lists to MERGED-ALL.txt - $0 --get-licenses - The licenses are not added to the general JSON file. This option will download the neccesary license JSON files for all add-ons. - $0 --download-licenses-json - $0 --download-free-webextensions - Downloads the latest version of the free WebExtensions. - $0 --verify-license-copy - Search for license files in the root directory in the downloaded WebExtensions (.xpi files) - $0 --merged-free - Generates MERGED-ALL-freedom_status.txt - $0 --make-wiki - Generates .wiki files - - --all-recommended - $0 --fresh-build - $0 --make-search-list - $0 --merge-lists - $0 --get-licenses - $0 --merged-free - $0 --make-wiki - -" && exit 1 - - ;; - - --fresh-build) - - if [ -d "build" ]; then - - rm -fr $RUN_PATH/build - - fi - - mkdir -p $RUN_PATH/build - - ;; - - --make-search-list) - - - cd build/ || exit - mkdir json - cd json || exit - - # API documentation: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html - - - page="1" - - function foo() { - - # Do not evaluate the number of extensions seen in https://addons.mozilla.org/en-US/firefox/search/ in a Quantum based browser since that will hide legacy add-ons. - uri_query="page=$page&platform=linux&sort=users&type=extension"; - - - # Get the most popular WebExtensions - wget "https://addons.mozilla.org/api/v3/addons/search/?$uri_query" -O "index.html?$uri_query.json" - - ((page++)) - - } - - - foo - - - - if [ "$debug" == false ]; then - - while [ "$(jq ".results[].average_daily_users" "index.html?$uri_query.json" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do - - foo - - done - - fi - - - # Sort by file modification time stamp for the downloaded JSON files - find index*.json -type f -printf "%Tc %p\n" | sort -n | awk '{print $NF}' | while read -r file; do - - - - - line="0"; # Should not be 1 - - for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do - - - if [ "$(jq ".results[$line].has_eula" "$file")" == "false" ]; then - - has_eula="eulafree"; - - else - - has_eula="eula"; - - fi - - - # Clean titles - - name=$(jq .results[$line].name $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - - if [ "$name" == "null" ]; then - - name=$(jq .results[$line].name $RUN_PATH/build/json/$file | sed "s|en-GB|enGB|;" | jq .enGB | sed "s|^\"||; s|\"$||;"); - - fi - - - # Clean titles. Example of bad titles: "Ghostery – Privacy Ad Blocker" - - # - - # Hard to filter out "-" since many add-ons use it, see for example https://addons.mozilla.org/en-US/firefox/search/?platform=linux&q=1-click - # Flash Video Downloader - YouTube HD Download [4K] - - name=$(echo $name | sed ' -s/ — /\n/g; -s/ – /\n/g; -s/ - /\n/g; # looks identical to the prevoius line but is unique -s/: /\n/g; -s/(/\n/g; -s/,/\n/g; -' | head -n 1 | sed 's/ $//'); - - - - - - - # Exclude add-ons with to low average daily users from index.html* - if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then - - echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file\t$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")\t$name\t$has_eula" >> ../merged-search.txt - - - fi - - ((line++)) - - done - - - done - - ;; - --make-repository-list) - - # It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs. - # This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync - # your addons.mozilla.org collection list with a separate text file. - - # https://github.com/mozilla/addons/issues/722 - # Cannot list collections in desired formats - # Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list). - # So we have to parse it from HTML. - - - # Get the links to the collections - - function get_repositories { - - check_type="repositories"; - check_files - - cd build/ || exit - - for i in $(cat ../license-reporter-repositories); do - - user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;"); - rm -f repository-$user - - - function wget_repository { - - match="^ <a href=\"/en-US/firefox/collections/" - wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user - - } - - # Figure out if there are several pages - number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||"); - - if [ -z "$number_of_pages" ]; then - - echo "$i doesn't have any sub pages"; - wget_repository - - else - - echo "$i has sub pages"; - - page_number=1; - - while [ $page_number -le $number_of_pages ] - do - - wget_repository__page_extension="?page=$page_number"; - echo "$wget_repository__page_extension" - wget_repository - page_number=$(( $page_number + 1 )) - - done - - fi - - done - } - get_repositories - - - # Download the JSON files from the collections - - cd .. - for x in $(find build/ -name repository* | sort); do - - for i in $(cat $x); do - - collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); - wget $i"format%3Ajson" -O "build/collection-$collection_name.json" - - done - - done - - ;; - - --make-collection-list) - - check_type="collections"; - check_files - - cd build/ || exit - for i in $(cat ../license-reporter-collections); do - - # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson - collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); - wget $i"format%3Ajson" -O "collection-$collection_name.json" - - done - - ;; - - --make-custom-list) - - check_type="custom"; - check_files - - src/download-custom - - ;; - - --merge-lists) - - cd build || exit - - cat merged-search.txt >> MERGED-ALL.txt - cat merged-custom.txt >> MERGED-ALL.txt - - ;; - --get-licenses) - - # API: Please adopt SPDX - https://github.com/mozilla/addons-server/issues/8706 - # Allow GNU [L]GPL x.x or later - https://github.com/mozilla/addons-server/issues/8707 - - cd build/ || exit - mkdir -p json/current_versions - cd json/current_versions || exit - - line="0"; # Should not be 1 - all_lines="1" # Should not be 0 - - # Begin template - unset file - while IFS= read -r table; do - - verify_variables "$1" - - if [ "$approved_program" = true ]; then - # End template - - echo "ddd"; - - id=$(echo $table | awk '{print $2}'); - file="$RUN_PATH/build/json/"$(echo $table | awk '{print $3}'); - has_eula=$(echo "$table" | cut -f 6); - - # line_status - - total=$(wc -l < $RUN_PATH/build/MERGED-ALL.txt) - echo "####################################### -Entry: $all_lines (approved) / $total" - - - # echo $(jq ".results[$line].slug" "$file"); - - echo "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" - - - # wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "$id.json" - - previous_file="$file"; - ((all_lines++)) - - - # Begin template - fi - - ((line++)) - - done < $RUN_PATH/build/MERGED-ALL.txt - # End template - - ;; - --make-wiki) - - cd build/ || exit - if [ ! -d "wiki" ]; then mkdir wiki; fi - - echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\" -! WebExtension -! Description -! Claimed license -! Users -! Rating -! Updated" > "wiki/IceCat WebExtensions (proposed).wiki" - - - # Begin template - unset file - while IFS= read -r table; do - - verify_variables "$1" - - if [ "$approved_program" = true ]; then - # End template - - #################################### - # Build the core wiki pages - - average_daily_users=$(echo $table | awk '{print $1}'); - id=$(echo $table | awk '{print $2}'); - file=$(echo $table | awk '{print $3}'); - slug=$(echo $table | awk '{print $4}'); - name=$(echo "$table" | cut -f 5); - has_eula=$(echo "$table" | cut -f 6); - freedom_status=$(echo "$table" | cut -f 7); - - license=$(jq .license.name $RUN_PATH/build/json/current_versions/$id.json | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - - - if [ "$license" == "Apache License, Version 2.0" ]; then license="Apache2.0"; - elif [ "$license" == "Apache License, version 2.0" ]; then license="Apache2.0"; - elif [ "$license" == "BSD License" ]; then license="BSD 2Clause"; - elif [ "$license" == "GNU General Public License, version 3.0" ]; then license="GPLv3"; - elif [ "$license" == "GNU General Public License, version 2.0" ]; then license="GPLv2"; - elif [ "$license" == "GNU Lesser General Public License, version 3.0" ]; then license="LGPLv3"; - elif [ "$license" == "GNU Lesser General Public License, version 2.1" ]; then license="LGPLv2.1"; - elif [ "$license" == "MIT/X11 License" ]; then license="X11"; - elif [ "$license" == "Mozilla Public License, version 2.0" ]; then license="MPLv2.0"; - elif [ "$license" == "Mozilla Public License Version 1.1" ]; then license="MPLv1.1"; - fi - - - - - short_description_raw=$(jq .results[$line].summary $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|\\\n|\n|g; s|\\\||g; -s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues -s|http|\nhttp|g; -"); - short_description_again=$(echo "$short_description_raw" | sed "/outgoing.prod.mozaws.net/d;"); - - short_description=$(echo "$short_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g'); - - full_description_raw=$(jq .results[$line].description $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||; - s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues -"); - full_description_again=$(echo "$full_description_raw" | sed "/outgoing.prod.mozaws.net/d;"); - - full_description=$(echo "$full_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g'); - - - - - homepage_url=$(jq .results[$line].homepage $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - version=$(jq .results[$line].current_version.version $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;"); - version_date=$(jq .results[$line].last_updated $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g"); - version_id=$(jq .results[$line].current_version.id $RUN_PATH/build/json/$file); - license_copyright=$(jq .results[$line].authors[].name $RUN_PATH/build/json/$file | sed "s|^\"||; s|\"$||;"); - bayesian_average=$(jq .results[$line].ratings.bayesian_average $RUN_PATH/build/json/$file); - bayesian_average__simple=$(printf "%.1f" "$(echo "$bayesian_average" | sed "s|\.|,|")"); - - last_review_date=$(date +"%Y/%m/%d"); - last_review_by="wikisysbot"; - submitted_by="wikisysbot"; - - if [[ "$(jq .results[$line].current_version.compatibility.seamonkey $RUN_PATH/build/json/$file)" != "null" ]]; then - set_seamonkey="Iceape,"; - fi - - if [[ "$(jq .results[$line].current_version.compatibility.firefox $RUN_PATH/build/json/$file)" != "null" ]]; then - set_icecat="IceCat,"; - fi - - # if [[ "$(jq .results[$line].current_version.compatibility.android $RUN_PATH/build/json/$file)" != "null" ]] || [[ "$(jq .results[$line].current_version.compatibility.mobile $RUN_PATH/build/json/$file)" != "null" ]]; then - # set_icecatmobile="IceCatMobile,"; - # fi - - if [[ "$(jq .results[$line].current_version.compatibility.thunderbird $RUN_PATH/build/json/$file)" != "null" ]]; then - set_thunderbird="Icedove,"; - fi - - # Do not list IceCatMobile - extension_of=$(echo "$set_seamonkey$set_icecat$set_thunderbird" | sed "s|,$||;"); - - - - github_true=$(echo "$homepage_url" | grep "github.com"); - - if [[ $github_true != "" ]]; then - - # Always use https, and remove anchors - homepage_url__for__vcs_checkout_command="https://github.com$(echo $github_true | sed "s|https://github.com||; s|http://github.com||; s|#|\n|;" | head -n 1)"; - # Remove trailing slash - homepage_url__for__vcs_checkout_command="${homepage_url__for__vcs_checkout_command%/}" - vcs_checkout_command="git clone $homepage_url__for__vcs_checkout_command.git"; - - fi - - if [ "$name" == "GNU LibreJS" ]; then - - is_gnu="Yes"; - - else - - is_gnu="No"; - - fi - - support_url=$(jq .results[$line].support_url $RUN_PATH/build/json/$file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); - - # List JSON structure: cat json/$file | js '.' - # XML value "homepage" is not always set in AMO API, but the XML require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically. - echo "{{Entry" > wiki/$id.wiki - - # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341 - # Avoid specific versions since it's not compatible with all IceCat versions: - # |Version download=https://addons.mozilla.org/firefox/downloads/latest/$id/addon-$version_id-latest.xpi - # |Version identifier=$version - # |Version date=$version_date - - - echo "|Name=$name -|Short description=$short_description -|Full description=$full_description -|Homepage URL=$homepage_url -|Extension of=$extension_of -|VCS checkout command=$vcs_checkout_command -|Version download=https://addons.mozilla.org/en-US/firefox/addon/$slug/versions/ -|Last review by=$last_review_by -|Last review date=$last_review_date -|Submitted by=$submitted_by -|Submitted date=2018/06/20 -|Is GNU=$is_gnu -}}" >> wiki/$id.wiki - - - - # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/ - echo "{{Project license -|License=$license -|License copyright=$license_copyright -|License note=\"License: $original_license\" listed -at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/ -}}" >> wiki/$id.wiki - - if [[ "$support_url" != "" ]]; - then - - echo "{{Resource -|Resource audience=Users -|Resource kind=Support -|Resource URL=$support_url -}}" >> "wiki/$id.wiki" - - fi - - - echo "|- -| [[$name]] -| $short_description -| [[License:$license|$license]] -| $average_daily_users -| $bayesian_average__simple -| $version_date" >> "wiki/IceCat WebExtensions (proposed).wiki" - - - - # Begin template - fi - - ((line++)) - - done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt - # End template - - echo "|}" >> "wiki/IceCat WebExtensions (proposed).wiki" - - ;; - - --download-free-webextensions) - - cd build || exit - rm -fr free_webextensions - mkdir free_webextensions - cd free_webextensions || exit - - - - # Begin template - unset file - while IFS= read -r table; do - - verify_variables "$1" - - if [ "$approved_program" = true ]; then - - # End template - - id=$(echo $table | awk '{print $2}'); - file=$(echo $table | awk '{print $3}'); - slug=$(echo $table | awk '{print $4}'); - - # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). - wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" - - - - # Begin template - fi - - ((line++)) - - done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt - # End template - - ;; - --verify-license-copy) - - cd build || exit - rm -fr VERIFY-LICENSE-COPY.txt - cd free_webextensions || exit - - - - # Begin template - unset file - while IFS= read -r table; do - - verify_variables "$1" - - if [ "$approved_program" = true ]; then - # End template - - id=$(echo $table | awk '{print $2}'); - file=$(echo $table | awk '{print $3}'); - slug=$(echo $table | awk '{print $4}'); - - echo "-------------------------- -$name ($id): $license"; - unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING" - unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.txt" - unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.md" - unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE" - unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.txt" - unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.md" - - # Begin template - fi - - ((line++)) - - done < $RUN_PATH/build/MERGED-ALL-freedom_status.txt - # End template - - - ;; - --verify-license-copy-old) - - - while IFS= read -r id; do - - - - - if [ ! -f "addon-$id-latest.xpi" ]; then - - echo "Nonfree" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE$"; then - - echo "LICENSE" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.txt$"; then - - echo "LICENSE.txt" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.md$"; then - - echo "LICENSE.md" >> ../VERIFY-LICENSE-COPY.txt - - - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING$"; then - - echo "COPYING" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.txt$"; then - - echo "COPYING.txt" >> ../VERIFY-LICENSE-COPY.txt - - elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.md$"; then - - echo "COPYING.md" >> ../VERIFY-LICENSE-COPY.txt - - else - - echo "Request" >> ../VERIFY-LICENSE-COPY.txt - - fi - - done < ../id.txt - - - - line="0"; - - while IFS= read -r status; do - - - ((line++)) - - average_daily_users="$(sed -n ${line}p ../average_daily_users.txt)"; - id="$(sed -n ${line}p ../id.txt)"; - slug="$(sed -n ${line}p ../slug.txt)"; - license_name_en_US="$(sed -n ${line}p ../license.name.enUS.txt)"; - homepage_en_US="$(sed -n ${line}p ../homepage.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; - support_email_en_US="$(sed -n ${line}p ../support_email.en-US.txt)"; - support_url_en_US="$(sed -n ${line}p ../support_url.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; - - - - if [ "$license_name_en_US" == "GNU General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/gpl-3.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU General Public License, version 2.0" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/lgpl-3.0.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 2.1" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt"; license_file_name="COPYING"; - elif [ "$license_name_en_US" == "Mozilla Public License, version 2.0" ]; then license_txt="https://www.mozilla.org/media/MPL/2.0/index.txt"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "Mozilla Public License Version 1.1" ]; then license_txt="https://www.mozilla.org/media/MPL/1.1/index.txt"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "BSD License" ]; then license_txt="https://directory.fsf.org/wiki?title=License:FreeBSD"; license_file_name="LICENSE"; - elif [ "$license_name_en_US" == "MIT/X11 License" ]; then license_txt="https://directory.fsf.org/wiki/License:X11"; license_file_name="LICENSE"; - fi - - - if [ "$(sed -n ${line}p ../name.en-US.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-US.txt)"; - elif [ "$(sed -n ${line}p ../name.en-GB.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-GB.txt)"; - else name="$slug"; - fi - - - if [ "$status" = "Request" ]; then - - ((count_reports++)) - - # Authors that don't provide contact information will be ignored, its impossible to track conversations in Reviews anyway, because they will be deleted. - - if [ "$homepage_en_US" != "null" ] || [ "$support_url_en_US" != "null" ] || [ "$support_email_en_US" != "null" ]; then - - decimals="$(echo "$average_daily_users" | sed -r ':L;s=\b([0-9]+)([0-9]{3})\b=\1,\2=g;t L')"; - - echo " -$decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/addon/$slug/ - Request: \"You have distributed $name under $license_name_en_US on https://addons.mozilla.org/en-US/firefox/addon/$slug/, but you have not added a copy of the license. can you please download the text of the license from $license_txt and add save it to file name $license_file_name and put it in the root directory of you program?\" by contacting:"; - - if [ "$homepage_en_US" != "null" ]; then echo " * Homepage (please submit Pull Request for $license_file_name if possible): $homepage_en_US"; fi - if [ "$support_url_en_US" != "null" ]; then echo " * Support URL (please submit Pull Request for $license_file_name if possible): $support_url_en_US"; fi - if [ "$support_email_en_US" != "null" ]; then echo " * Support E-mail: $support_email_en_US"; fi - - fi - - fi - - done < ../VERIFY-LICENSE-COPY.txt - - echo " -######################### -$count_reports reports to submit"; - - ;; - - --make-xml) - # Deprecated. - - - # XML pages look like this: - #... - # <sha1>tlybh1k9mnxz92kccg46hqv9jvyrkp0</sha1> - # </revision> - # </page> - #</mediawiki> - # - # However. It's not possible to insert the checsum in <sha1> for the page since it will break the checksum itself. - - # Get number of edits: - revid="$(wget -qO- "https://directory.fsf.org/w/api.php?action=query&list=recentchanges&rclimit=1&format=json" | jq .query.recentchanges[0].revid)"; - xml_header="$(wget -qO- "https://directory.fsf.org/wiki/Special:Export/Main_Page" | head -n 42)"; - - ;; - - --all-recommended) - $0 --fresh-build "$2" - $0 --make-search-list "$2" - $0 --merge-lists "$2" - $0 --get-licenses "$2" - $0 --merged-free "$2" - $0 --make-wiki "$2" - ;; - - -esac diff --git a/subprojects/freeamo/COPYING b/subprojects/freeamo/COPYING new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/subprojects/freeamo/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<https://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<https://www.gnu.org/licenses/why-not-lgpl.html>. diff --git a/addons.mozilla.org-fsd/README.md b/subprojects/freeamo/README.md index 5d2fb5a..208749d 100644 --- a/addons.mozilla.org-fsd/README.md +++ b/subprojects/freeamo/README.md @@ -55,6 +55,9 @@ looks. You can also use AMO to find add-ons for Mozilla Thunderbird and SeaMonkey." - https://addons.mozilla.org/en-US/about +API documentation: +https://addons-server.readthedocs.io/en/latest/topics/api/addons.html + ##################################################################### About WebExtensions diff --git a/subprojects/freeamo/freeamo b/subprojects/freeamo/freeamo new file mode 100755 index 0000000..dec2d5c --- /dev/null +++ b/subprojects/freeamo/freeamo @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2017, 2018 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +source freeamo.conf + +scriptsrc=$(readlink -f -- "${BASH_SOURCE[0]}") +run_path=$(dirname "${scriptsrc}" || echo .) +filename=$(basename "$0"); + +output_file_1="$run_path/build/index-search-pages.txt"; +output_file_2="$run_path/build/index-search-pages--free-licenses.txt" + +# source "$run_path/build/err" + +# Evaluate debug mode +if [ "$1" == "--debug" ] || [ "$2" == "--debug" ]; then + + debug=true + debug_arg="--debug"; + +else + + debug=false; + unset debug_arg; + +fi + +# Delete empty files to re-generate them -- Typical case scenario: 1) wget tried to download a file 2) The script is interupted 3) The script is run again +if [ -d "$run_path/build" ]; then + + find "$run_path/build" -type f -empty -delete + +fi + +export -p debug + +if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi +if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi + +if [[ $exit == true ]]; then exit 1; fi + +case "$1" in + + ""|--debug) + + $0 --disclaimer "$debug_arg" + $0 --remove-build "$debug_arg" + $0 --search-pages "$debug_arg" + $0 --get-licenses "$debug_arg" + $0 --make-directory.fsf.org-wiki "$debug_arg" + + ;; + + -help) + + [ "$1" = "" ] && echo "Usage: $filename [--option] [--debug] + +OPTIONS + $0 --disclaimer + Display which icecat version the add-on will be compatible with. + $0 --remove-build + Remove the build directory if it exists. + $0 --search-pages + Downloads the most popular WebExtensions from https://addons.mozilla.org/en-US/firefox/search/... + $0 --get-licenses + The licenses are not added to the general JSON file. This option will download the neccesary license JSON files for all add-ons. + $0 --make-directory.fsf.org-wiki + Generates .wiki files specifically for https://directory.fsf.org/wiki/ + +Disabled options due to limited development time + $0 --make-repository-list + Required file: FreeAMO-collections + File content syntax: https://addons.mozilla.org/en-US/firefox/collections/<user>/ + $0 --make-collection-list + Required file: FreeAMO-collections + File content syntax: https://addons.mozilla.org/en-US/firefox/collections/<user>/<collection name> + $0 --make-custom-list + Required file: FreeAMO-custom + File content syntax: https://addons.mozilla.org/en-US/firefox/addon/<add-on name>/ + $0 --download-licenses-json + $0 --download-free-webextensions + Downloads the latest version of the free WebExtensions. + $0 --verify-license-copy + Search for license files in the root directory in the downloaded WebExtensions (.xpi files) +" && exit 1 + + ;; + + --disclaimer) + + echo "$notes" + + ;; + + --remove-build) + + rm -fr "$run_path/build" + + ;; + + --search-pages) + + # This option is named after named after https://addons.mozilla.org/en-US/firefox/search/ + $0 --search-pages--download "$debug_arg" + $0 --search-pages--filter "$debug_arg" + + ;; + + --search-pages--download) + + mkdir -p "$run_path/build/json/search-pages" + source src/search-pages + search-pages--download + + ;; + + --search-pages--filter) + + mkdir -p "$run_path/build/json/search-pages" + source src/search-pages + search-pages--filter + + ;; + + --get-licenses) + + $0 --get-licenses--download "$debug_arg" + $0 --get-licenses--filter "$debug_arg" + + ;; + + --get-licenses--download) + + mkdir -p "$run_path/build/json/current_versions"; + source src/get-licenses + get-licenses--download + + ;; + + --get-licenses--filter) + + source src/get-licenses + get-licenses--filter + + ;; + + --make-directory.fsf.org-wiki) + + mkdir -p "$run_path/build/wiki"; + source src/make-wiki + + ;; + +esac diff --git a/subprojects/freeamo/freeamo.conf b/subprojects/freeamo/freeamo.conf new file mode 100644 index 0000000..fd133e7 --- /dev/null +++ b/subprojects/freeamo/freeamo.conf @@ -0,0 +1,13 @@ +# Mozilla API parameters: Add-ons -> Search +app="firefox" +appversion="$(wget -qO- "http://archive.trisquel.info/trisquel/pool/main/i/icecat/?C=N;O=D" | grep icecat_ | head -n 1 | sed "s|\"|\\n|g; s|-|\\n|g" | grep icecat_ | sed "s|icecat_||" | head -n 1)" # Latest icecat version for Trisquel +page_size=50 # page_size=50 -- this is the maximum accepted number, it makes the download much faster: 20 seconds for 10 x page_size=50, 35 seconds for 20 x page_size=25 (default) +platform="linux" +sort="users" # sort=users. sort=name not working yet - https://github.com/mozilla/addons-server/issues/7600 +type="extension" + +# Mozilla API parameters: Add-ons -> Detail +minimum_average_daily_users="10000" + +# Extra +notes="WebExtensions compatible with the latest icecat package version for the latest stable Trisquel GNU/Linux distribution. GNU IceCat is based on Firefox Extended Support Release (ESR)." diff --git a/subprojects/freeamo/freeamo.conf~ b/subprojects/freeamo/freeamo.conf~ new file mode 100644 index 0000000..51d4992 --- /dev/null +++ b/subprojects/freeamo/freeamo.conf~ @@ -0,0 +1,4 @@ +latest_trisquel_icecat_version="$(wget -qO- "http://archive.trisquel.info/trisquel/pool/main/i/icecat/?C=N;O=D" | grep icecat_ | head -n 1 | sed "s|\"|\\n|g; s|-|\\n|g" | grep icecat_ | sed "s|icecat_||" | head -n 1)" +firefox_compatibility="$latest_trisquel_icecat_version" +minimum_average_daily_users="10000" + diff --git a/subprojects/freeamo/freeamo~ b/subprojects/freeamo/freeamo~ new file mode 100755 index 0000000..bd28ff8 --- /dev/null +++ b/subprojects/freeamo/freeamo~ @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2017, 2018 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +if [ "$1" == "--debug" ] || [ "$2" == "--debug" ]; then + + debug=true + debug_arg="--debug"; + +else + + debug=false; + unset debug_arg; + +fi + +scriptsrc=$(readlink -f -- "${BASH_SOURCE[0]}") +run_path=$(dirname "${scriptsrc}" || echo .) +filename=$(basename "$0"); + +output_file_1="$run_path/build/index-search-pages.txt"; +output_file_2="$run_path/build/index-search-pages--free-licenses.txt" + +latest_trisquel_icecat_version="$(wget -qO- "http://archive.trisquel.info/trisquel/pool/main/i/icecat/?C=N;O=D" | grep icecat_ | head -n 1 | sed "s|\"|\\n|g; s|-|\\n|g" | grep icecat_ | sed "s|icecat_||" | head -n 1)" + +minimum_average_daily_users="10000" + +# source "$run_path/build/err" + +# Delete empty files to re-generate them -- Typical case scenario: 1) wget tried to download a file 2) The script is interupted 3) The script is run again +if [ -d "$run_path/build" ]; then + + find "$run_path/build" -type f -empty -delete + +fi + +export -p debug + +if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi +if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi + +if [[ $exit == true ]]; then exit 1; fi + +case "$1" in + + ""|--debug) + + $0 --disclaimer "$debug_arg" + $0 --remove-build "$debug_arg" + $0 --search-pages "$debug_arg" + $0 --get-licenses "$debug_arg" + $0 --make-directory.fsf.org-wiki "$debug_arg" + + ;; + + -help) + + [ "$1" = "" ] && echo "Usage: $filename [--option] [--debug] + +OPTIONS + $0 --disclaimer + Display which icecat version the add-on will be compatible with. + $0 --remove-build + Remove the build directory if it exists. + $0 --search-pages + Downloads the most popular WebExtensions from https://addons.mozilla.org/en-US/firefox/search/... + $0 --get-licenses + The licenses are not added to the general JSON file. This option will download the neccesary license JSON files for all add-ons. + $0 --make-directory.fsf.org-wiki + Generates .wiki files specifically for https://directory.fsf.org/wiki/ + +Disabled options due to limited development time + $0 --make-repository-list + Required file: FreeAMO-collections + File content syntax: https://addons.mozilla.org/en-US/firefox/collections/<user>/ + $0 --make-collection-list + Required file: FreeAMO-collections + File content syntax: https://addons.mozilla.org/en-US/firefox/collections/<user>/<collection name> + $0 --make-custom-list + Required file: FreeAMO-custom + File content syntax: https://addons.mozilla.org/en-US/firefox/addon/<add-on name>/ + $0 --download-licenses-json + $0 --download-free-webextensions + Downloads the latest version of the free WebExtensions. + $0 --verify-license-copy + Search for license files in the root directory in the downloaded WebExtensions (.xpi files) +" && exit 1 + + ;; + + --disclaimer) + + bold=$(tput bold) + normal=$(tput sgr0) + + echo "${bold}icecat-$latest_trisquel_icecat_version${normal} is the latest package version for Trisquel GNU/Linux. +This program will download meta-data that is compatible with that version." + + ;; + + --remove-build) + + rm -fr "$run_path/build" + + ;; + + --search-pages) + + # This option is named after named after https://addons.mozilla.org/en-US/firefox/search/ + $0 --search-pages--download "$debug_arg" + $0 --search-pages--filter "$debug_arg" + + ;; + + --search-pages--download) + + mkdir -p "$run_path/build/json/search-pages" + source src/search-pages + search-pages--download + + ;; + + --search-pages--filter) + + mkdir -p "$run_path/build/json/search-pages" + source src/search-pages + search-pages--filter + + ;; + + --get-licenses) + + $0 --get-licenses--download "$debug_arg" + $0 --get-licenses--filter "$debug_arg" + + ;; + + --get-licenses--download) + + mkdir -p "$run_path/build/json/current_versions"; + source src/get-licenses + get-licenses--download + + ;; + + --get-licenses--filter) + + source src/get-licenses + get-licenses--filter + + ;; + + --make-directory.fsf.org-wiki) + + mkdir -p "$run_path/build/wiki"; + source src/make-wiki + + ;; + +esac diff --git a/addons.mozilla.org-fsd/get-data/err b/subprojects/freeamo/src/err index 97b3cc3..97b3cc3 100755 --- a/addons.mozilla.org-fsd/get-data/err +++ b/subprojects/freeamo/src/err diff --git a/subprojects/freeamo/src/get-licenses b/subprojects/freeamo/src/get-licenses new file mode 100644 index 0000000..c7e5559 --- /dev/null +++ b/subprojects/freeamo/src/get-licenses @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2018 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# API: Please adopt SPDX - https://github.com/mozilla/addons-server/issues/8706 +# Allow GNU [L]GPL x.x or later - https://github.com/mozilla/addons-server/issues/8707 + +cd "build/json/search-pages" + +function get-licenses--download { + + echo -e "\\nDownloading license pages:" + + + output_file="$output_file_1"; + pv__size="$(wc -l < "$output_file")" + source "$run_path/src/main_functions" + + while IFS= read -r table; do + + ((++line)) + + if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then + + file=$(echo $table | awk '{print $1}'); + entry=$(echo $table | awk '{print $2}'); + + source "$run_path/src/main_functions" + amo_variables + + total=$(wc -l < "$output_file_1") + # echo -e "#================== + #Entry $line (of $total): $name\n" + + # Do now download the same license files over and over + wget -nc -q "https://addons.mozilla.org/api/v3/addons/addon/$slug/versions/$current_version__id/" -O "$run_path/build/json/current_versions/$current_version__id.json" + + fi + + + echo "foo" # Must be here to work with pv + done < "$output_file_1" | progress + +} + +function get-licenses--filter { + + echo -e "\\nFilter license pages:" + + output_file="$output_file_1"; + pv__size="$(wc -l < "$output_file")" + source "$run_path/src/main_functions" + rm -f "$run_path/build/index-search-pages--free-licenses.txt" + + + while IFS= read -r table; do + + ((++line)) + + if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then + + file=$(echo $table | awk '{print $1}'); + entry=$(echo $table | awk '{print $2}'); + + source "$run_path/src/main_functions" + amo_variables + amo_variables__licenses + + # license="$(jq ".license.name" "$run_path/build/json/current_versions/$current_version__id.json" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;")" + + # echo "$run_path/build/json/current_versions/$(jq ".results[$entry].current_version.id" "$file")" + + # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". + # Make a drop-down menu with all SPDX licenses - https://github.com/mozilla/addons-server/issues/8866 + # See https://addons.mozilla.org/en-US/developers/addon/<your add-on>/ownership for the current add-on list. Note that not all licenses are added to the list. + # DuckDuckGo Privacy Essentials is distributed under Apache License 2.0. + # Apache License, version 2.0 is not on the add-on license list - https://github.com/mozilla/addons-server/issues/8545 + if + [ "$license__amo_full_name" == "BSD License" ] || + [ "$license__amo_full_name" == "GNU General Public License, version 3.0" ] || + [ "$license__amo_full_name" == "GNU General Public License, version 2.0" ] || + [ "$license__amo_full_name" == "GNU Lesser General Public License, version 3.0" ] || + [ "$license__amo_full_name" == "GNU Lesser General Public License, version 2.1" ] || + [ "$license__amo_full_name" == "MIT/X11 License" ] || + [ "$license__amo_full_name" == "Mozilla Public License, version 2.0" ] || + [ "$license__amo_full_name" == "Mozilla Public License Version 1.1" ] + then + + # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files. + # echo "$name: $license__amo_full_name" + echo "$file $entry" >> "$output_file_2" + + freedom_status="free"; + + elif [ "$license__amo_full_name" == "All Rights Reserved" ]; then + + freedom_status="nonfree"; + + else + + freedom_status="unknown"; + fi + + + + fi + + echo "foo" # Must be here to work with pv + + done < "$output_file_1" | progress + +} diff --git a/subprojects/freeamo/src/get-licenses~ b/subprojects/freeamo/src/get-licenses~ new file mode 100644 index 0000000..0c1494b --- /dev/null +++ b/subprojects/freeamo/src/get-licenses~ @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2018 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# API: Please adopt SPDX - https://github.com/mozilla/addons-server/issues/8706 +# Allow GNU [L]GPL x.x or later - https://github.com/mozilla/addons-server/issues/8707 + +cd "build/json/search-pages" + +function get-licenses--download { + + echo -e "\\nDownloading license pages:" + + +output_file="$output_file_1"; +source "$run_path/src/main_functions" + + while IFS= read -r table; do + + ((++line)) + + if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then + + file=$(echo $table | awk '{print $1}'); + entry=$(echo $table | awk '{print $2}'); + + source "$run_path/src/main_functions" + amo_variables + + total=$(wc -l < "$output_file_1") +# echo -e "#================== +#Entry $line (of $total): $name\n" + + # Do now download the same license files over and over + wget -nc -q "https://addons.mozilla.org/api/v3/addons/addon/$slug/versions/$current_version__id/" -O "$run_path/build/json/current_versions/$current_version__id.json" + + fi + + + echo "foo" # Must be here to work with pv + done < "$output_file_1" | progress + +} + +function get-licenses--filter { + + echo -e "\\nFilter license pages:" + + output_file="$output_file_1"; + source "$run_path/src/main_functions" + rm -f "$run_path/build/index-search-pages--free-licenses.txt" + + + while IFS= read -r table; do + + ((++line)) + + if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then + + file=$(echo $table | awk '{print $1}'); + entry=$(echo $table | awk '{print $2}'); + + source "$run_path/src/main_functions" + amo_variables + amo_variables__licenses + +# license="$(jq ".license.name" "$run_path/build/json/current_versions/$current_version__id.json" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;")" + +# echo "$run_path/build/json/current_versions/$(jq ".results[$entry].current_version.id" "$file")" + + # This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License". + # Make a drop-down menu with all SPDX licenses - https://github.com/mozilla/addons-server/issues/8866 + # See https://addons.mozilla.org/en-US/developers/addon/<your add-on>/ownership for the current add-on list. Note that not all licenses are added to the list. + # DuckDuckGo Privacy Essentials is distributed under Apache License 2.0. + # Apache License, version 2.0 is not on the add-on license list - https://github.com/mozilla/addons-server/issues/8545 + if + [ "$license__amo_full_name" == "BSD License" ] || + [ "$license__amo_full_name" == "GNU General Public License, version 3.0" ] || + [ "$license__amo_full_name" == "GNU General Public License, version 2.0" ] || + [ "$license__amo_full_name" == "GNU Lesser General Public License, version 3.0" ] || + [ "$license__amo_full_name" == "GNU Lesser General Public License, version 2.1" ] || + [ "$license__amo_full_name" == "MIT/X11 License" ] || + [ "$license__amo_full_name" == "Mozilla Public License, version 2.0" ] || + [ "$license__amo_full_name" == "Mozilla Public License Version 1.1" ] + then + + # The nonfree add-ons should not be removed since the line number must correspond with the entry number in the search result JSON files. +# echo "$name: $license__amo_full_name" + echo "$file $entry" >> "$output_file_2" + + freedom_status="free"; + + elif [ "$license__amo_full_name" == "All Rights Reserved" ]; then + + freedom_status="nonfree"; + + else + + freedom_status="unknown"; + fi + + + + fi + + echo "foo" # Must be here to work with pv + + done < "$output_file_1" | progress + +} diff --git a/subprojects/freeamo/src/main_functions b/subprojects/freeamo/src/main_functions new file mode 100644 index 0000000..a96ada3 --- /dev/null +++ b/subprojects/freeamo/src/main_functions @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2018 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +function check_files { + + if [ ! -f "$run_path/FreeAMO-$check_type" ]; then + + echo "$run_path/FreeAMO-$check_type doesn't exist."; + echo "Exiting" && exit + + else + + + # Make sure all links are using SSL + if grep -q "http://addons.mozilla.org/" $run_path/FreeAMO-$check_type; then + + echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)." + echo "Exiting" && exit + + fi + + fi + +} + +function line_status { + + + if [ "$file" == "$previous_file" ]; then + + ((++line)) + + else + + line="0"; # Should not be 1 + + fi + +} + +function amo_variables() { + id="$(jq ".results[$entry].id" "$file")"; + average_daily_users="$(jq ".results[$entry].average_daily_users" "$file")"; + has_eula="$(jq ".results[$entry].has_eula" "$file")"; + slug="$(jq ".results[$entry].slug" "$file" | sed "s|^\"||; s|\"$||;")"; + current_version__id=$(jq ".results[$entry].current_version.id" "$file"); + + # Names + name__enUS=$(jq ".results[$entry].name" "$file" | sed "s|-||;" | jq .enUS | sed "s|^\"||; s|\"$||;" ) + name__enGB=$(jq ".results[$entry].name" "$file" | sed "s|-||;" | jq .enGB | sed "s|^\"||; s|\"$||;" ) + if [ "$name__enUS" != "null" ]; then name="$name__enUS"; + elif [ "$name__enGB" != "null" ]; then name="$name__enGB"; + else name=$(jq ".results[$entry].name[]" "$file" | sed "s|-||;" | jq . | sed "s|^\"||; s|\"$||;"); # Use the first avalible name if English is not avaliable + fi + + + if jq .results[$entry].current_version.files[0].permissions "$file" | grep -q "nativeMessaging"; then + + nativeMessaging__pass=false; + + else + + nativeMessaging__pass=true; + + fi + +} + +function amo_variables__licenses { + + license__amo_full_name="$(jq ".license.name" "$run_path/build/json/current_versions/$current_version__id.json" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;")" + +} + +function essential_addon { + + if [ "$has_eula" = false ] && [ "$nativeMessaging__pass" = true ] && [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then + + essential_addon=true; + + else + + essential_addon=false; + + fi + +} + +#function hard_to_evaluate_addon { + + # curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging + + + # API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668 + # The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies. + + # Old solution: + + # 1: Add this to FreeAMO-dependencies: + #belgium-eid eid-mw + + # 2: Uncomment this code: + # depends_on_external_program=false; + # while IFS= read -r dependencies; do + # + # dependency_slug=$(echo $dependencies | awk '{print $1}'); + # dependency=$(echo "$dependencies" | cut -f 2); + # + # if [ "$dependency_slug" == "$slug" ]; then + # + # depends_on_external_program=true; + # + # fi + # + # done < $run_path/FreeAMO-dependencies + +#} + +function progress { + + (>&2 echo -en "\r"; pv --progress --line-mode --size $pv__size --eta --timer) > /dev/null + +} + + +function html_to_mediawiki { + + sed " +# Remove Mozilla tracking links +s|https://outgoing.prod.mozaws.net/v1/.\{65\}||g; s|%3A|:|g; s|%23|#|g + +# Fix general links +s|<a rel=\\\\\\\"nofollow\\\\\\\" href=\\\\\\\"|[|g +s|\\\\\\\">| |g +s|</a>|]|g + +# Fix line breaks +s|\\\\n|\\n|g + +# Fix quotes +s|\\\\\\\"|\\\"|g + +# Fix bullets, many developers failed to use <li> needed to make line break +s|\\n- |\\n* |g +" + +} diff --git a/subprojects/freeamo/src/main_functions~ b/subprojects/freeamo/src/main_functions~ new file mode 100644 index 0000000..7aec7e8 --- /dev/null +++ b/subprojects/freeamo/src/main_functions~ @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2018 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +function check_files { + + if [ ! -f "$run_path/FreeAMO-$check_type" ]; then + + echo "$run_path/FreeAMO-$check_type doesn't exist."; + echo "Exiting" && exit + + else + + + # Make sure all links are using SSL + if grep -q "http://addons.mozilla.org/" $run_path/FreeAMO-$check_type; then + + echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)." + echo "Exiting" && exit + + fi + + fi + +} + +function line_status { + + + if [ "$file" == "$previous_file" ]; then + + ((++line)) + + else + + line="0"; # Should not be 1 + + fi + +} + +function amo_variables() { + id="$(jq ".results[$entry].id" "$file")"; + average_daily_users="$(jq ".results[$entry].average_daily_users" "$file")"; + has_eula="$(jq ".results[$entry].has_eula" "$file")"; + slug="$(jq ".results[$entry].slug" "$file" | sed "s|^\"||; s|\"$||;")"; + current_version__id=$(jq ".results[$entry].current_version.id" "$file"); + + # Names + name__enUS=$(jq ".results[$entry].name" "$file" | sed "s|-||;" | jq .enUS | sed "s|^\"||; s|\"$||;" ) + name__enGB=$(jq ".results[$entry].name" "$file" | sed "s|-||;" | jq .enGB | sed "s|^\"||; s|\"$||;" ) + if [ "$name__enUS" != "null" ]; then name="$name__enUS"; + elif [ "$name__enGB" != "null" ]; then name="$name__enGB"; + else name=$(jq ".results[$entry].name[]" "$file" | sed "s|-||;" | jq . | sed "s|^\"||; s|\"$||;"); # Use the first avalible name if English is not avaliable + fi + + + if jq .results[$entry].current_version.files[0].permissions "$file" | grep -q "nativeMessaging"; then + + nativeMessaging__pass=false; + + else + + nativeMessaging__pass=true; + + fi + +} + +function amo_variables__licenses { + + license__amo_full_name="$(jq ".license.name" "$run_path/build/json/current_versions/$current_version__id.json" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;")" + +} + +function essential_addon { + + if [ "$has_eula" = false ] && [ "$nativeMessaging__pass" = true ] && [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then + + essential_addon=true; + + else + + essential_addon=false; + + fi + +} + +#function hard_to_evaluate_addon { + + # curl "https://addons.mozilla.org/api/v3/addons/addon/nemid-nøglefilsprogram/" | jq .current_version.files[0].permissions | grep "nativeMessaging + + + # API data: Add software dependencies - https://github.com/mozilla/addons-server/issues/8668 + # The API should be fixed. there are to many add-ons to keep track on if they have or not have software dependencies. + + # Old solution: + + # 1: Add this to FreeAMO-dependencies: + #belgium-eid eid-mw + + # 2: Uncomment this code: + # depends_on_external_program=false; + # while IFS= read -r dependencies; do + # + # dependency_slug=$(echo $dependencies | awk '{print $1}'); + # dependency=$(echo "$dependencies" | cut -f 2); + # + # if [ "$dependency_slug" == "$slug" ]; then + # + # depends_on_external_program=true; + # + # fi + # + # done < $run_path/FreeAMO-dependencies + +#} + +function progress { + + (>&2 echo -en "\r"; pv --progress --line-mode --size $pv__size --eta --timer) > /dev/null + +} diff --git a/subprojects/freeamo/src/make-wiki b/subprojects/freeamo/src/make-wiki new file mode 100644 index 0000000..19e782f --- /dev/null +++ b/subprojects/freeamo/src/make-wiki @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2018 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +cd "build/json/search-pages" + +echo -e "\\nGenerating .wiki files for https://directory.fsf.org/wiki/:" + +echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\" +! WebExtension +! Description +! Claimed license +! Users +! Rating +! Updated" > "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki" + + +pv__size=$(ls . | wc -l) +output_file="$output_file_2"; +source "$run_path/src/main_functions" + +while IFS= read -r table; do + + ((++line)) + + if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then + + file=$(echo $table | awk '{print $1}'); + entry=$(echo $table | awk '{print $2}'); + + source "$run_path/src/main_functions" + amo_variables + amo_variables__licenses + + if [ "$license__amo_full_name" == "BSD License" ]; then license__fsd_short_name="BSD 2Clause"; + elif [ "$license__amo_full_name" == "GNU General Public License, version 3.0" ]; then license__fsd_short_name="GPLv3"; + elif [ "$license__amo_full_name" == "GNU General Public License, version 2.0" ]; then license__fsd_short_name="GPLv2"; + elif [ "$license__amo_full_name" == "GNU Lesser General Public License, version 3.0" ]; then license__fsd_short_name="LGPLv3"; + elif [ "$license__amo_full_name" == "GNU Lesser General Public License, version 2.1" ]; then license__fsd_short_name="LGPLv2.1"; + elif [ "$license__amo_full_name" == "MIT/X11 License" ]; then license__fsd_short_name="X11"; + elif [ "$license__amo_full_name" == "Mozilla Public License, version 2.0" ]; then license__fsd_short_name="MPLv2.0"; + elif [ "$license__amo_full_name" == "Mozilla Public License Version 1.1" ]; then license__fsd_short_name="MPLv1.1"; + fi + + source "$run_path/src/main_functions" + full_description=$(jq .results[$entry].description $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | html_to_mediawiki) + short_description=$(jq .results[$entry].summary $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | html_to_mediawiki) + + homepage_url=$(jq .results[$entry].homepage $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + version=$(jq .results[$entry].current_version.version $file | sed "s|^\"||; s|\"$||;"); + version_date=$(jq .results[$entry].last_updated $file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g"); + version_id=$(jq .results[$entry].current_version.id $file); + license_copyright=$(jq .results[$entry].authors[].name $file | sed "s|^\"||; s|\"$||;"); + bayesian_average=$(jq .results[$entry].ratings.bayesian_average $file); + bayesian_average__simple=$(printf "%.1f" "$(echo "$bayesian_average" | sed "s|\.|,|")"); + + last_review_date=$(date +"%Y/%m/%d"); + last_review_by="wikisysbot"; + submitted_by="wikisysbot"; + + if [[ "$(jq .results[$entry].current_version.compatibility.seamonkey $file)" != "null" ]]; then + set_seamonkey="Iceape,"; + fi + + if [[ "$(jq .results[$entry].current_version.compatibility.firefox $file)" != "null" ]]; then + set_icecat="IceCat,"; + fi + + # if [[ "$(jq .results[$entry].current_version.compatibility.android $file)" != "null" ]] || [[ "$(jq .results[$entry].current_version.compatibility.mobile $file)" != "null" ]]; then + # set_icecatmobile="IceCatMobile,"; + # fi + + if [[ "$(jq .results[$entry].current_version.compatibility.thunderbird $file)" != "null" ]]; then + set_thunderbird="Icedove,"; + fi + + # Do not list IceCatMobile + extension_of=$(echo "$set_seamonkey$set_icecat$set_thunderbird" | sed "s|,$||;"); + + + + github_true=$(echo "$homepage_url" | grep "github.com"); + + if [[ $github_true != "" ]]; then + + # Always use https, and remove anchors + homepage_url__for__vcs_checkout_command="https://github.com$(echo $github_true | sed "s|https://github.com||; s|http://github.com||; s|#|\n|;" | head -n 1)"; + # Remove trailing slash + homepage_url__for__vcs_checkout_command="${homepage_url__for__vcs_checkout_command%/}" + vcs_checkout_command="git clone $homepage_url__for__vcs_checkout_command.git"; + + fi + + if [ "$name" == "GNU LibreJS" ]; then + + is_gnu="Yes"; + + else + + is_gnu="No"; + + fi + + support_url=$(jq .results[$entry].support_url $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + + # List JSON structure: cat json/$file | js '.' + # XML value "homepage" is not always set in AMO API, but the XML require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically. + echo "{{Entry" > $run_path/build/wiki/$slug.wiki + + # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341 + # Avoid specific versions since it's not compatible with all IceCat versions: + # |Version download=https://addons.mozilla.org/firefox/downloads/latest/$id/addon-$version_id-latest.xpi + # |Version identifier=$version + # |Version date=$version_date + + + echo "|Name=$name +|Short description=$short_description +|Full description=$full_description +|Homepage URL=$homepage_url +|Extension of=$extension_of +|VCS checkout command=$vcs_checkout_command +|Version download=https://addons.mozilla.org/en-US/firefox/addon/$slug/versions/ +|Last review by=$last_review_by +|Last review date=$last_review_date +|Submitted by=$submitted_by +|Submitted date=2018/06/20 +|Is GNU=$is_gnu +}}" >> $run_path/build/wiki/$slug.wiki + + + + # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/ + echo "{{Project license +|License=$license__fsd_short_name +|License copyright=$license_copyright +|License note=\"License: $license__amo_full_name\" listed +at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/ +}}" >> $run_path/build/wiki/$slug.wiki + + if [[ "$support_url" != "" ]]; + then + + echo "{{Resource +|Resource audience=Users +|Resource kind=Support +|Resource URL=$support_url +}}" >> "$run_path/build/wiki/$slug.wiki" + + fi + + + echo "|- +| [[$name]] +| $short_description +| [[License:$license__fsd_short_name|$license__fsd_short_name]] +| $average_daily_users +| $bayesian_average__simple +| $version_date" >> "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki" + + fi + + echo "foo" # Must be here to work with pv +done < "$output_file_2" | progress +# End template + +echo "|}" >> "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki" diff --git a/subprojects/freeamo/src/make-wiki~ b/subprojects/freeamo/src/make-wiki~ new file mode 100644 index 0000000..d5aeb51 --- /dev/null +++ b/subprojects/freeamo/src/make-wiki~ @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2018 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +cd "build/json/search-pages" + +echo -e "\\nGenerating .wiki files for https://directory.fsf.org/wiki/:" + +echo "{| class=\"wikitable sortable\" border=\"1\" style=\"font-size:smaller\" +! WebExtension +! Description +! Claimed license +! Users +! Rating +! Updated" > "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki" + + +pv__size=$(ls . | wc -l) +output_file="$output_file_2"; +source "$run_path/src/main_functions" + +while IFS= read -r table; do + + ((++line)) + + if [ "$debug" = true ] && [ "$line" -le "3" ] || [ "$debug" = false ] ; then + + file=$(echo $table | awk '{print $1}'); + entry=$(echo $table | awk '{print $2}'); + + source "$run_path/src/main_functions" + amo_variables + amo_variables__licenses + + if [ "$license__amo_full_name" == "BSD License" ]; then license__fsd_short_name="BSD 2Clause"; + elif [ "$license__amo_full_name" == "GNU General Public License, version 3.0" ]; then license__fsd_short_name="GPLv3"; + elif [ "$license__amo_full_name" == "GNU General Public License, version 2.0" ]; then license__fsd_short_name="GPLv2"; + elif [ "$license__amo_full_name" == "GNU Lesser General Public License, version 3.0" ]; then license__fsd_short_name="LGPLv3"; + elif [ "$license__amo_full_name" == "GNU Lesser General Public License, version 2.1" ]; then license__fsd_short_name="LGPLv2.1"; + elif [ "$license__amo_full_name" == "MIT/X11 License" ]; then license__fsd_short_name="X11"; + elif [ "$license__amo_full_name" == "Mozilla Public License, version 2.0" ]; then license__fsd_short_name="MPLv2.0"; + elif [ "$license__amo_full_name" == "Mozilla Public License Version 1.1" ]; then license__fsd_short_name="MPLv1.1"; + fi + + short_description_raw=$(jq .results[$line].summary $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;" | sed "s|\\\n|\n|g; s|\\\||g; +s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues +s|http|\nhttp|g; +"); + short_description_again=$(echo "$short_description_raw" | sed "/outgoing.prod.mozaws.net/d;"); + + short_description=$(echo "$short_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g'); + + full_description_raw=$(jq .results[$line].description $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||; +s|<a rel=\"nofollow\" href=\"|[|g; s|\">| |g; s|</a>|]|g; # URL issues +"); + full_description_again=$(echo "$full_description_raw" | sed "/outgoing.prod.mozaws.net/d;"); + + full_description=$(echo "$full_description_again" | sed ':a;N;$!ba;s/\nhttp/http/g'); + + + + + homepage_url=$(jq .results[$line].homepage $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + version=$(jq .results[$line].current_version.version $file | sed "s|^\"||; s|\"$||;"); + version_date=$(jq .results[$line].last_updated $file | sed "s|^\"||; s|\"$||;" | sed "s|T|\n|" | head -n 1 | sed "s|-|/|g"); + version_id=$(jq .results[$line].current_version.id $file); + license_copyright=$(jq .results[$line].authors[].name $file | sed "s|^\"||; s|\"$||;"); + bayesian_average=$(jq .results[$line].ratings.bayesian_average $file); + bayesian_average__simple=$(printf "%.1f" "$(echo "$bayesian_average" | sed "s|\.|,|")"); + + last_review_date=$(date +"%Y/%m/%d"); + last_review_by="wikisysbot"; + submitted_by="wikisysbot"; + + if [[ "$(jq .results[$line].current_version.compatibility.seamonkey $file)" != "null" ]]; then + set_seamonkey="Iceape,"; + fi + + if [[ "$(jq .results[$line].current_version.compatibility.firefox $file)" != "null" ]]; then + set_icecat="IceCat,"; + fi + + # if [[ "$(jq .results[$line].current_version.compatibility.android $file)" != "null" ]] || [[ "$(jq .results[$line].current_version.compatibility.mobile $file)" != "null" ]]; then + # set_icecatmobile="IceCatMobile,"; + # fi + + if [[ "$(jq .results[$line].current_version.compatibility.thunderbird $file)" != "null" ]]; then + set_thunderbird="Icedove,"; + fi + + # Do not list IceCatMobile + extension_of=$(echo "$set_seamonkey$set_icecat$set_thunderbird" | sed "s|,$||;"); + + + + github_true=$(echo "$homepage_url" | grep "github.com"); + + if [[ $github_true != "" ]]; then + + # Always use https, and remove anchors + homepage_url__for__vcs_checkout_command="https://github.com$(echo $github_true | sed "s|https://github.com||; s|http://github.com||; s|#|\n|;" | head -n 1)"; + # Remove trailing slash + homepage_url__for__vcs_checkout_command="${homepage_url__for__vcs_checkout_command%/}" + vcs_checkout_command="git clone $homepage_url__for__vcs_checkout_command.git"; + + fi + + if [ "$name" == "GNU LibreJS" ]; then + + is_gnu="Yes"; + + else + + is_gnu="No"; + + fi + + support_url=$(jq .results[$line].support_url $file | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + + # List JSON structure: cat json/$file | js '.' + # XML value "homepage" is not always set in AMO API, but the XML require "Homepage URL" to be set: Therfore we use the AMO page itself in Homepage URL to complete the build of the repo automatically. + echo "{{Entry" > $run_path/build/wiki/$slug.wiki + + # Dash have to be removed in jq 1.3: https://github.com/stedolan/jq/issues/341 + # Avoid specific versions since it's not compatible with all IceCat versions: + # |Version download=https://addons.mozilla.org/firefox/downloads/latest/$id/addon-$version_id-latest.xpi + # |Version identifier=$version + # |Version date=$version_date + + + echo "|Name=$name +|Short description=$short_description +|Full description=$full_description +|Homepage URL=$homepage_url +|Extension of=$extension_of +|VCS checkout command=$vcs_checkout_command +|Version download=https://addons.mozilla.org/en-US/firefox/addon/$slug/versions/ +|Last review by=$last_review_by +|Last review date=$last_review_date +|Submitted by=$submitted_by +|Submitted date=2018/06/20 +|Is GNU=$is_gnu +}}" >> $run_path/build/wiki/$slug.wiki + + + + # https://addons.mozilla.org/api/v3/addons/addon/noscript/versions/1910123/ + echo "{{Project license +|License=$license__fsd_short_name +|License copyright=$license_copyright +|License note=\"License: $license__amo_full_name\" listed +at https://addons.mozilla.org/en-US/firefox/addon/adblock-plus/ +}}" >> $run_path/build/wiki/$slug.wiki + + if [[ "$support_url" != "" ]]; + then + + echo "{{Resource +|Resource audience=Users +|Resource kind=Support +|Resource URL=$support_url +}}" >> "$run_path/build/wiki/$slug.wiki" + + fi + + + echo "|- +| [[$name]] +| $short_description +| [[License:$license__fsd_short_name|$license__fsd_short_name]] +| $average_daily_users +| $bayesian_average__simple +| $version_date" >> "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki" + + + + fi + + echo "foo" # Must be here to work with pv +done < "$output_file_2" | progress +# End template + +echo "|}" >> "$run_path/build/wiki/IceCat WebExtensions (proposed).wiki" diff --git a/subprojects/freeamo/src/search-pages b/subprojects/freeamo/src/search-pages new file mode 100644 index 0000000..a723601 --- /dev/null +++ b/subprojects/freeamo/src/search-pages @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2017 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +cd "$run_path/build/json/search-pages" || exit + +base_uri="https://addons.mozilla.org/api/v3/addons/search/?platform=$platform&sort=$sort&type=$type&app=$app&appversion=$appversion&page_size=$page_size&page="; + +function search-pages--download() { + + echo -e "\\nDownloading search pages from $base_uri :" | sed "s|page=|page=\$page|" + + echo "This file is used to describe the settings that were used to generate the files. + +build/json/search-pages/ +$wget_uri" > "$run_path/build/declared_settings.txt" + # indicate that the page page variable are the page files + sed -i "s|page=1|page=\$page|g" "$run_path/build/declared_settings.txt" + + + + ##################################################### + + function wget_page { + + + wget_uri="$base_uri$page"; + + wget -q "$wget_uri" --no-verbose -O "$page.json" + + average_daily_users="$(jq ".results[].average_daily_users" "$page.json" | tail -n 1)" + + } + + page="1"; + wget_page + + page_count="$(jq ".page_count" "$page.json")"; + pv__size="$page_count" + source "$run_path/src/main_functions" + + + if [ "$debug" = true ]; then + + pv__size="2"; + + else + + pv__size="$page_count" + + fi + + + while [ "$page" -lt "$page_count" ]; do + + ((++page)) + + if ( [ "$debug" = true ] && [ "$page" -le "3" ] || [ "$debug" = false ] ) && ( [ "$average_daily_users" -ge "$minimum_average_daily_users" ] ); then + + echo "$page" + wget_page + + else + + exit 0 + + fi + + echo "foo" # Must be here to work with pv + done | progress + +} + +function search-pages--filter() { + + + echo -e "\\nFilter search pages -- no EULA, no native messaging, and minimum average daily users ($minimum_average_daily_users):" + + # source "$run_path/src/main_functions" + # amo_variables + echo " +build/index-search-pages.txt +has_eula=false +nativeMessaging__pass=true +minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/declared_settings.txt" + + pv__size=$(ls . | wc -l) + output_file="$output_file_1"; + source "$run_path/src/main_functions" + + rm -f $output_file_1 + + # Sort by file modification time stamp for the downloaded JSON files + find . -type f | sort -n | while read -r file; do + + + filename="$(echo $(basename "$file"))"; + + entry="0"; # Should not be 1 + + for id in $(jq ".results[].average_daily_users" "$file"); do + + source "$run_path/src/main_functions" + amo_variables + + essential_addon + if [ "$essential_addon" = true ]; then + + echo -e "$filename\\t$entry" >> "$output_file_1" + + fi + + ((++entry)) + + done + + echo "$filename" + + + echo "foo" # Must be here to work with pv + done | progress + +} diff --git a/subprojects/freeamo/src/search-pages~ b/subprojects/freeamo/src/search-pages~ new file mode 100644 index 0000000..3ffa9dd --- /dev/null +++ b/subprojects/freeamo/src/search-pages~ @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2017 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +cd "$run_path/build/json/search-pages" || exit + +function search-pages--download() { + + echo "$debug" + + echo -e "\\nDownloading search pages:" + + echo "This file is used to describe the settings that were used to generate the files. + +build/json/search-pages/ +$wget_uri" > "$run_path/build/declared_settings.txt" + # indicate that the page page variable are the page files + sed -i "s|page=1|page=\$page|g" "$run_path/build/declared_settings.txt" + + + + ##################################################### + + function wget_page { + + base_uri="https://addons.mozilla.org/api/v3/addons/search/?platform=linux&sort=users&type=extension&app=firefox&appversion=$latest_trisquel_icecat_version&page_size=50&page="; + # sort=users. sort=name not working yet - https://github.com/mozilla/addons-server/issues/7600 + # page_size=50 -- this is the maximum accepted number, it makes the download much faster: 20 seconds for 10 x page_size=50, 35 seconds for 20 x page_size=25 (default) + # page. Add this dynamic variable as the last variable to make it easy to read + wget_uri="$base_uri$page"; + + wget -q "$wget_uri" --no-verbose -O "$page.json" + + average_daily_users="$(jq ".results[].average_daily_users" "$page.json" | tail -n 1)" + + } + + page="1"; + wget_page + + page_count="$(jq ".page_count" "$page.json")"; + pv__size="$page_count" + source "$run_path/src/main_functions" + + + if [ "$debug" = true ]; then + + pv__size="2"; + + else + + pv__size="$page_count" + + fi + + + while [ "$page" -lt "$page_count" ]; do + + ((++page)) + + if ( [ "$debug" = true ] && [ "$page" -le "3" ] || [ "$debug" = false ] ) && ( [ "$average_daily_users" -ge "$minimum_average_daily_users" ] ); then + + echo "$page" + wget_page + + else + + exit 0 + + fi + + echo "foo" # Must be here to work with pv + done | progress + +} + +function search-pages--filter() { + + + echo -e "\\nFilter search pages:" + + # source "$run_path/src/main_functions" + # amo_variables + echo " +build/index-search-pages.txt +has_eula=false +nativeMessaging__pass=true +minimum_average_daily_users=$minimum_average_daily_users" >> "$run_path/build/declared_settings.txt" + + pv__size=$(ls . | wc -l) + output_file="$output_file_1"; + source "$run_path/src/main_functions" + + rm -f $output_file_1 + + # Sort by file modification time stamp for the downloaded JSON files + find . -type f | sort -n | while read -r file; do + + + filename="$(echo $(basename "$file"))"; + + entry="0"; # Should not be 1 + + for id in $(jq ".results[].average_daily_users" "$file"); do + + source "$run_path/src/main_functions" + amo_variables + + essential_addon + if [ "$essential_addon" = true ]; then + + echo -e "$filename\\t$entry" >> "$output_file_1" + + fi + + ((++entry)) + + done + + echo "$filename" + + + echo "foo" # Must be here to work with pv + done | progress + +} diff --git a/addons.mozilla.org-fsd/get-data/TODO b/subprojects/freeamo/src/unmaintained/TODO index 52b5c0e..80df787 100644 --- a/addons.mozilla.org-fsd/get-data/TODO +++ b/subprojects/freeamo/src/unmaintained/TODO @@ -11,7 +11,7 @@ I cannot install licenseutils for the moment because I held broken packages in m Look in GitHub repositories for LICENSE or COPYING. Sometime the add-on is delayed or have not been released to addons.mozilla.org." ############################### -license-reporter-anti-features +FreeAMO-anti-features -Work on license-reporter-anti-features +Work on FreeAMO-anti-features diff --git a/addons.mozilla.org-fsd/get-data/TODO~ b/subprojects/freeamo/src/unmaintained/TODO~ index 5bb2611..5bb2611 100644 --- a/addons.mozilla.org-fsd/get-data/TODO~ +++ b/subprojects/freeamo/src/unmaintained/TODO~ diff --git a/subprojects/freeamo/src/unmaintained/bad_names b/subprojects/freeamo/src/unmaintained/bad_names new file mode 100644 index 0000000..6ba8891 --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/bad_names @@ -0,0 +1,31 @@ + + +function search-pages--filter__bad_names() { + +name=$(jq .results[$entry].name "$run_path/build/json/$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + +if [ "$name" == "null" ]; then + +name=$(jq .results[$entry].name "$run_path/build/json/$file" | sed "s|en-GB|enGB|;" | jq .enGB | sed "s|^\"||; s|\"$||;"); + +fi + +# The API `name` parameter makes bad MediaWiki URL's - https://github.com/mozilla/addons-server/issues/8790 +# Clean titles. Example of bad titles: "Ghostery – Privacy Ad Blocker" + +# - +# Hard to filter out "-" since many add-ons use it, see for example https://addons.mozilla.org/en-US/firefox/search/?platform=linux&q=1-click +# Flash Video Downloader - YouTube HD Download [4K] + +# Find all names that don't contain Aa-Zz 0-9 and white spaces should be much more reliable than this: +name=$(echo "$name" | sed ' +s/ — /\n/g; +s/ – /\n/g; +s/ - /\n/g; # looks identical to the prevoius line but is unique +s/: /\n/g; +s/(/\n/g; +s/,/\n/g; +s/™/\n/g; +' | head -n 1 | sed 's/ $//'); + +} diff --git a/subprojects/freeamo/src/unmaintained/bad_names~ b/subprojects/freeamo/src/unmaintained/bad_names~ new file mode 100644 index 0000000..065ca82 --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/bad_names~ @@ -0,0 +1,31 @@ + + +function search-pages--filter__bad_names() { + +name=$(jq .results[$entry].name "$run_path/build/json/$file" | sed "s|en-US|enUS|;" | jq .enUS | sed "s|^\"||; s|\"$||;"); + +if [ "$name" == "null" ]; then + +name=$(jq .results[$entry].name "$run_path/build/json/$file" | sed "s|en-GB|enGB|;" | jq .enGB | sed "s|^\"||; s|\"$||;"); + +fi + +# Warn about clutter in Name - https://github.com/mozilla/addons-server/issues/8790 +# Clean titles. Example of bad titles: "Ghostery – Privacy Ad Blocker" + +# - +# Hard to filter out "-" since many add-ons use it, see for example https://addons.mozilla.org/en-US/firefox/search/?platform=linux&q=1-click +# Flash Video Downloader - YouTube HD Download [4K] + +# Find all names that don't contain Aa-Zz 0-9 and white spaces should be much more reliable than this: +name=$(echo "$name" | sed ' +s/ — /\n/g; +s/ – /\n/g; +s/ - /\n/g; # looks identical to the prevoius line but is unique +s/: /\n/g; +s/(/\n/g; +s/,/\n/g; +s/™/\n/g; +' | head -n 1 | sed 's/ $//'); + +} diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-anti-features b/subprojects/freeamo/src/unmaintained/conf/license-reporter-anti-features index 30e8a41..30e8a41 100644 --- a/addons.mozilla.org-fsd/get-data/license-reporter-anti-features +++ b/subprojects/freeamo/src/unmaintained/conf/license-reporter-anti-features diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-collections b/subprojects/freeamo/src/unmaintained/conf/license-reporter-collections index 1d58d53..1d58d53 100644 --- a/addons.mozilla.org-fsd/get-data/license-reporter-collections +++ b/subprojects/freeamo/src/unmaintained/conf/license-reporter-collections diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-custom b/subprojects/freeamo/src/unmaintained/conf/license-reporter-custom index dd185b0..dd185b0 100644 --- a/addons.mozilla.org-fsd/get-data/license-reporter-custom +++ b/subprojects/freeamo/src/unmaintained/conf/license-reporter-custom diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-custom~ b/subprojects/freeamo/src/unmaintained/conf/license-reporter-custom~ index 6927339..6927339 100644 --- a/addons.mozilla.org-fsd/get-data/license-reporter-custom~ +++ b/subprojects/freeamo/src/unmaintained/conf/license-reporter-custom~ diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-log b/subprojects/freeamo/src/unmaintained/conf/license-reporter-log index f8aada4..f8aada4 100644 --- a/addons.mozilla.org-fsd/get-data/license-reporter-log +++ b/subprojects/freeamo/src/unmaintained/conf/license-reporter-log diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-log~ b/subprojects/freeamo/src/unmaintained/conf/license-reporter-log~ index 7c65a97..7c65a97 100644 --- a/addons.mozilla.org-fsd/get-data/license-reporter-log~ +++ b/subprojects/freeamo/src/unmaintained/conf/license-reporter-log~ diff --git a/addons.mozilla.org-fsd/get-data/license-reporter-repositories b/subprojects/freeamo/src/unmaintained/conf/license-reporter-repositories index 9e75120..9e75120 100644 --- a/addons.mozilla.org-fsd/get-data/license-reporter-repositories +++ b/subprojects/freeamo/src/unmaintained/conf/license-reporter-repositories diff --git a/addons.mozilla.org-fsd/get-data/src/download-custom b/subprojects/freeamo/src/unmaintained/download-custom index b6c5def..57bc5d7 100755 --- a/addons.mozilla.org-fsd/get-data/src/download-custom +++ b/subprojects/freeamo/src/unmaintained/download-custom @@ -1,14 +1,13 @@ #!/usr/bin/env bash - -# This file is part of license-reporter +# This file is part of FreeAMO # Copyright (C) 2017 David Hedlund # -# license-reporter is free software: you can redistribute it and/or modify +# FreeAMO is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# license-reporter is distributed in the hope that it will be useful, +# FreeAMO is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. @@ -19,21 +18,21 @@ cd build || exit mkdir json -cp -a "$RUN_PATH/license-reporter-custom" custom_addons.txt +cp -a "$run_path/FreeAMO-custom" custom_addons.txt # Sanitize a copy from empty lines, they use to occur at the end of the file -sed -i '/^\s*$/d' license-reporter-custom +sed -i '/^\s*$/d' FreeAMO-custom sed -i "s|https://addons.mozilla.org/en-US/firefox/addon/||g; s|/||" custom_addons.txt rm -fr \ - custom_addons-json-source.txt \ - custom_addons-average_daily_users.txt \ - custom_addons-id.txt custom_addons-slug.txt \ - custom_addons-merged.txt \ - custom_addons-report-expired_links.txt \ - custom_addons-merged-sorted.txt + custom_addons-json-source.txt \ + custom_addons-average_daily_users.txt \ + custom_addons-id.txt custom_addons-slug.txt \ + custom_addons-merged.txt \ + custom_addons-report-expired_links.txt \ + custom_addons-merged-sorted.txt cd json || exit @@ -55,7 +54,7 @@ while IFS= read -r slug; do if [ "$(jq ".average_daily_users" "$id.json")" -gt "$minimum_average_daily_users" ]; then - echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tAlready in the main repository\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$RUN_PATH/license-reporter-log" + echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tAlready in the main repository\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$run_path/FreeAMO-log" else @@ -73,7 +72,7 @@ while IFS= read -r slug; do else - echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tLink rot\t\t\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$RUN_PATH/license-reporter-log" + echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tLink rot\t\t\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$run_path/FreeAMO-log" fi diff --git a/addons.mozilla.org-fsd/get-data/src/download-custom~ b/subprojects/freeamo/src/unmaintained/download-custom~ index 4af8ce1..3724aba 100755 --- a/addons.mozilla.org-fsd/get-data/src/download-custom~ +++ b/subprojects/freeamo/src/unmaintained/download-custom~ @@ -1,14 +1,14 @@ #!/usr/bin/env bash -# This file is part of license-reporter +# This file is part of FreeAMO # Copyright (C) 2017 David Hedlund # -# license-reporter is free software: you can redistribute it and/or modify +# FreeAMO is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# license-reporter is distributed in the hope that it will be useful, +# FreeAMO is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. @@ -19,10 +19,10 @@ cd build || exit mkdir json -cp -a "$RUN_PATH/license-reporter-custom" custom_addons.txt +cp -a "$run_path/FreeAMO-custom" custom_addons.txt # Sanitize a copy from empty lines, they use to occur at the end of the file -sed -i '/^\s*$/d' license-reporter-custom +sed -i '/^\s*$/d' FreeAMO-custom sed -i "s|https://addons.mozilla.org/en-US/firefox/addon/||g; s|/||" custom_addons.txt @@ -55,7 +55,7 @@ while IFS= read -r slug; do if [ "$(jq ".average_daily_users" "$id.json")" -gt "$minimum_average_daily_users" ]; then - echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tAlready in the main repository\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$RUN_PATH/license-reporter-log" + echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tAlready in the main repository\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$run_path/FreeAMO-log" else @@ -73,7 +73,7 @@ while IFS= read -r slug; do else - echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tLink rot\t\t\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$RUN_PATH/license-reporter-log" + echo -e "$(date +%Y-%m-%d" "%H:%M)\tCustom list\tLink rot\t\t\thttps://addons.mozilla.org/en-US/firefox/addon/$slug/" >> "$run_path/FreeAMO-log" fi diff --git a/subprojects/freeamo/src/unmaintained/download-free-webextensions b/subprojects/freeamo/src/unmaintained/download-free-webextensions new file mode 100644 index 0000000..7084b65 --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/download-free-webextensions @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2017 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +cd build || exit +rm -fr free_webextensions +mkdir free_webextensions +cd free_webextensions || exit + + + +# Begin template +unset file +while IFS= read -r table; do + + verify_variables "$1" + + if [ "$approved_program" = true ]; then + + # End template + + id=$(echo $table | awk '{print $2}'); + file=$(echo $table | awk '{print $3}'); + slug=$(echo $table | awk '{print $4}'); + + # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). + wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" + + + + # Begin template + fi + + ((line++)) + +done < $run_path/build/MERGED-ALL-freedom_status.txt +# End template diff --git a/subprojects/freeamo/src/unmaintained/download-free-webextensions~ b/subprojects/freeamo/src/unmaintained/download-free-webextensions~ new file mode 100644 index 0000000..dd02d52 --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/download-free-webextensions~ @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +cd build || exit +rm -fr free_webextensions +mkdir free_webextensions +cd free_webextensions || exit + + + +# Begin template +unset file +while IFS= read -r table; do + + verify_variables "$1" + + if [ "$approved_program" = true ]; then + + # End template + + id=$(echo $table | awk '{print $2}'); + file=$(echo $table | awk '{print $3}'); + slug=$(echo $table | awk '{print $4}'); + + # Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough). + wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi" + + + + # Begin template + fi + + ((line++)) + +done < $run_path/build/MERGED-ALL-freedom_status.txt +# End template diff --git a/subprojects/freeamo/src/unmaintained/make-collection-list b/subprojects/freeamo/src/unmaintained/make-collection-list new file mode 100644 index 0000000..f311129 --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/make-collection-list @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2018 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +cd build/ || exit +for i in $(cat ../FreeAMO-collections); do + + # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson + collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); + wget $i"format%3Ajson" -O "collection-$collection_name.json" + +done diff --git a/subprojects/freeamo/src/unmaintained/make-collection-list~ b/subprojects/freeamo/src/unmaintained/make-collection-list~ new file mode 100644 index 0000000..2e65310 --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/make-collection-list~ @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +cd build/ || exit +for i in $(cat ../FreeAMO-collections); do + + # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson + collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); + wget $i"format%3Ajson" -O "collection-$collection_name.json" + +done diff --git a/subprojects/freeamo/src/unmaintained/make-repository-list b/subprojects/freeamo/src/unmaintained/make-repository-list new file mode 100644 index 0000000..d043659 --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/make-repository-list @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2018 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs. +# This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync +# your addons.mozilla.org collection list with a separate text file. + +# https://github.com/mozilla/addons/issues/722 +# Cannot list collections in desired formats +# Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list). +# So we have to parse it from HTML. + + +# Get the links to the collections + +function get_repositories { + + check_type="repositories"; + check_files + + cd build/ || exit + + for i in $(cat ../FreeAMO-repositories); do + + user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;"); + rm -f repository-$user + + + function wget_repository { + + match="^ <a href=\"/en-US/firefox/collections/" + wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user + + } + + # Figure out if there are several pages + number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||"); + + if [ -z "$number_of_pages" ]; then + + echo "$i doesn't have any sub pages"; + wget_repository + + else + + echo "$i has sub pages"; + + page_number=1; + + while [ $page_number -le $number_of_pages ] + do + + wget_repository__page_extension="?page=$page_number"; + echo "$wget_repository__page_extension" + wget_repository + page_number=$(( $page_number + 1 )) + + done + + fi + + done +} +get_repositories + + +# Download the JSON files from the collections + +cd .. +for x in $(find build/ -name repository* | sort); do + + for i in $(cat $x); do + + collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); + wget $i"format%3Ajson" -O "build/collection-$collection_name.json" + + done + +done diff --git a/subprojects/freeamo/src/unmaintained/make-repository-list~ b/subprojects/freeamo/src/unmaintained/make-repository-list~ new file mode 100644 index 0000000..97dc82d --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/make-repository-list~ @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +# It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs. +# This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync +# your addons.mozilla.org collection list with a separate text file. + +# https://github.com/mozilla/addons/issues/722 +# Cannot list collections in desired formats +# Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list). +# So we have to parse it from HTML. + + +# Get the links to the collections + +function get_repositories { + + check_type="repositories"; + check_files + + cd build/ || exit + + for i in $(cat ../FreeAMO-repositories); do + + user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;"); + rm -f repository-$user + + + function wget_repository { + + match="^ <a href=\"/en-US/firefox/collections/" + wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user + + } + + # Figure out if there are several pages + number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||"); + + if [ -z "$number_of_pages" ]; then + + echo "$i doesn't have any sub pages"; + wget_repository + + else + + echo "$i has sub pages"; + + page_number=1; + + while [ $page_number -le $number_of_pages ] + do + + wget_repository__page_extension="?page=$page_number"; + echo "$wget_repository__page_extension" + wget_repository + page_number=$(( $page_number + 1 )) + + done + + fi + + done +} +get_repositories + + +# Download the JSON files from the collections + +cd .. +for x in $(find build/ -name repository* | sort); do + + for i in $(cat $x); do + + collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); + wget $i"format%3Ajson" -O "build/collection-$collection_name.json" + + done + +done diff --git a/subprojects/freeamo/src/unmaintained/make-xml b/subprojects/freeamo/src/unmaintained/make-xml new file mode 100644 index 0000000..59d2286 --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/make-xml @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2017 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# Deprecated. + + + # XML pages look like this: + #... + # <sha1>tlybh1k9mnxz92kccg46hqv9jvyrkp0</sha1> + # </revision> + # </page> + #</mediawiki> + # + # However. It's not possible to insert the checsum in <sha1> for the page since it will break the checksum itself. + + # Get number of edits: + revid="$(wget -qO- "https://directory.fsf.org/w/api.php?action=query&list=recentchanges&rclimit=1&format=json" | jq .query.recentchanges[0].revid)"; + xml_header="$(wget -qO- "https://directory.fsf.org/wiki/Special:Export/Main_Page" | head -n 42)"; diff --git a/subprojects/freeamo/src/unmaintained/make-xml~ b/subprojects/freeamo/src/unmaintained/make-xml~ new file mode 100644 index 0000000..557c7e4 --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/make-xml~ @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Deprecated. + + + # XML pages look like this: + #... + # <sha1>tlybh1k9mnxz92kccg46hqv9jvyrkp0</sha1> + # </revision> + # </page> + #</mediawiki> + # + # However. It's not possible to insert the checsum in <sha1> for the page since it will break the checksum itself. + + # Get number of edits: + revid="$(wget -qO- "https://directory.fsf.org/w/api.php?action=query&list=recentchanges&rclimit=1&format=json" | jq .query.recentchanges[0].revid)"; + xml_header="$(wget -qO- "https://directory.fsf.org/wiki/Special:Export/Main_Page" | head -n 42)"; diff --git a/addons.mozilla.org-fsd/sync-data/Text_templates/issue__intro.txt b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue__intro.txt index 8318161..8318161 100644 --- a/addons.mozilla.org-fsd/sync-data/Text_templates/issue__intro.txt +++ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue__intro.txt diff --git a/addons.mozilla.org-fsd/sync-data/Text_templates/issue__intro.txt~ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue__intro.txt~ index 4b526c9..4b526c9 100644 --- a/addons.mozilla.org-fsd/sync-data/Text_templates/issue__intro.txt~ +++ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue__intro.txt~ diff --git a/addons.mozilla.org-fsd/sync-data/Text_templates/issue__lack_of_full_license_headers_in_each_file.txt b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue__lack_of_full_license_headers_in_each_file.txt index 9aa890e..9aa890e 100644 --- a/addons.mozilla.org-fsd/sync-data/Text_templates/issue__lack_of_full_license_headers_in_each_file.txt +++ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue__lack_of_full_license_headers_in_each_file.txt diff --git a/addons.mozilla.org-fsd/sync-data/Text_templates/issue__license_copy.txt b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue__license_copy.txt index f817164..f817164 100644 --- a/addons.mozilla.org-fsd/sync-data/Text_templates/issue__license_copy.txt +++ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue__license_copy.txt diff --git a/addons.mozilla.org-fsd/sync-data/Text_templates/issue__license_copy.txt~ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue__license_copy.txt~ index 5419295..5419295 100644 --- a/addons.mozilla.org-fsd/sync-data/Text_templates/issue__license_copy.txt~ +++ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue__license_copy.txt~ diff --git a/addons.mozilla.org-fsd/sync-data/Text_templates/issue__license_not_displayed_on_AMO.txt b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue__license_not_displayed_on_AMO.txt index 5c2f135..5c2f135 100644 --- a/addons.mozilla.org-fsd/sync-data/Text_templates/issue__license_not_displayed_on_AMO.txt +++ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue__license_not_displayed_on_AMO.txt diff --git a/addons.mozilla.org-fsd/sync-data/Text_templates/issue_intro.txt~ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue_intro.txt~ index 123d39e..123d39e 100644 --- a/addons.mozilla.org-fsd/sync-data/Text_templates/issue_intro.txt~ +++ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/issue_intro.txt~ diff --git a/addons.mozilla.org-fsd/sync-data/Text_templates/output.txt b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/output.txt index 3059065..3059065 100644 --- a/addons.mozilla.org-fsd/sync-data/Text_templates/output.txt +++ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/output.txt diff --git a/addons.mozilla.org-fsd/sync-data/Text_templates/outro.txt b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/outro.txt index d8d420d..d8d420d 100644 --- a/addons.mozilla.org-fsd/sync-data/Text_templates/outro.txt +++ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/outro.txt diff --git a/addons.mozilla.org-fsd/sync-data/Text_templates/outro.txt~ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/outro.txt~ index 123d39e..123d39e 100644 --- a/addons.mozilla.org-fsd/sync-data/Text_templates/outro.txt~ +++ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/outro.txt~ diff --git a/addons.mozilla.org-fsd/sync-data/Text_templates/request~ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/request~ index 76dbaa4..76dbaa4 100755 --- a/addons.mozilla.org-fsd/sync-data/Text_templates/request~ +++ b/subprojects/freeamo/src/unmaintained/sync-data/Text_templates/request~ diff --git a/addons.mozilla.org-fsd/sync-data/fsd-amo-bot b/subprojects/freeamo/src/unmaintained/sync-data/fsd-amo-bot index 4947ae7..4947ae7 100755 --- a/addons.mozilla.org-fsd/sync-data/fsd-amo-bot +++ b/subprojects/freeamo/src/unmaintained/sync-data/fsd-amo-bot diff --git a/addons.mozilla.org-fsd/sync-data/fsd-amo-bot-issues.txt b/subprojects/freeamo/src/unmaintained/sync-data/fsd-amo-bot-issues.txt index 3a159f0..3a159f0 100644 --- a/addons.mozilla.org-fsd/sync-data/fsd-amo-bot-issues.txt +++ b/subprojects/freeamo/src/unmaintained/sync-data/fsd-amo-bot-issues.txt diff --git a/addons.mozilla.org-fsd/sync-data/fsd-amo-bot~ b/subprojects/freeamo/src/unmaintained/sync-data/fsd-amo-bot~ index 8f1d6f3..8f1d6f3 100755 --- a/addons.mozilla.org-fsd/sync-data/fsd-amo-bot~ +++ b/subprojects/freeamo/src/unmaintained/sync-data/fsd-amo-bot~ diff --git a/addons.mozilla.org-fsd/sync-data/fsdedits.xml b/subprojects/freeamo/src/unmaintained/sync-data/fsdedits.xml index 187b37c..187b37c 100644 --- a/addons.mozilla.org-fsd/sync-data/fsdedits.xml +++ b/subprojects/freeamo/src/unmaintained/sync-data/fsdedits.xml diff --git a/addons.mozilla.org-fsd/sync-data/index.html b/subprojects/freeamo/src/unmaintained/sync-data/index.html index 3caf835..3caf835 100644 --- a/addons.mozilla.org-fsd/sync-data/index.html +++ b/subprojects/freeamo/src/unmaintained/sync-data/index.html diff --git a/subprojects/freeamo/src/unmaintained/verify-license-copy b/subprojects/freeamo/src/unmaintained/verify-license-copy new file mode 100644 index 0000000..773c48e --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/verify-license-copy @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2018 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +cd build || exit +rm -fr VERIFY-LICENSE-COPY.txt +cd free_webextensions || exit + + + +# Begin template +unset file +while IFS= read -r table; do + + verify_variables "$1" + + if [ "$approved_program" = true ]; then + # End template + + id=$(echo $table | awk '{print $2}'); + file=$(echo $table | awk '{print $3}'); + slug=$(echo $table | awk '{print $4}'); + + echo "-------------------------- +$name ($id): $license"; + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.txt" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.md" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.txt" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.md" + + # Begin template + fi + + ((line++)) + +done < $run_path/build/MERGED-ALL-freedom_status.txt +# End template diff --git a/subprojects/freeamo/src/unmaintained/verify-license-copy-old b/subprojects/freeamo/src/unmaintained/verify-license-copy-old new file mode 100644 index 0000000..640d6d5 --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/verify-license-copy-old @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# This file is part of FreeAMO +# Copyright (C) 2017 David Hedlund +# +# FreeAMO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FreeAMO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +while IFS= read -r id; do + + + + + if [ ! -f "addon-$id-latest.xpi" ]; then + + echo "Nonfree" >> ../VERIFY-LICENSE-COPY.txt + + elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE$"; then + + echo "LICENSE" >> ../VERIFY-LICENSE-COPY.txt + + elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.txt$"; then + + echo "LICENSE.txt" >> ../VERIFY-LICENSE-COPY.txt + + elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.md$"; then + + echo "LICENSE.md" >> ../VERIFY-LICENSE-COPY.txt + + + + elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING$"; then + + echo "COPYING" >> ../VERIFY-LICENSE-COPY.txt + + elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.txt$"; then + + echo "COPYING.txt" >> ../VERIFY-LICENSE-COPY.txt + + elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.md$"; then + + echo "COPYING.md" >> ../VERIFY-LICENSE-COPY.txt + + else + + echo "Request" >> ../VERIFY-LICENSE-COPY.txt + + fi + +done < ../id.txt + + + +line="0"; + +while IFS= read -r status; do + + + ((line++)) + + average_daily_users="$(sed -n ${line}p ../average_daily_users.txt)"; + id="$(sed -n ${line}p ../id.txt)"; + slug="$(sed -n ${line}p ../slug.txt)"; + license_name_en_US="$(sed -n ${line}p ../license.name.enUS.txt)"; + homepage_en_US="$(sed -n ${line}p ../homepage.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; + support_email_en_US="$(sed -n ${line}p ../support_email.en-US.txt)"; + support_url_en_US="$(sed -n ${line}p ../support_url.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; + + + + if [ "$license_name_en_US" == "GNU General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/gpl-3.0.txt"; license_file_name="COPYING"; + elif [ "$license_name_en_US" == "GNU General Public License, version 2.0" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"; license_file_name="COPYING"; + elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/lgpl-3.0.txt"; license_file_name="COPYING"; + elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 2.1" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt"; license_file_name="COPYING"; + elif [ "$license_name_en_US" == "Mozilla Public License, version 2.0" ]; then license_txt="https://www.mozilla.org/media/MPL/2.0/index.txt"; license_file_name="LICENSE"; + elif [ "$license_name_en_US" == "Mozilla Public License Version 1.1" ]; then license_txt="https://www.mozilla.org/media/MPL/1.1/index.txt"; license_file_name="LICENSE"; + elif [ "$license_name_en_US" == "BSD License" ]; then license_txt="https://directory.fsf.org/wiki?title=License:FreeBSD"; license_file_name="LICENSE"; + elif [ "$license_name_en_US" == "MIT/X11 License" ]; then license_txt="https://directory.fsf.org/wiki/License:X11"; license_file_name="LICENSE"; + fi + + + if [ "$(sed -n ${line}p ../name.en-US.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-US.txt)"; + elif [ "$(sed -n ${line}p ../name.en-GB.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-GB.txt)"; + else name="$slug"; + fi + + + if [ "$status" = "Request" ]; then + + ((count_reports++)) + + # Authors that don't provide contact information will be ignored, its impossible to track conversations in Reviews anyway, because they will be deleted. + + if [ "$homepage_en_US" != "null" ] || [ "$support_url_en_US" != "null" ] || [ "$support_email_en_US" != "null" ]; then + + decimals="$(echo "$average_daily_users" | sed -r ':L;s=\b([0-9]+)([0-9]{3})\b=\1,\2=g;t L')"; + + echo " +$decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/addon/$slug/ + Request: \"You have distributed $name under $license_name_en_US on https://addons.mozilla.org/en-US/firefox/addon/$slug/, but you have not added a copy of the license. can you please download the text of the license from $license_txt and add save it to file name $license_file_name and put it in the root directory of you program?\" by contacting:"; + + if [ "$homepage_en_US" != "null" ]; then echo " * Homepage (please submit Pull Request for $license_file_name if possible): $homepage_en_US"; fi + if [ "$support_url_en_US" != "null" ]; then echo " * Support URL (please submit Pull Request for $license_file_name if possible): $support_url_en_US"; fi + if [ "$support_email_en_US" != "null" ]; then echo " * Support E-mail: $support_email_en_US"; fi + + fi + + fi + +done < ../VERIFY-LICENSE-COPY.txt + +echo " +######################### +$count_reports reports to submit"; diff --git a/subprojects/freeamo/src/unmaintained/verify-license-copy-old~ b/subprojects/freeamo/src/unmaintained/verify-license-copy-old~ new file mode 100644 index 0000000..75d9509 --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/verify-license-copy-old~ @@ -0,0 +1,108 @@ +#!/usr/bin/env bash + +while IFS= read -r id; do + + + + + if [ ! -f "addon-$id-latest.xpi" ]; then + + echo "Nonfree" >> ../VERIFY-LICENSE-COPY.txt + + elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE$"; then + + echo "LICENSE" >> ../VERIFY-LICENSE-COPY.txt + + elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.txt$"; then + + echo "LICENSE.txt" >> ../VERIFY-LICENSE-COPY.txt + + elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.md$"; then + + echo "LICENSE.md" >> ../VERIFY-LICENSE-COPY.txt + + + + elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING$"; then + + echo "COPYING" >> ../VERIFY-LICENSE-COPY.txt + + elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.txt$"; then + + echo "COPYING.txt" >> ../VERIFY-LICENSE-COPY.txt + + elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.md$"; then + + echo "COPYING.md" >> ../VERIFY-LICENSE-COPY.txt + + else + + echo "Request" >> ../VERIFY-LICENSE-COPY.txt + + fi + +done < ../id.txt + + + +line="0"; + +while IFS= read -r status; do + + + ((line++)) + + average_daily_users="$(sed -n ${line}p ../average_daily_users.txt)"; + id="$(sed -n ${line}p ../id.txt)"; + slug="$(sed -n ${line}p ../slug.txt)"; + license_name_en_US="$(sed -n ${line}p ../license.name.enUS.txt)"; + homepage_en_US="$(sed -n ${line}p ../homepage.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; + support_email_en_US="$(sed -n ${line}p ../support_email.en-US.txt)"; + support_url_en_US="$(sed -n ${line}p ../support_url.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)"; + + + + if [ "$license_name_en_US" == "GNU General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/gpl-3.0.txt"; license_file_name="COPYING"; + elif [ "$license_name_en_US" == "GNU General Public License, version 2.0" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"; license_file_name="COPYING"; + elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/lgpl-3.0.txt"; license_file_name="COPYING"; + elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 2.1" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt"; license_file_name="COPYING"; + elif [ "$license_name_en_US" == "Mozilla Public License, version 2.0" ]; then license_txt="https://www.mozilla.org/media/MPL/2.0/index.txt"; license_file_name="LICENSE"; + elif [ "$license_name_en_US" == "Mozilla Public License Version 1.1" ]; then license_txt="https://www.mozilla.org/media/MPL/1.1/index.txt"; license_file_name="LICENSE"; + elif [ "$license_name_en_US" == "BSD License" ]; then license_txt="https://directory.fsf.org/wiki?title=License:FreeBSD"; license_file_name="LICENSE"; + elif [ "$license_name_en_US" == "MIT/X11 License" ]; then license_txt="https://directory.fsf.org/wiki/License:X11"; license_file_name="LICENSE"; + fi + + + if [ "$(sed -n ${line}p ../name.en-US.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-US.txt)"; + elif [ "$(sed -n ${line}p ../name.en-GB.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-GB.txt)"; + else name="$slug"; + fi + + + if [ "$status" = "Request" ]; then + + ((count_reports++)) + + # Authors that don't provide contact information will be ignored, its impossible to track conversations in Reviews anyway, because they will be deleted. + + if [ "$homepage_en_US" != "null" ] || [ "$support_url_en_US" != "null" ] || [ "$support_email_en_US" != "null" ]; then + + decimals="$(echo "$average_daily_users" | sed -r ':L;s=\b([0-9]+)([0-9]{3})\b=\1,\2=g;t L')"; + + echo " +$decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/addon/$slug/ + Request: \"You have distributed $name under $license_name_en_US on https://addons.mozilla.org/en-US/firefox/addon/$slug/, but you have not added a copy of the license. can you please download the text of the license from $license_txt and add save it to file name $license_file_name and put it in the root directory of you program?\" by contacting:"; + + if [ "$homepage_en_US" != "null" ]; then echo " * Homepage (please submit Pull Request for $license_file_name if possible): $homepage_en_US"; fi + if [ "$support_url_en_US" != "null" ]; then echo " * Support URL (please submit Pull Request for $license_file_name if possible): $support_url_en_US"; fi + if [ "$support_email_en_US" != "null" ]; then echo " * Support E-mail: $support_email_en_US"; fi + + fi + + fi + +done < ../VERIFY-LICENSE-COPY.txt + +echo " +######################### +$count_reports reports to submit"; diff --git a/subprojects/freeamo/src/unmaintained/verify-license-copy~ b/subprojects/freeamo/src/unmaintained/verify-license-copy~ new file mode 100644 index 0000000..a77340b --- /dev/null +++ b/subprojects/freeamo/src/unmaintained/verify-license-copy~ @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +cd build || exit +rm -fr VERIFY-LICENSE-COPY.txt +cd free_webextensions || exit + + + +# Begin template +unset file +while IFS= read -r table; do + + verify_variables "$1" + + if [ "$approved_program" = true ]; then + # End template + + id=$(echo $table | awk '{print $2}'); + file=$(echo $table | awk '{print $3}'); + slug=$(echo $table | awk '{print $4}'); + + echo "-------------------------- +$name ($id): $license"; + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.txt" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "COPYING.md" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.txt" + unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep "LICENSE.md" + + # Begin template + fi + + ((line++)) + +done < $run_path/build/MERGED-ALL-freedom_status.txt +# End template |