From 7019dc9dcc6aa13505aa9e27c6b99796d7d695bb Mon Sep 17 00:00:00 2001 From: david Date: Thu, 21 Jun 2018 15:18:26 +0200 Subject: Added documentation for the options in addons.mozilla.org-fsd/get-data/license-reporter --- addons.mozilla.org-fsd/get-data/license-reporter | 237 ++++++++++++----------- 1 file changed, 123 insertions(+), 114 deletions(-) (limited to 'addons.mozilla.org-fsd/get-data') diff --git a/addons.mozilla.org-fsd/get-data/license-reporter b/addons.mozilla.org-fsd/get-data/license-reporter index 3a73598..47c28a8 100755 --- a/addons.mozilla.org-fsd/get-data/license-reporter +++ b/addons.mozilla.org-fsd/get-data/license-reporter @@ -92,20 +92,30 @@ case "$1" in [ "$1" = "" ] && echo "Usage: $filename [--option] [--debug] OPTIONS - $0 --all $0 --fresh-build + Remove the build directory and create a new one. + $0 --make-search-list + Downloads the most popular WebExtensions from https://addons.mozilla.org/en-US/firefox/search/... $0 --make-repository-list - Downloads add-on data for free add-ons from collections shared by defined users in license-reporter-repositories. Example: - https://addons.mozilla.org/en-US/firefox/collections/mozilla/ + Required file: license-reporter-collections + File content syntax: https://addons.mozilla.org/en-US/firefox/collections// $0 --make-collection-list + Required file: license-reporter-collections + File content syntax: https://addons.mozilla.org/en-US/firefox/collections// $0 --make-custom-list - $0 --make-search-list - $0 --get-licenses + Required file: license-reporter-custom + File content syntax: https://addons.mozilla.org/en-US/firefox/addon// $0 --merge-lists - $0 --generate-proposed-list + Merge the lists to MERGED-ALL.txt + $0 --get-licenses + The licenses are not added to the general JSON file. This option will download the neccesary license JSON files for all add-ons. $0 --download-licenses-json $0 --download-free-webextensions-for-gnu-and-linux $0 --verify-license-copy + $0 --merged-free + Generates MERGED-ALL-freedom_status.txt + $0 --make-wiki + Generates .wiki files --all-recommended $0 --fresh-build @@ -123,117 +133,11 @@ OPTIONS if [ -d "build" ]; then - rm -fr build + rm -fr $RUN_PATH/build fi - mkdir -p build - - ;; - - --make-repository-list) - - # It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs. - # This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync - # your addons.mozilla.org collection list with a separate text file. - - # https://github.com/mozilla/addons/issues/722 - # Cannot list collections in desired formats - # Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list). - # So we have to parse it from HTML. - - - # Get the links to the collections - - function get_repositories { - - check_type="repositories"; - check_files - - cd build/ || exit - - for i in $(cat ../license-reporter-repositories); do - - user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;"); - rm -f repository-$user - - - function wget_repository { - - match="^ ||" >> repository-$user - - } - - # Figure out if there are several pages - number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||"); - - if [ -z "$number_of_pages" ]; then - - echo "$i doesn't have any sub pages"; - wget_repository - - else - - echo "$i has sub pages"; - - page_number=1; - - while [ $page_number -le $number_of_pages ] - do - - wget_repository__page_extension="?page=$page_number"; - echo "$wget_repository__page_extension" - wget_repository - page_number=$(( $page_number + 1 )) - - done - - fi - - done - } - get_repositories - - - # Download the JSON files from the collections - - cd .. - for x in $(find build/ -name repository* | sort); do - - for i in $(cat $x); do - - collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); - wget $i"format%3Ajson" -O "build/collection-$collection_name.json" - - done - - done - - ;; - - --make-collection-list) - - check_type="collections"; - check_files - - cd build/ || exit - for i in $(cat ../license-reporter-collections); do - - # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson - collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); - wget $i"format%3Ajson" -O "collection-$collection_name.json" - - done - - ;; - - --make-custom-list) - - check_type="custom"; - check_files - - src/download-custom + mkdir -p $RUN_PATH/build ;; @@ -346,6 +250,111 @@ s/,/\n/g; done + ;; + --make-repository-list) + + # It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs. + # This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync + # your addons.mozilla.org collection list with a separate text file. + + # https://github.com/mozilla/addons/issues/722 + # Cannot list collections in desired formats + # Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list). + # So we have to parse it from HTML. + + + # Get the links to the collections + + function get_repositories { + + check_type="repositories"; + check_files + + cd build/ || exit + + for i in $(cat ../license-reporter-repositories); do + + user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;"); + rm -f repository-$user + + + function wget_repository { + + match="^ ||" >> repository-$user + + } + + # Figure out if there are several pages + number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||"); + + if [ -z "$number_of_pages" ]; then + + echo "$i doesn't have any sub pages"; + wget_repository + + else + + echo "$i has sub pages"; + + page_number=1; + + while [ $page_number -le $number_of_pages ] + do + + wget_repository__page_extension="?page=$page_number"; + echo "$wget_repository__page_extension" + wget_repository + page_number=$(( $page_number + 1 )) + + done + + fi + + done + } + get_repositories + + + # Download the JSON files from the collections + + cd .. + for x in $(find build/ -name repository* | sort); do + + for i in $(cat $x); do + + collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); + wget $i"format%3Ajson" -O "build/collection-$collection_name.json" + + done + + done + + ;; + + --make-collection-list) + + check_type="collections"; + check_files + + cd build/ || exit + for i in $(cat ../license-reporter-collections); do + + # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson + collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); + wget $i"format%3Ajson" -O "collection-$collection_name.json" + + done + + ;; + + --make-custom-list) + + check_type="custom"; + check_files + + src/download-custom + ;; --merge-lists) -- cgit v1.2.3