aboutsummaryrefslogtreecommitdiff
path: root/addons.mozilla.org-fsd/get-data
diff options
context:
space:
mode:
authordavid <public@beloved.name>2018-06-21 15:18:26 +0200
committerdavid <public@beloved.name>2018-06-21 15:18:26 +0200
commit7019dc9dcc6aa13505aa9e27c6b99796d7d695bb (patch)
treec2d55499e6041845941f070fcde314374b0e59cc /addons.mozilla.org-fsd/get-data
parent222f7c16acc41bc94b3acafbe719442c8131a73d (diff)
Added documentation for the options in addons.mozilla.org-fsd/get-data/license-reporter
Diffstat (limited to 'addons.mozilla.org-fsd/get-data')
-rwxr-xr-xaddons.mozilla.org-fsd/get-data/license-reporter237
1 files changed, 123 insertions, 114 deletions
diff --git a/addons.mozilla.org-fsd/get-data/license-reporter b/addons.mozilla.org-fsd/get-data/license-reporter
index 3a73598..47c28a8 100755
--- a/addons.mozilla.org-fsd/get-data/license-reporter
+++ b/addons.mozilla.org-fsd/get-data/license-reporter
@@ -92,20 +92,30 @@ case "$1" in
[ "$1" = "" ] && echo "Usage: $filename [--option] [--debug]
OPTIONS
- $0 --all
$0 --fresh-build
+ Remove the build directory and create a new one.
+ $0 --make-search-list
+ Downloads the most popular WebExtensions from https://addons.mozilla.org/en-US/firefox/search/...
$0 --make-repository-list
- Downloads add-on data for free add-ons from collections shared by defined users in license-reporter-repositories. Example:
- https://addons.mozilla.org/en-US/firefox/collections/mozilla/
+ Required file: license-reporter-collections
+ File content syntax: https://addons.mozilla.org/en-US/firefox/collections/<user>/
$0 --make-collection-list
+ Required file: license-reporter-collections
+ File content syntax: https://addons.mozilla.org/en-US/firefox/collections/<user>/<collection name>
$0 --make-custom-list
- $0 --make-search-list
- $0 --get-licenses
+ Required file: license-reporter-custom
+ File content syntax: https://addons.mozilla.org/en-US/firefox/addon/<add-on name>/
$0 --merge-lists
- $0 --generate-proposed-list
+ Merge the lists to MERGED-ALL.txt
+ $0 --get-licenses
+ The licenses are not added to the general JSON file. This option will download the neccesary license JSON files for all add-ons.
$0 --download-licenses-json
$0 --download-free-webextensions-for-gnu-and-linux
$0 --verify-license-copy
+ $0 --merged-free
+ Generates MERGED-ALL-freedom_status.txt
+ $0 --make-wiki
+ Generates .wiki files
--all-recommended
$0 --fresh-build
@@ -123,117 +133,11 @@ OPTIONS
if [ -d "build" ]; then
- rm -fr build
+ rm -fr $RUN_PATH/build
fi
- mkdir -p build
-
- ;;
-
- --make-repository-list)
-
- # It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs.
- # This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync
- # your addons.mozilla.org collection list with a separate text file.
-
- # https://github.com/mozilla/addons/issues/722
- # Cannot list collections in desired formats
- # Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list).
- # So we have to parse it from HTML.
-
-
- # Get the links to the collections
-
- function get_repositories {
-
- check_type="repositories";
- check_files
-
- cd build/ || exit
-
- for i in $(cat ../license-reporter-repositories); do
-
- user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;");
- rm -f repository-$user
-
-
- function wget_repository {
-
- match="^ <a href=\"/en-US/firefox/collections/"
- wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user
-
- }
-
- # Figure out if there are several pages
- number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||");
-
- if [ -z "$number_of_pages" ]; then
-
- echo "$i doesn't have any sub pages";
- wget_repository
-
- else
-
- echo "$i has sub pages";
-
- page_number=1;
-
- while [ $page_number -le $number_of_pages ]
- do
-
- wget_repository__page_extension="?page=$page_number";
- echo "$wget_repository__page_extension"
- wget_repository
- page_number=$(( $page_number + 1 ))
-
- done
-
- fi
-
- done
- }
- get_repositories
-
-
- # Download the JSON files from the collections
-
- cd ..
- for x in $(find build/ -name repository* | sort); do
-
- for i in $(cat $x); do
-
- collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||");
- wget $i"format%3Ajson" -O "build/collection-$collection_name.json"
-
- done
-
- done
-
- ;;
-
- --make-collection-list)
-
- check_type="collections";
- check_files
-
- cd build/ || exit
- for i in $(cat ../license-reporter-collections); do
-
- # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson
- collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||");
- wget $i"format%3Ajson" -O "collection-$collection_name.json"
-
- done
-
- ;;
-
- --make-custom-list)
-
- check_type="custom";
- check_files
-
- src/download-custom
+ mkdir -p $RUN_PATH/build
;;
@@ -347,6 +251,111 @@ s/,/\n/g;
done
;;
+ --make-repository-list)
+
+ # It is ok to distribute only-free programs on addons.mozilla.org even if they distribute nonfree programs.
+ # This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync
+ # your addons.mozilla.org collection list with a separate text file.
+
+ # https://github.com/mozilla/addons/issues/722
+ # Cannot list collections in desired formats
+ # Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list).
+ # So we have to parse it from HTML.
+
+
+ # Get the links to the collections
+
+ function get_repositories {
+
+ check_type="repositories";
+ check_files
+
+ cd build/ || exit
+
+ for i in $(cat ../license-reporter-repositories); do
+
+ user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;");
+ rm -f repository-$user
+
+
+ function wget_repository {
+
+ match="^ <a href=\"/en-US/firefox/collections/"
+ wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user
+
+ }
+
+ # Figure out if there are several pages
+ number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||");
+
+ if [ -z "$number_of_pages" ]; then
+
+ echo "$i doesn't have any sub pages";
+ wget_repository
+
+ else
+
+ echo "$i has sub pages";
+
+ page_number=1;
+
+ while [ $page_number -le $number_of_pages ]
+ do
+
+ wget_repository__page_extension="?page=$page_number";
+ echo "$wget_repository__page_extension"
+ wget_repository
+ page_number=$(( $page_number + 1 ))
+
+ done
+
+ fi
+
+ done
+ }
+ get_repositories
+
+
+ # Download the JSON files from the collections
+
+ cd ..
+ for x in $(find build/ -name repository* | sort); do
+
+ for i in $(cat $x); do
+
+ collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||");
+ wget $i"format%3Ajson" -O "build/collection-$collection_name.json"
+
+ done
+
+ done
+
+ ;;
+
+ --make-collection-list)
+
+ check_type="collections";
+ check_files
+
+ cd build/ || exit
+ for i in $(cat ../license-reporter-collections); do
+
+ # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson
+ collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||");
+ wget $i"format%3Ajson" -O "collection-$collection_name.json"
+
+ done
+
+ ;;
+
+ --make-custom-list)
+
+ check_type="custom";
+ check_files
+
+ src/download-custom
+
+ ;;
--merge-lists)