aboutsummaryrefslogtreecommitdiff
path: root/addons.mozilla.org-fsd/get-data/license-reporter
diff options
context:
space:
mode:
Diffstat (limited to 'addons.mozilla.org-fsd/get-data/license-reporter')
-rwxr-xr-xaddons.mozilla.org-fsd/get-data/license-reporter144
1 files changed, 111 insertions, 33 deletions
diff --git a/addons.mozilla.org-fsd/get-data/license-reporter b/addons.mozilla.org-fsd/get-data/license-reporter
index ba92cff..68a05f9 100755
--- a/addons.mozilla.org-fsd/get-data/license-reporter
+++ b/addons.mozilla.org-fsd/get-data/license-reporter
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
# This file is part of license-reporter
-# Copyright (C) 2017 David Hedlund
+# Copyright (C) 2017, 2018 David Hedlund
#
# license-reporter is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -47,6 +47,28 @@ if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi
if [[ $exit == true ]]; then exit 1; fi
+function check_files {
+
+ if [ ! -f "$RUN_PATH/license-reporter-$check_type" ]; then
+
+ echo "$RUN_PATH/license-reporter-$check_type doesn't exist.";
+ echo "Exiting" && exit
+
+ else
+
+
+ # Make sure all links are using SSL
+ if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-$check_type; then
+
+ echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)."
+ echo "Exiting" && exit
+
+ fi
+
+ fi
+
+}
+
case "$1" in
@@ -58,6 +80,10 @@ case "$1" in
OPTIONS
$0 --all
$0 --fresh-build
+ $0 --make-repository-list
+ Downloads add-on data from collections shared by defined users in license-reporter-repositories. Example:
+ https://addons.mozilla.org/en-US/firefox/collections/mozilla/
+
$0 --make-collection-list
$0 --make-custom-list
$0 --make-search-list
@@ -87,52 +113,103 @@ OPTIONS
# This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync
# your addons.mozilla.org collection list with a separate text file.
- # Cannot list collections in desired formats - https://github.com/mozilla/addons/issues/722
-
- ;;
-
- --make-collection-list)
+ # https://github.com/mozilla/addons/issues/722
+ # Cannot list collections in desired formats
+ # Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list).
+ # So we have to parse it from HTML.
+
+
+ # Get the links to the collections
- if [ ! -f "$RUN_PATH/license-reporter-collections" ]; then
+ function get_repositories {
+
+ check_type="repositories";
+ check_files
- echo "$RUN_PATH/license-reporter-collections doesn't exist. Skipping custom list.";
-
- else
+ cd build/ || exit
+ for i in $(cat ../license-reporter-repositories); do
+
+ user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;");
+ rm -f repository-$user
- # Make sure all repositories are using SSL
- if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-repository; then
-
- echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)."
-
- else
- cd build/ || exit
- for i in $(cat ../license-reporter-collections); do
+ function wget_repository {
- # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson
- collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||");
- wget $i"format%3Ajson" -O "collection-$collection_name.json"
+ match="^ <a href=\"/en-US/firefox/collections/"
+ wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user
- done
+ }
- fi
-
- fi
+ # Figure out if there are several pages
+ number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||");
+
+ if [ -z "$number_of_pages" ]; then
+
+ echo "$i doesn't have any sub pages";
+ wget_repository
+
+ else
+
+ echo "$i has sub pages";
+
+ page_number=1;
+
+ while [ $page_number -le $number_of_pages ]
+ do
+
+ wget_repository__page_extension="?page=$page_number";
+ echo "$wget_repository__page_extension"
+ wget_repository
+ page_number=$(( $page_number + 1 ))
+
+ done
+
+ fi
+
+ done
+ }
+ get_repositories
+
+
+ # Download the JSON files from the collections
+
+ cd ..
+ for x in $(find build/ -name repository* | sort); do
+
+ for i in $(cat $x); do
+
+ collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||");
+ wget $i"format%3Ajson" -O "build/collection-$collection_name.json"
+
+ done
+
+ done
;;
- --make-custom-list)
+ --make-collection-list)
- if [ ! -f "$RUN_PATH/license-reporter-custom" ]; then
+ check_type="collections";
+ check_files
+
+ cd build/ || exit
+ for i in $(cat ../license-reporter-collections); do
+
+ # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson
+ collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||");
+ wget $i"format%3Ajson" -O "collection-$collection_name.json"
+
+ done
+
+ ;;
+
+ --make-custom-list)
- echo "$RUN_PATH/license-reporter-custom doesn't exist. Skipping custom list.";
-
- else
+ check_type="custom";
+ check_files
- src/download-custom
-
- fi
+ src/download-custom
;;
@@ -363,6 +440,7 @@ $count_reports reports to submit";
--all)
$0 --fresh-build "$2"
+ $0 --make-repository-list "$2"
$0 --make-collection-list "$2"
$0 --make-custom-list "$2"
$0 --make-search-list "$2"