diff options
Diffstat (limited to 'addons.mozilla.org-fsd/get-data/license-reporter')
| -rwxr-xr-x | addons.mozilla.org-fsd/get-data/license-reporter | 144 |
1 files changed, 111 insertions, 33 deletions
diff --git a/addons.mozilla.org-fsd/get-data/license-reporter b/addons.mozilla.org-fsd/get-data/license-reporter index ba92cff..68a05f9 100755 --- a/addons.mozilla.org-fsd/get-data/license-reporter +++ b/addons.mozilla.org-fsd/get-data/license-reporter @@ -1,7 +1,7 @@ #!/usr/bin/env bash # This file is part of license-reporter -# Copyright (C) 2017 David Hedlund +# Copyright (C) 2017, 2018 David Hedlund # # license-reporter is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -47,6 +47,28 @@ if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi if [[ $exit == true ]]; then exit 1; fi +function check_files { + + if [ ! -f "$RUN_PATH/license-reporter-$check_type" ]; then + + echo "$RUN_PATH/license-reporter-$check_type doesn't exist."; + echo "Exiting" && exit + + else + + + # Make sure all links are using SSL + if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-$check_type; then + + echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)." + echo "Exiting" && exit + + fi + + fi + +} + case "$1" in @@ -58,6 +80,10 @@ case "$1" in OPTIONS $0 --all $0 --fresh-build + $0 --make-repository-list + Downloads add-on data from collections shared by defined users in license-reporter-repositories. Example: + https://addons.mozilla.org/en-US/firefox/collections/mozilla/ + $0 --make-collection-list $0 --make-custom-list $0 --make-search-list @@ -87,52 +113,103 @@ OPTIONS # This function requires _much_ less maintainance for custom add-on's, you don't need to manually sync # your addons.mozilla.org collection list with a separate text file. - # Cannot list collections in desired formats - https://github.com/mozilla/addons/issues/722 - - ;; - - --make-collection-list) + # https://github.com/mozilla/addons/issues/722 + # Cannot list collections in desired formats + # Cannot get a list of names of Collections from other users like https://addons.mozilla.org/collections/mozilla/ without authenication (see https://addons-server.readthedocs.io/en/latest/topics/api/collections.html#list). + # So we have to parse it from HTML. + + + # Get the links to the collections - if [ ! -f "$RUN_PATH/license-reporter-collections" ]; then + function get_repositories { + + check_type="repositories"; + check_files - echo "$RUN_PATH/license-reporter-collections doesn't exist. Skipping custom list."; - - else + cd build/ || exit + for i in $(cat ../license-reporter-repositories); do + + user=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/||;"); + rm -f repository-$user - # Make sure all repositories are using SSL - if grep -q "http://addons.mozilla.org/" $RUN_PATH/license-reporter-repository; then - - echo "Change http://addons.mozilla.org/ to https://addons.mozilla.org/ (https)." - - else - cd build/ || exit - for i in $(cat ../license-reporter-collections); do + function wget_repository { - # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson - collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); - wget $i"format%3Ajson" -O "collection-$collection_name.json" + match="^ <a href=\"/en-US/firefox/collections/" + wget -qO- $i$wget_repository__page_extension | grep "$match" | sed "s|$match|https://addons.mozilla.org/en-US/firefox/collections/|; s|\">||" >> repository-$user - done + } - fi - - fi + # Figure out if there are several pages + number_of_pages=$(wget -qO- $i | grep "?page=" | tail -n 2 | head -n 1 | sed "s|page=|\npage=|g; s|\"|\n\"|g" | grep "page=" | sed "s|page=||"); + + if [ -z "$number_of_pages" ]; then + + echo "$i doesn't have any sub pages"; + wget_repository + + else + + echo "$i has sub pages"; + + page_number=1; + + while [ $page_number -le $number_of_pages ] + do + + wget_repository__page_extension="?page=$page_number"; + echo "$wget_repository__page_extension" + wget_repository + page_number=$(( $page_number + 1 )) + + done + + fi + + done + } + get_repositories + + + # Download the JSON files from the collections + + cd .. + for x in $(find build/ -name repository* | sort); do + + for i in $(cat $x); do + + collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); + wget $i"format%3Ajson" -O "build/collection-$collection_name.json" + + done + + done ;; - --make-custom-list) + --make-collection-list) - if [ ! -f "$RUN_PATH/license-reporter-custom" ]; then + check_type="collections"; + check_files + + cd build/ || exit + for i in $(cat ../license-reporter-collections); do + + # https://addons.mozilla.org/en-US/firefox/collections/DavidHedlund/webextensions/format%3Ajson + collection_name=$(echo $i | sed "s|https://addons.mozilla.org/en-US/firefox/collections/||; s|/|-|; s|/$||"); + wget $i"format%3Ajson" -O "collection-$collection_name.json" + + done + + ;; + + --make-custom-list) - echo "$RUN_PATH/license-reporter-custom doesn't exist. Skipping custom list."; - - else + check_type="custom"; + check_files - src/download-custom - - fi + src/download-custom ;; @@ -363,6 +440,7 @@ $count_reports reports to submit"; --all) $0 --fresh-build "$2" + $0 --make-repository-list "$2" $0 --make-collection-list "$2" $0 --make-custom-list "$2" $0 --make-search-list "$2" |