diff options
| -rwxr-xr-x[-rw-r--r--] | doall.sh | 11 | ||||
| -rw-r--r-- | load_changelog.py | 9 | ||||
| -rw-r--r-- | load_copyright.py | 8 |
3 files changed, 14 insertions, 14 deletions
@@ -1,6 +1,7 @@ #!/bin/bash set -e +set -x rm distfiles output *.log *.h5 -rf mkdir distfiles @@ -26,7 +27,7 @@ wget -O - http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2 pv distfiles/Translation-en | python load_descriptions.py if ! [ -d metadata.ftp-master.debian.org/changelogs ]; then -rm downloadlist* -f + rm downloadlist* -f for PACKAGE in $PACKAGES; do LETTER=$(echo $PACKAGE |cut -c1) [ 1$(echo $PACKAGE |cut -c-3) = 1'lib' ] && LETTER=$(echo $PACKAGE |cut -c-4) @@ -34,7 +35,7 @@ rm downloadlist* -f echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_changelog >> downloadlist done -wget -x --continue -i downloadlist + wget -x --continue -i downloadlist for PACKAGE in $PACKAGES; do LETTER=$(echo $PACKAGE |cut -c1) @@ -42,12 +43,12 @@ wget -x --continue -i downloadlist [ -f metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_copyright ] || echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_$PACKAGE.copyright >> downloadlist404 done -wget -x --continue -i downloadlist404 + wget -x --continue -i downloadlist404 fi -python load_copyright.py metadata.ftp-master.debian.org/changelogs/main/*/*/stable_copyright | tee cp_import.log -python load_changelog.py metadata.ftp-master.debian.org/changelogs/main/*/*/stable_changelog | tee cl_import.log +python load_copyright.py &> cp_import.log +python load_changelog.py &> cl_import.log python export.py python export_json.py diff --git a/load_changelog.py b/load_changelog.py index d5dd4d9..eb65772 100644 --- a/load_changelog.py +++ b/load_changelog.py @@ -6,6 +6,7 @@ import pandas as pd import os import sys from dateutil import parser +import subprocess def import_one(pkgname, fh): try: @@ -24,10 +25,9 @@ def get_pkgname(path): else: return base -def main(paths): +def main(): versions = [] - - for path in paths: + for path in subprocess.check_output("find metadata.ftp-master.debian.org -name stable_changelog".split()).strip().split(): pkgname = get_pkgname(path) print pkgname, path data = import_one(pkgname, file(path)) @@ -42,5 +42,4 @@ def main(paths): store.close() if __name__ == '__main__': - main(sys.argv[1:]) - + main() diff --git a/load_copyright.py b/load_copyright.py index f54a3ce..838d907 100644 --- a/load_copyright.py +++ b/load_copyright.py @@ -6,6 +6,7 @@ import sys import debian.deb822 import pandas as pd +import subprocess #from IPython.core import ultratb #sys.excepthook = ultratb.FormattedTB(mode='Verbose', @@ -114,12 +115,12 @@ def get_pkgname(path): else: return base -def main(paths): +def main(): summaries = [] files = [] licenses = [] - for path in paths: + for path in subprocess.check_output("find metadata.ftp-master.debian.org -name stable_copyright".split()).strip().split(): pkgname = get_pkgname(path) print pkgname, path data = import_one(pkgname, file(path)) @@ -149,5 +150,4 @@ def main(paths): store.close() if __name__ == '__main__': - main(sys.argv[1:]) - + main() |