aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-x[-rw-r--r--]doall.sh11
-rw-r--r--load_changelog.py9
-rw-r--r--load_copyright.py8
3 files changed, 14 insertions, 14 deletions
diff --git a/doall.sh b/doall.sh
index 1eb18d3..ed1c61e 100644..100755
--- a/doall.sh
+++ b/doall.sh
@@ -1,6 +1,7 @@
#!/bin/bash
set -e
+set -x
rm distfiles output *.log *.h5 -rf
mkdir distfiles
@@ -26,7 +27,7 @@ wget -O - http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2
pv distfiles/Translation-en | python load_descriptions.py
if ! [ -d metadata.ftp-master.debian.org/changelogs ]; then
-rm downloadlist* -f
+ rm downloadlist* -f
for PACKAGE in $PACKAGES; do
LETTER=$(echo $PACKAGE |cut -c1)
[ 1$(echo $PACKAGE |cut -c-3) = 1'lib' ] && LETTER=$(echo $PACKAGE |cut -c-4)
@@ -34,7 +35,7 @@ rm downloadlist* -f
echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_changelog >> downloadlist
done
-wget -x --continue -i downloadlist
+ wget -x --continue -i downloadlist
for PACKAGE in $PACKAGES; do
LETTER=$(echo $PACKAGE |cut -c1)
@@ -42,12 +43,12 @@ wget -x --continue -i downloadlist
[ -f metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_copyright ] || echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_$PACKAGE.copyright >> downloadlist404
done
-wget -x --continue -i downloadlist404
+ wget -x --continue -i downloadlist404
fi
-python load_copyright.py metadata.ftp-master.debian.org/changelogs/main/*/*/stable_copyright | tee cp_import.log
-python load_changelog.py metadata.ftp-master.debian.org/changelogs/main/*/*/stable_changelog | tee cl_import.log
+python load_copyright.py &> cp_import.log
+python load_changelog.py &> cl_import.log
python export.py
python export_json.py
diff --git a/load_changelog.py b/load_changelog.py
index d5dd4d9..eb65772 100644
--- a/load_changelog.py
+++ b/load_changelog.py
@@ -6,6 +6,7 @@ import pandas as pd
import os
import sys
from dateutil import parser
+import subprocess
def import_one(pkgname, fh):
try:
@@ -24,10 +25,9 @@ def get_pkgname(path):
else:
return base
-def main(paths):
+def main():
versions = []
-
- for path in paths:
+ for path in subprocess.check_output("find metadata.ftp-master.debian.org -name stable_changelog".split()).strip().split():
pkgname = get_pkgname(path)
print pkgname, path
data = import_one(pkgname, file(path))
@@ -42,5 +42,4 @@ def main(paths):
store.close()
if __name__ == '__main__':
- main(sys.argv[1:])
-
+ main()
diff --git a/load_copyright.py b/load_copyright.py
index f54a3ce..838d907 100644
--- a/load_copyright.py
+++ b/load_copyright.py
@@ -6,6 +6,7 @@ import sys
import debian.deb822
import pandas as pd
+import subprocess
#from IPython.core import ultratb
#sys.excepthook = ultratb.FormattedTB(mode='Verbose',
@@ -114,12 +115,12 @@ def get_pkgname(path):
else:
return base
-def main(paths):
+def main():
summaries = []
files = []
licenses = []
- for path in paths:
+ for path in subprocess.check_output("find metadata.ftp-master.debian.org -name stable_copyright".split()).strip().split():
pkgname = get_pkgname(path)
print pkgname, path
data = import_one(pkgname, file(path))
@@ -149,5 +150,4 @@ def main(paths):
store.close()
if __name__ == '__main__':
- main(sys.argv[1:])
-
+ main()