fix argument list too long error

3 files changed, 14 insertions, 14 deletions

diff --git a/doall.sh b/doall.sh
index 1eb18d3..ed1c61e 100644..100755
--- a/doall.sh
+++ b/doall.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -e
+set -x
 
 rm distfiles output *.log *.h5 -rf
 mkdir distfiles
@@ -26,7 +27,7 @@ wget -O - http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2
 pv distfiles/Translation-en | python load_descriptions.py
 
 if ! [ -d metadata.ftp-master.debian.org/changelogs ]; then
-rm downloadlist* -f
+  rm downloadlist* -f
   for PACKAGE in $PACKAGES; do
     LETTER=$(echo $PACKAGE |cut -c1)
     [ 1$(echo $PACKAGE |cut -c-3) = 1'lib' ] && LETTER=$(echo $PACKAGE |cut -c-4)
@@ -34,7 +35,7 @@ rm downloadlist* -f
     echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_changelog >> downloadlist
   done
 
-wget  -x --continue -i downloadlist
+  wget  -x --continue -i downloadlist
 
   for PACKAGE in $PACKAGES; do
     LETTER=$(echo $PACKAGE |cut -c1)
@@ -42,12 +43,12 @@ wget  -x --continue -i downloadlist
     [ -f metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_copyright ] || echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_$PACKAGE.copyright >> downloadlist404
   done
 
-wget -x --continue -i downloadlist404
+  wget -x --continue -i downloadlist404
 
 fi
 
-python load_copyright.py metadata.ftp-master.debian.org/changelogs/main/*/*/stable_copyright | tee cp_import.log
-python load_changelog.py metadata.ftp-master.debian.org/changelogs/main/*/*/stable_changelog | tee cl_import.log
+python load_copyright.py &> cp_import.log
+python load_changelog.py &> cl_import.log
 
 python export.py
 python export_json.py
diff --git a/load_changelog.py b/load_changelog.py
index d5dd4d9..eb65772 100644
--- a/load_changelog.py
+++ b/load_changelog.py
@@ -6,6 +6,7 @@ import pandas as pd
 import os
 import sys
 from dateutil import parser
+import subprocess
 
 def import_one(pkgname, fh):
     try:
@@ -24,10 +25,9 @@ def get_pkgname(path):
     else:
         return base
 
-def main(paths):
+def main():
     versions = []
-
-    for path in paths:
+    for path in subprocess.check_output("find metadata.ftp-master.debian.org -name stable_changelog".split()).strip().split():
         pkgname = get_pkgname(path)
         print pkgname, path
         data = import_one(pkgname, file(path))
@@ -42,5 +42,4 @@ def main(paths):
     store.close()
 
 if __name__ == '__main__':
-    main(sys.argv[1:])
-
+    main()
diff --git a/load_copyright.py b/load_copyright.py
index f54a3ce..838d907 100644
--- a/load_copyright.py
+++ b/load_copyright.py
@@ -6,6 +6,7 @@ import sys
 
 import debian.deb822
 import pandas as pd
+import subprocess
 
 #from IPython.core import ultratb
 #sys.excepthook = ultratb.FormattedTB(mode='Verbose',
@@ -114,12 +115,12 @@ def get_pkgname(path):
     else:
         return base
 
-def main(paths):
+def main():
     summaries = []
     files = []
     licenses = []
 
-    for path in paths:
+    for path in subprocess.check_output("find metadata.ftp-master.debian.org -name stable_copyright".split()).strip().split():
         pkgname = get_pkgname(path)
         print pkgname, path
         data = import_one(pkgname, file(path))
@@ -149,5 +150,4 @@ def main(paths):
     store.close()
 
 if __name__ == '__main__':
-    main(sys.argv[1:])
-
+    main()