Fix download issues -- Thanks to Ian

1 files changed, 28 insertions, 10 deletions

diff --git a/doall.sh b/doall.sh
index c3825bd..a71d9d7 100755
--- a/doall.sh
+++ b/doall.sh
@@ -7,7 +7,7 @@ set -x
 x="$(readlink -f -- "$BASH_SOURCE")"; cd ${x%/*} # directory of this file
 
 
-rm distfiles output *.log *.h5 -rf
+rm distfiles output *.log *.h5 -rf metadata.ftp-master.debian.org
 mkdir distfiles
 
 wget -O - http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz | gunzip >> distfiles/Sources
@@ -30,26 +30,44 @@ wget -O - http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2
 
 pv distfiles/Translation-en | python load_descriptions.py
 
-if ! [ -d metadata.ftp-master.debian.org/changelogs ]; then
-  rm downloadlist* -f
-  for PACKAGE in $PACKAGES; do
+
+rm downloadlist* -f
+for PACKAGE in $PACKAGES; do
     LETTER=$(echo $PACKAGE |cut -c1)
     [ 1$(echo $PACKAGE |cut -c-3) = 1'lib' ] && LETTER=$(echo $PACKAGE |cut -c-4)
     echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_copyright >> downloadlist
     echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_changelog >> downloadlist
-  done
+done
+
+
+# this may return 8, because debian has removed a package from the time we made downloadlist, to the time we actually did the wget on a url in it
+# so, we do a check to see if we downloaded at least the number of files - 10
+while (( i=0; i<10; i++)); do
+
+    if wget -nc -nv -x --continue -i downloadlist; then
+        
+        break
+        
+    fi
+    
+done
+
+if [[ $(find metadata.ftp-master.debian.org -type f |wc -l) -lt $(( $(wc -l <downloadlist) - 10)) ]]; then
+    
+    echo "$0: error: failed to download enough files";
+    exit 1;
+    
+fi
 
-  wget -x --continue -i downloadlist
 
-  for PACKAGE in $PACKAGES; do
+for PACKAGE in $PACKAGES; do
     LETTER=$(echo $PACKAGE |cut -c1)
     [ 1$(echo $PACKAGE |cut -c-3) = 1'lib' ] && LETTER=$(echo $PACKAGE |cut -c-4)
     [ -f metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_copyright ] || echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_$PACKAGE.copyright >> downloadlist404
-  done
+done
 
-  wget -x --continue -i downloadlist404
+wget -v -x --continue -i downloadlist404
 
-fi
 
 python load_copyright.py &> cp_import.log
 python load_changelog.py &> cl_import.log