aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xdoall.sh38
1 files changed, 28 insertions, 10 deletions
diff --git a/doall.sh b/doall.sh
index c3825bd..a71d9d7 100755
--- a/doall.sh
+++ b/doall.sh
@@ -7,7 +7,7 @@ set -x
x="$(readlink -f -- "$BASH_SOURCE")"; cd ${x%/*} # directory of this file
-rm distfiles output *.log *.h5 -rf
+rm distfiles output *.log *.h5 -rf metadata.ftp-master.debian.org
mkdir distfiles
wget -O - http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz | gunzip >> distfiles/Sources
@@ -30,26 +30,44 @@ wget -O - http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2
pv distfiles/Translation-en | python load_descriptions.py
-if ! [ -d metadata.ftp-master.debian.org/changelogs ]; then
- rm downloadlist* -f
- for PACKAGE in $PACKAGES; do
+
+rm downloadlist* -f
+for PACKAGE in $PACKAGES; do
LETTER=$(echo $PACKAGE |cut -c1)
[ 1$(echo $PACKAGE |cut -c-3) = 1'lib' ] && LETTER=$(echo $PACKAGE |cut -c-4)
echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_copyright >> downloadlist
echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_changelog >> downloadlist
- done
+done
+
+
+# this may return 8, because debian has removed a package from the time we made downloadlist, to the time we actually did the wget on a url in it
+# so, we do a check to see if we downloaded at least the number of files - 10
+while (( i=0; i<10; i++)); do
+
+ if wget -nc -nv -x --continue -i downloadlist; then
+
+ break
+
+ fi
+
+done
+
+if [[ $(find metadata.ftp-master.debian.org -type f |wc -l) -lt $(( $(wc -l <downloadlist) - 10)) ]]; then
+
+ echo "$0: error: failed to download enough files";
+ exit 1;
+
+fi
- wget -x --continue -i downloadlist
- for PACKAGE in $PACKAGES; do
+for PACKAGE in $PACKAGES; do
LETTER=$(echo $PACKAGE |cut -c1)
[ 1$(echo $PACKAGE |cut -c-3) = 1'lib' ] && LETTER=$(echo $PACKAGE |cut -c-4)
[ -f metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_copyright ] || echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_$PACKAGE.copyright >> downloadlist404
- done
+done
- wget -x --continue -i downloadlist404
+wget -v -x --continue -i downloadlist404
-fi
python load_copyright.py &> cp_import.log
python load_changelog.py &> cl_import.log