diff options
| -rwxr-xr-x | doall.sh | 38 |
1 files changed, 28 insertions, 10 deletions
@@ -7,7 +7,7 @@ set -x x="$(readlink -f -- "$BASH_SOURCE")"; cd ${x%/*} # directory of this file -rm distfiles output *.log *.h5 -rf +rm distfiles output *.log *.h5 -rf metadata.ftp-master.debian.org mkdir distfiles wget -O - http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz | gunzip >> distfiles/Sources @@ -30,26 +30,44 @@ wget -O - http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2 pv distfiles/Translation-en | python load_descriptions.py -if ! [ -d metadata.ftp-master.debian.org/changelogs ]; then - rm downloadlist* -f - for PACKAGE in $PACKAGES; do + +rm downloadlist* -f +for PACKAGE in $PACKAGES; do LETTER=$(echo $PACKAGE |cut -c1) [ 1$(echo $PACKAGE |cut -c-3) = 1'lib' ] && LETTER=$(echo $PACKAGE |cut -c-4) echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_copyright >> downloadlist echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_changelog >> downloadlist - done +done + + +# this may return 8, because debian has removed a package from the time we made downloadlist, to the time we actually did the wget on a url in it +# so, we do a check to see if we downloaded at least the number of files - 10 +while (( i=0; i<10; i++)); do + + if wget -nc -nv -x --continue -i downloadlist; then + + break + + fi + +done + +if [[ $(find metadata.ftp-master.debian.org -type f |wc -l) -lt $(( $(wc -l <downloadlist) - 10)) ]]; then + + echo "$0: error: failed to download enough files"; + exit 1; + +fi - wget -x --continue -i downloadlist - for PACKAGE in $PACKAGES; do +for PACKAGE in $PACKAGES; do LETTER=$(echo $PACKAGE |cut -c1) [ 1$(echo $PACKAGE |cut -c-3) = 1'lib' ] && LETTER=$(echo $PACKAGE |cut -c-4) [ -f metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_copyright ] || echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_$PACKAGE.copyright >> downloadlist404 - done +done - wget -x --continue -i downloadlist404 +wget -v -x --continue -i downloadlist404 -fi python load_copyright.py &> cp_import.log python load_changelog.py &> cl_import.log |