From c98d2847d77a461356d86a035531eefecd730b86 Mon Sep 17 00:00:00 2001 From: david Date: Fri, 27 Jul 2018 14:28:33 +0200 Subject: renamed doall.sh to doall --- doall | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ doall.sh | 84 ---------------------------------------------------------------- 2 files changed, 84 insertions(+), 84 deletions(-) create mode 100755 doall delete mode 100755 doall.sh diff --git a/doall b/doall new file mode 100755 index 0000000..5a3e48e --- /dev/null +++ b/doall @@ -0,0 +1,84 @@ +#!/bin/bash + +source err +set -x + +x="$(readlink -f -- "$BASH_SOURCE")"; cd ${x%/*} # directory of this file + + +rm -rf distfiles output *.log *.h5 metadata.ftp-master.debian.org +mkdir distfiles + +wget -O - http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz | gunzip >> distfiles/Sources +#wget -O - http://ftp.debian.org/debian/dists/stable-updates/main/source/Sources.gz | gunzip >> distfiles/Sources +#wget -O - http://security.debian.org/debian-security/dists/stable/updates/main/source/Sources.gz | gunzip >> distfiles/Sources + +PACKAGES=$(grep ^Package: distfiles/Sources |sort -u| sed 's/Package: //') +pv distfiles/Sources | python load_sources.py + +wget -O - http://ftp.debian.org/debian/dists/stable/main/binary-all/Packages.gz | gunzip >> distfiles/Packages +wget -O - http://ftp.debian.org/debian/dists/stable/main/binary-amd64/Packages.gz | gunzip >> distfiles/Packages +#wget -O - http://ftp.debian.org/debian/dists/stable-updates/main/binary-amd64/Packages.gz | gunzip >> distfiles/Packages +#wget -O - http://security.debian.org/debian-security/dists/stable/updates/main/binary-all/Packages.gz | gunzip >> distfiles/Packages + +pv distfiles/Packages | python load_packages.py + +wget -O - http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2 | bunzip2 >> distfiles/Translation-en +#wget -O - http://ftp.debian.org/debian/dists/stable-updates/main/i18n/Translation-en.bz2 | bunzip2 >> distfiles/Translation-en +#wget -O - http://security.debian.org/debian-security/dists/stable/updates/main/i18n/Translation-en.bz2 | bunzip2 >> distfiles/Translation-en + +pv distfiles/Translation-en | python load_descriptions.py + + +rm downloadlist* -f +for PACKAGE in $PACKAGES; do + LETTER=$(echo $PACKAGE |cut -c1) + [ 1$(echo $PACKAGE |cut -c-3) = 1'lib' ] && LETTER=$(echo $PACKAGE |cut -c-4) + echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_copyright >> downloadlist + echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_changelog >> downloadlist +done + + +# this may return 8, because debian has removed a package from the time we made downloadlist, to the time we actually did the wget on a url in it +# so, we do a check to see if we downloaded at least the number of files - 10 +for (( i=0; i<10; i++ )); do + + if wget -nc -nv -x --continue -i downloadlist; then + + break + + fi + +done + +# Emailed 490848@bugs.debian.org 2018-07-06 with subject: 30 missing stable_copyright and stable_changelog files -- see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=490848#51 +if [[ $(find metadata.ftp-master.debian.org -type f |wc -l) -lt $(( $(wc -l > downloadlist404 +done + +wget -v -x --continue -i downloadlist404 + + +python load_copyright.py &> cp_import.log +python load_changelog.py &> cl_import.log + +python export.py +python export_json.py + +echo empty files: > broken +find output -type f -empty >> broken +find output -type f -empty -delete + +echo no license: >> broken +grep "Project license" output/* -c |grep :0|sed 's/:0//' >> broken +grep "Project license" output/* -c |grep :0|sed 's/:0//'|xargs rm diff --git a/doall.sh b/doall.sh deleted file mode 100755 index 5a3e48e..0000000 --- a/doall.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -source err -set -x - -x="$(readlink -f -- "$BASH_SOURCE")"; cd ${x%/*} # directory of this file - - -rm -rf distfiles output *.log *.h5 metadata.ftp-master.debian.org -mkdir distfiles - -wget -O - http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz | gunzip >> distfiles/Sources -#wget -O - http://ftp.debian.org/debian/dists/stable-updates/main/source/Sources.gz | gunzip >> distfiles/Sources -#wget -O - http://security.debian.org/debian-security/dists/stable/updates/main/source/Sources.gz | gunzip >> distfiles/Sources - -PACKAGES=$(grep ^Package: distfiles/Sources |sort -u| sed 's/Package: //') -pv distfiles/Sources | python load_sources.py - -wget -O - http://ftp.debian.org/debian/dists/stable/main/binary-all/Packages.gz | gunzip >> distfiles/Packages -wget -O - http://ftp.debian.org/debian/dists/stable/main/binary-amd64/Packages.gz | gunzip >> distfiles/Packages -#wget -O - http://ftp.debian.org/debian/dists/stable-updates/main/binary-amd64/Packages.gz | gunzip >> distfiles/Packages -#wget -O - http://security.debian.org/debian-security/dists/stable/updates/main/binary-all/Packages.gz | gunzip >> distfiles/Packages - -pv distfiles/Packages | python load_packages.py - -wget -O - http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2 | bunzip2 >> distfiles/Translation-en -#wget -O - http://ftp.debian.org/debian/dists/stable-updates/main/i18n/Translation-en.bz2 | bunzip2 >> distfiles/Translation-en -#wget -O - http://security.debian.org/debian-security/dists/stable/updates/main/i18n/Translation-en.bz2 | bunzip2 >> distfiles/Translation-en - -pv distfiles/Translation-en | python load_descriptions.py - - -rm downloadlist* -f -for PACKAGE in $PACKAGES; do - LETTER=$(echo $PACKAGE |cut -c1) - [ 1$(echo $PACKAGE |cut -c-3) = 1'lib' ] && LETTER=$(echo $PACKAGE |cut -c-4) - echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_copyright >> downloadlist - echo http://metadata.ftp-master.debian.org/changelogs/main/$LETTER/$PACKAGE/stable_changelog >> downloadlist -done - - -# this may return 8, because debian has removed a package from the time we made downloadlist, to the time we actually did the wget on a url in it -# so, we do a check to see if we downloaded at least the number of files - 10 -for (( i=0; i<10; i++ )); do - - if wget -nc -nv -x --continue -i downloadlist; then - - break - - fi - -done - -# Emailed 490848@bugs.debian.org 2018-07-06 with subject: 30 missing stable_copyright and stable_changelog files -- see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=490848#51 -if [[ $(find metadata.ftp-master.debian.org -type f |wc -l) -lt $(( $(wc -l > downloadlist404 -done - -wget -v -x --continue -i downloadlist404 - - -python load_copyright.py &> cp_import.log -python load_changelog.py &> cl_import.log - -python export.py -python export_json.py - -echo empty files: > broken -find output -type f -empty >> broken -find output -type f -empty -delete - -echo no license: >> broken -grep "Project license" output/* -c |grep :0|sed 's/:0//' >> broken -grep "Project license" output/* -c |grep :0|sed 's/:0//'|xargs rm -- cgit v1.2.3