From e54837d04682c9b47582b3764d6216b71a334fa9 Mon Sep 17 00:00:00 2001 From: Andrew Harvey Date: Mon, 16 Oct 2023 16:57:50 +1100 Subject: output to gitlab pages --- .gitlab-ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 934ba5d..0a11a57 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,6 +14,7 @@ stages: - conflate - candidate - upload + - pages docker prepare: image: docker:latest @@ -275,3 +276,15 @@ build compareSuburb: needs: ['prepare'] dependencies: - prepare + +pages: + stage: pages + script: + - cp -r www public + - cp dist/conflate/*.changes.json public/data/ + artifacts: + paths: + - public + needs: ['conflate'] + dependencies: + - conflate -- cgit v1.2.3 From 268e6c1dd0d31ecfabc69e1aa21fc0a1a0fba4ca Mon Sep 17 00:00:00 2001 From: Andrew Harvey Date: Mon, 16 Oct 2023 19:01:12 +1100 Subject: conflate now needs gdal-bin to convert results to fgb --- docker/conflate.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/conflate.Dockerfile b/docker/conflate.Dockerfile index 693c4ee..fd6fb49 100644 --- a/docker/conflate.Dockerfile +++ b/docker/conflate.Dockerfile @@ -1,5 +1,5 @@ FROM debian:bullseye-slim -RUN apt-get -y update && apt-get -y install curl wget gnupg make +RUN apt-get -y update && apt-get -y install curl wget gnupg make gdal-bin RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list -- cgit v1.2.3 From a899d8839c2aee1129ed3ceab753f3d3a7f54eb6 Mon Sep 17 00:00:00 2001 From: Andrew Harvey Date: Mon, 16 Oct 2023 19:03:04 +1100 Subject: lint fixes --- www/mrPreview.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/mrPreview.html b/www/mrPreview.html index eb10dd2..c4f312e 100644 --- a/www/mrPreview.html +++ b/www/mrPreview.html @@ -1,4 +1,4 @@ - + -- cgit v1.2.3 From 37afa3063500a97c71896789d125d2ef53c6d661 Mon Sep 17 00:00:00 2001 From: Andrew Harvey Date: Mon, 16 Oct 2023 19:03:15 +1100 Subject: log file --- www/mrPreview.html | 1 + 1 file changed, 1 insertion(+) diff --git a/www/mrPreview.html b/www/mrPreview.html index c4f312e..41d7d99 100644 --- a/www/mrPreview.html +++ b/www/mrPreview.html @@ -65,6 +65,7 @@ let tagAddCount = 0 let tagModifyCount = 0 let tagRemoveCount = 0 + console.log(`Fetch ${q.get('changes') || 'changes.json'}`) fetch(q.get('changes') || 'changes.json') .then(res => res.json()) -- cgit v1.2.3 From 206b89cb0c1f925ecf142d1068c6ecf60088835f Mon Sep 17 00:00:00 2001 From: Andrew Harvey Date: Tue, 17 Oct 2023 10:19:53 +1100 Subject: reduce threads --- .gitlab-ci.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a11a57..56be485 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -97,8 +97,8 @@ prepare: before_script: - b2 authorize-account $B2_KEYID $B2_APPKEY - mkdir -p dist data - - b2 sync --allowEmptySource --noProgress --delete dist b2://vicmap2osm/dist - - b2 sync --allowEmptySource --noProgress --delete data b2://vicmap2osm/data + - b2 sync --threads 4 --allowEmptySource --noProgress --delete dist b2://vicmap2osm/dist + - b2 sync --threads 4 --allowEmptySource --noProgress --delete data b2://vicmap2osm/data cache: <<: *global_cache policy: push @@ -121,7 +121,7 @@ build osm: - mkdir -p data dist - ls data dist - b2 authorize-account $B2_KEYID $B2_APPKEY - - b2 sync --allowEmptySource --noProgress b2://vicmap2osm/ . + - b2 sync --threads 4 --allowEmptySource --noProgress b2://vicmap2osm/ . cache: <<: *global_cache policy: pull-push @@ -142,8 +142,8 @@ build osm: dependencies: - prepare after_script: - - b2 sync --allowEmptySource --noProgress data b2://vicmap2osm/data - - b2 sync --allowEmptySource --noProgress dist b2://vicmap2osm/dist + - b2 sync --threads 4 --allowEmptySource --noProgress data b2://vicmap2osm/data + - b2 sync --threads 4 --allowEmptySource --noProgress dist b2://vicmap2osm/dist build vicmap: stage: build vicmap @@ -152,7 +152,7 @@ build vicmap: - mkdir -p data dist - ls data dist - b2 authorize-account $B2_KEYID $B2_APPKEY - - b2 sync --allowEmptySource --noProgress b2://vicmap2osm/ . + - b2 sync --threads 4 --allowEmptySource --noProgress b2://vicmap2osm/ . cache: <<: *global_cache policy: pull-push @@ -176,8 +176,8 @@ build vicmap: dependencies: - build osm after_script: - - b2 sync --allowEmptySource --noProgress dist b2://vicmap2osm/dist - - b2 sync --allowEmptySource --noProgress debug b2://vicmap2osm/debug + - b2 sync --threads 4 --allowEmptySource --noProgress dist b2://vicmap2osm/dist + - b2 sync --threads 4 --allowEmptySource --noProgress debug b2://vicmap2osm/debug conflate: stage: conflate @@ -186,7 +186,7 @@ conflate: - mkdir -p data dist - ls data dist - b2 authorize-account $B2_KEYID $B2_APPKEY - - b2 sync --allowEmptySource --noProgress b2://vicmap2osm/ . + - b2 sync --threads 4 --allowEmptySource --noProgress b2://vicmap2osm/ . cache: <<: *global_cache policy: pull @@ -204,8 +204,8 @@ conflate: - build vicmap - build osm after_script: - - b2 sync --allowEmptySource --noProgress data b2://vicmap2osm/data - - b2 sync --allowEmptySource --noProgress dist b2://vicmap2osm/dist + - b2 sync --threads 4 --allowEmptySource --noProgress data b2://vicmap2osm/data + - b2 sync --threads 4 --allowEmptySource --noProgress dist b2://vicmap2osm/dist candidate: stage: candidate @@ -214,7 +214,7 @@ candidate: - mkdir -p data dist - ls data dist - b2 authorize-account $B2_KEYID $B2_APPKEY - - b2 sync --allowEmptySource --noProgress b2://vicmap2osm/ . + - b2 sync --threads 4 --allowEmptySource --noProgress b2://vicmap2osm/ . cache: <<: *global_cache policy: pull @@ -227,7 +227,7 @@ candidate: dependencies: - conflate after_script: - - b2 sync --allowEmptySource --noProgress dist b2://vicmap2osm/dist + - b2 sync --threads 4 --allowEmptySource --noProgress dist b2://vicmap2osm/dist upload dev: stage: upload -- cgit v1.2.3 From e3a283b07a8d8275101e6bd702291ee4c87f43cc Mon Sep 17 00:00:00 2001 From: Andrew Harvey Date: Wed, 18 Oct 2023 22:50:18 +1100 Subject: update source vicmap data which was stalled at July 2022 --- .gitlab-ci.yml | 6 +++--- Makefile | 20 ++++++++++---------- README.md | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 56be485..bdfff17 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -106,7 +106,7 @@ prepare: - yarn install - cp src/polygon-lookup-patch.js node_modules/polygon-lookup/index.js - mkdir -p dist - - make data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb + - make data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb - rm -rf data/vicmap.geojson data/VICMAP_ADDRESS.zip - make data/vicmap.geojson after_script: @@ -163,7 +163,7 @@ build vicmap: - mkdir -p dist data - mkdir -p data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria - touch data/VICMAP_ADDRESS.zip - - touch data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb + - touch data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb - touch --no-create data/vicmap.geojson - time make dist/vicmap-osm-with-suburb.geojson - time make dist/vicmap-osm-uniq.geojson @@ -266,7 +266,7 @@ build compareSuburb: - mkdir -p dist data - mkdir -p data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria - touch data/VICMAP_ADDRESS.zip - - touch data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb + - touch data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb - touch --no-create data/vicmap.geojson - time make dist/vicmap-osm-with-suburb.geojson - time make dist/vicmapSuburbDiffersWithOSM.geojson diff --git a/Makefile b/Makefile index 92f716e..d7ff3f4 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,14 @@ # download Vicmap Address source data data/VICMAP_ADDRESS.zip: mkdir -p data - wget --no-verbose --output-document=$@ https://s3-ap-southeast-2.amazonaws.com/cl-isd-prd-datashare-s3-delivery/Order_BGJ5MV.zip + wget --no-verbose --output-document=$@ https://s3.ap-southeast-2.amazonaws.com/cl-isd-prd-datashare-s3-delivery/Order_FDBZT5.zip # download Vicmap Property cadastre (only used for debugging) data/VICMAP_PROPERTY.zip: mkdir -p data - wget --no-verbose --output-document=$@ https://s3-ap-southeast-2.amazonaws.com/cl-isd-prd-datashare-s3-delivery/Order_OTL5B2.zip + wget --no-verbose --output-document=$@ https://s3.ap-southeast-2.amazonaws.com/cl-isd-prd-datashare-s3-delivery/Order_EUKSRV.zip -data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb: data/VICMAP_ADDRESS.zip +data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb: data/VICMAP_ADDRESS.zip mkdir -p data/vicmap unzip -d data/vicmap -n $< # update mtime so that Make doesn't see it as outdated @@ -23,13 +23,13 @@ data/vicmap-property.fgb: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victor ogr2ogr -f FlatGeobuf -t_srs 'EPSG:4326' -nlt PROMOTE_TO_MULTI $@ $< PARCEL_VIEW data/vicmap.geojson: - ogr2ogr -f GeoJSONSeq -t_srs 'EPSG:4326' -mapFieldType DateTime=String $@ data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb + ogr2ogr -f GeoJSONSeq -t_srs 'EPSG:4326' -mapFieldType DateTime=String $@ data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb wc -l $@ # used for quick debugging -# ogr2ogr -f GeoJSONSeq -clipsrc 144.95392 -37.80260 144.97298 -37.79204 data/vicmap.geojson data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb +# ogr2ogr -f GeoJSONSeq -clipsrc 144.95392 -37.80260 144.97298 -37.79204 data/vicmap.geojson data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb vicmapExtract: - ogr2ogr -f GeoJSONSeq -mapFieldType DateTime=String -clipsrc 144.95366 -37.80284 145.00272 -37.77482 data/vicmap.geojson data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb + ogr2ogr -f GeoJSONSeq -mapFieldType DateTime=String -clipsrc 144.95366 -37.80284 145.00272 -37.77482 data/vicmap.geojson data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb cleanDist: rm -rf dist @@ -77,20 +77,20 @@ dist/canidates.geojson: dist/vicmap-osm-uniq-flats-withinrange.geojson loadPgOSM: dist/vicmap-osm.geojson ogr2ogr -f PostgreSQL PG: $< -lco UNLOGGED=YES -nln vm_osm -data/vicmap.fgb: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb +data/vicmap.fgb: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb ogr2ogr -f FlatGeobuf $@ $< dist/vicmap-osm.fgb: dist/vicmap-osm.geojson ogr2ogr -f FlatGeobuf $@ $< # useful for development to be able to query a database -loadPgAdd: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb +loadPgAdd: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb ogr2ogr -f PostgreSQL PG: $< -nln vmadd # index all columns for faster queries during development psql -f src/createIndexQuery.sql --tuples-only | psql -loadPgProp: data/vicmap/ll_gda94/sde_shape/whole/VIC/VMPROP/layer/property_view.shp - ogr2ogr -f PostgreSQL PG: $< -nln vmprop -nlt MULTIPOLYGON +loadPgProp: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMPROP.gdb + ogr2ogr -f PostgreSQL PG: $< -nln vmprop -nlt MULTIPOLYGON PARCEL_VIEW data/victoria.osm.pbf: wget --no-verbose --directory-prefix=data http://download.openstreetmap.fr/extracts/oceania/australia/victoria.osm.pbf diff --git a/README.md b/README.md index 768fa85..e147de6 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ We use GitLab CI/CD to automate data processing. 1. Download source Vicmap data (_prepare_ stage): ```sh -make data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb +make data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb ``` 2. Convert to GeoJSON (_prepare_ stage): -- cgit v1.2.3