Merge remote-tracking branch 'origin/master'

5 files changed, 43 insertions, 29 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 934ba5d..bdfff17 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,6 +14,7 @@ stages:
   - conflate
   - candidate
   - upload
+  - pages
 
 docker prepare:
   image: docker:latest
@@ -96,8 +97,8 @@ prepare:
   before_script:
     - b2 authorize-account $B2_KEYID $B2_APPKEY
     - mkdir -p dist data
-    - b2 sync --allowEmptySource --noProgress --delete dist b2://vicmap2osm/dist
-    - b2 sync --allowEmptySource --noProgress --delete data b2://vicmap2osm/data
+    - b2 sync --threads 4 --allowEmptySource --noProgress --delete dist b2://vicmap2osm/dist
+    - b2 sync --threads 4 --allowEmptySource --noProgress --delete data b2://vicmap2osm/data
   cache:
     <<: *global_cache
     policy: push
@@ -105,7 +106,7 @@ prepare:
     - yarn install
     - cp src/polygon-lookup-patch.js node_modules/polygon-lookup/index.js
     - mkdir -p dist
-    - make data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb
+    - make data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb
     - rm -rf data/vicmap.geojson data/VICMAP_ADDRESS.zip
     - make data/vicmap.geojson
   after_script:
@@ -120,7 +121,7 @@ build osm:
     - mkdir -p data dist
     - ls data dist
     - b2 authorize-account $B2_KEYID $B2_APPKEY
-    - b2 sync --allowEmptySource --noProgress b2://vicmap2osm/ .
+    - b2 sync --threads 4 --allowEmptySource --noProgress b2://vicmap2osm/ .
   cache:
     <<: *global_cache
     policy: pull-push
@@ -141,8 +142,8 @@ build osm:
   dependencies:
     - prepare
   after_script:
-    - b2 sync --allowEmptySource --noProgress data b2://vicmap2osm/data
-    - b2 sync --allowEmptySource --noProgress dist b2://vicmap2osm/dist
+    - b2 sync --threads 4 --allowEmptySource --noProgress data b2://vicmap2osm/data
+    - b2 sync --threads 4 --allowEmptySource --noProgress dist b2://vicmap2osm/dist
 
 build vicmap:
   stage: build vicmap
@@ -151,7 +152,7 @@ build vicmap:
     - mkdir -p data dist
     - ls data dist
     - b2 authorize-account $B2_KEYID $B2_APPKEY
-    - b2 sync --allowEmptySource --noProgress b2://vicmap2osm/ .
+    - b2 sync --threads 4 --allowEmptySource --noProgress b2://vicmap2osm/ .
   cache:
     <<: *global_cache
     policy: pull-push
@@ -162,7 +163,7 @@ build vicmap:
     - mkdir -p dist data
     - mkdir -p data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria
     - touch data/VICMAP_ADDRESS.zip
-    - touch data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb
+    - touch data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb
     - touch --no-create data/vicmap.geojson
     - time make dist/vicmap-osm-with-suburb.geojson
     - time make dist/vicmap-osm-uniq.geojson
@@ -175,8 +176,8 @@ build vicmap:
   dependencies:
     - build osm
   after_script:
-    - b2 sync --allowEmptySource --noProgress dist b2://vicmap2osm/dist
-    - b2 sync --allowEmptySource --noProgress debug b2://vicmap2osm/debug
+    - b2 sync --threads 4 --allowEmptySource --noProgress dist b2://vicmap2osm/dist
+    - b2 sync --threads 4 --allowEmptySource --noProgress debug b2://vicmap2osm/debug
 
 conflate:
   stage: conflate
@@ -185,7 +186,7 @@ conflate:
     - mkdir -p data dist
     - ls data dist
     - b2 authorize-account $B2_KEYID $B2_APPKEY
-    - b2 sync --allowEmptySource --noProgress b2://vicmap2osm/ .
+    - b2 sync --threads 4 --allowEmptySource --noProgress b2://vicmap2osm/ .
   cache:
     <<: *global_cache
     policy: pull
@@ -203,8 +204,8 @@ conflate:
     - build vicmap
     - build osm
   after_script:
-    - b2 sync --allowEmptySource --noProgress data b2://vicmap2osm/data
-    - b2 sync --allowEmptySource --noProgress dist b2://vicmap2osm/dist
+    - b2 sync --threads 4 --allowEmptySource --noProgress data b2://vicmap2osm/data
+    - b2 sync --threads 4 --allowEmptySource --noProgress dist b2://vicmap2osm/dist
 
 candidate:
   stage: candidate
@@ -213,7 +214,7 @@ candidate:
     - mkdir -p data dist
     - ls data dist
     - b2 authorize-account $B2_KEYID $B2_APPKEY
-    - b2 sync --allowEmptySource --noProgress b2://vicmap2osm/ .
+    - b2 sync --threads 4 --allowEmptySource --noProgress b2://vicmap2osm/ .
   cache:
     <<: *global_cache
     policy: pull
@@ -226,7 +227,7 @@ candidate:
   dependencies:
     - conflate
   after_script:
-    - b2 sync --allowEmptySource --noProgress dist b2://vicmap2osm/dist
+    - b2 sync --threads 4 --allowEmptySource --noProgress dist b2://vicmap2osm/dist
 
 upload dev:
   stage: upload
@@ -265,7 +266,7 @@ build compareSuburb:
     - mkdir -p dist data
     - mkdir -p data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria
     - touch data/VICMAP_ADDRESS.zip
-    - touch data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb
+    - touch data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb
     - touch --no-create data/vicmap.geojson
     - time make dist/vicmap-osm-with-suburb.geojson
     - time make dist/vicmapSuburbDiffersWithOSM.geojson
@@ -275,3 +276,15 @@ build compareSuburb:
   needs: ['prepare']
   dependencies:
     - prepare
+
+pages:
+  stage: pages
+  script:
+    - cp -r www public
+    - cp dist/conflate/*.changes.json public/data/
+  artifacts:
+    paths:
+      - public
+  needs: ['conflate']
+  dependencies:
+    - conflate
diff --git a/Makefile b/Makefile
index 263fc5e..45bf34c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,14 +1,14 @@
 # download Vicmap Address source data
 data/VICMAP_ADDRESS.zip:
 	mkdir -p data
-	wget --no-verbose --output-document=$@ https://s3-ap-southeast-2.amazonaws.com/cl-isd-prd-datashare-s3-delivery/Order_BGJ5MV.zip
+	wget --no-verbose --output-document=$@ https://s3.ap-southeast-2.amazonaws.com/cl-isd-prd-datashare-s3-delivery/Order_FDBZT5.zip
 
 # download Vicmap Property cadastre (only used for debugging)
 data/VICMAP_PROPERTY.zip:
 	mkdir -p data
-	wget --no-verbose --output-document=$@ https://s3-ap-southeast-2.amazonaws.com/cl-isd-prd-datashare-s3-delivery/Order_OTL5B2.zip
+	wget --no-verbose --output-document=$@ https://s3.ap-southeast-2.amazonaws.com/cl-isd-prd-datashare-s3-delivery/Order_EUKSRV.zip
 
-data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb: data/VICMAP_ADDRESS.zip
+data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb: data/VICMAP_ADDRESS.zip
 	mkdir -p data/vicmap
 	unzip -d data/vicmap -n $<
 	# update mtime so that Make doesn't see it as outdated
@@ -22,15 +22,15 @@ data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMPROP.gdb: data/VICMAP
 data/vicmap-property.fgb: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMPROP.gdb
 	ogr2ogr -f FlatGeobuf -t_srs 'EPSG:4326' -nlt PROMOTE_TO_MULTI $@ $< PARCEL_VIEW
 
-data/vicmap-original.geojson: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb
+data/vicmap-original.geojson: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb
 	ogr2ogr -f GeoJSONSeq -t_srs 'EPSG:4326' -mapFieldType DateTime=String $@ $<
 	wc -l $@
 
-data/vicmap-debug.geojson: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb
+data/vicmap-debug.geojson: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb
 	ogr2ogr -f GeoJSONSeq -clipsrc 144.95392 -37.80260 144.97298 -37.79204 $@ $<
 	wc -l $@
 
-data/vicmap-extracted.geojson: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb
+data/vicmap-extracted.geojson: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb
 	ogr2ogr -f GeoJSONSeq -mapFieldType DateTime=String -clipsrc 144.95366 -37.80284 145.00272 -37.77482 $@ $^
 	wc -l $@
 
@@ -94,20 +94,20 @@ dist/canidates.geojson: dist/vicmap-osm-uniq-flats-withinrange.geojson
 loadPgOSM: dist/vicmap-osm.geojson
 	ogr2ogr -f PostgreSQL PG: $< -lco UNLOGGED=YES -nln vm_osm
 
-data/vicmap.fgb: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb
+data/vicmap.fgb: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb
 	ogr2ogr -f FlatGeobuf $@ $<
 
 dist/vicmap-osm.fgb: dist/vicmap-osm.geojson
 	ogr2ogr -f FlatGeobuf $@ $<
 
 # useful for development to be able to query a database
-loadPgAdd: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb
+loadPgAdd: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb
 	ogr2ogr -f PostgreSQL PG: $< -nln vmadd
 	# index all columns for faster queries during development
 	psql -f src/createIndexQuery.sql --tuples-only | psql
 
-loadPgProp: data/vicmap/ll_gda94/sde_shape/whole/VIC/VMPROP/layer/property_view.shp
-	ogr2ogr -f PostgreSQL PG: $< -nln vmprop -nlt MULTIPOLYGON
+loadPgProp: data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMPROP.gdb
+	ogr2ogr -f PostgreSQL PG: $< -nln vmprop -nlt MULTIPOLYGON PARCEL_VIEW
 
 data/victoria.osm.pbf:
 	wget --no-verbose --directory-prefix=data http://download.openstreetmap.fr/extracts/oceania/australia/victoria.osm.pbf
diff --git a/README.md b/README.md
index 84e839c..f54e9f5 100644
--- a/README.md
+++ b/README.md
@@ -27,7 +27,7 @@ We use GitLab CI/CD to automate data processing.
 1. Download source Vicmap data (_prepare_ stage):
 
 ```sh
-make data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VICMAP_ADDRESS.gdb
+make data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb
 ```
 
 2. Convert to GeoJSON (_prepare_ stage):
diff --git a/docker/conflate.Dockerfile b/docker/conflate.Dockerfile
index 693c4ee..fd6fb49 100644
--- a/docker/conflate.Dockerfile
+++ b/docker/conflate.Dockerfile
@@ -1,5 +1,5 @@
 FROM debian:bullseye-slim
-RUN apt-get -y update && apt-get -y install curl wget gnupg make
+RUN apt-get -y update && apt-get -y install curl wget gnupg make gdal-bin
 RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash -
 RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
 RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
diff --git a/www/mrPreview.html b/www/mrPreview.html
index eb10dd2..41d7d99 100644
--- a/www/mrPreview.html
+++ b/www/mrPreview.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html>                                                                                                        
+<!DOCTYPE html>
 <html>
 <head>
     <meta charset='utf-8' />
@@ -65,6 +65,7 @@
     let tagAddCount = 0
     let tagModifyCount = 0
     let tagRemoveCount = 0
+    console.log(`Fetch ${q.get('changes') || 'changes.json'}`)
 
     fetch(q.get('changes') || 'changes.json')
         .then(res => res.json())