refactor pipeline stages

6 files changed, 65 insertions, 30 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d868db7..c491a86 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,12 +6,15 @@ cache: &global_cache
     - .yarn
     - node_modules
     - data/vicmap.geojson
+    - dist/vicmap-osm-uniq-flats.geojson
+    - data/victoria-addr.osm.geojson
+    - dist/blocksByOSMAddr.geojson
 
 stages:
   - docker
   - prepare
   - build
-  - build
+  - conflate
 
 docker prepare:
   image: docker:latest
@@ -28,7 +31,7 @@ docker prepare:
   when: manual
   cache: {}
 
-docker build:
+docker build-vicmap:
   image: docker:latest
   services:
     - docker:dind
@@ -36,8 +39,23 @@ docker build:
   before_script:
     - echo "$CI_JOB_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
   script:
-    - docker build --pull -t "$CI_REGISTRY_IMAGE":build -f "docker/build.Dockerfile" .
-    - docker push "$CI_REGISTRY_IMAGE":build
+    - docker build --pull -t "$CI_REGISTRY_IMAGE":build-vicmap -f "docker/build-vicmap.Dockerfile" .
+    - docker push "$CI_REGISTRY_IMAGE":build-vicmap
+  after_script:
+    - docker logout $CI_REGISTRY
+  when: manual
+  cache: {}
+
+docker build-osm:
+  image: docker:latest
+  services:
+    - docker:dind
+  stage: docker
+  before_script:
+    - echo "$CI_JOB_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
+  script:
+    - docker build --pull -t "$CI_REGISTRY_IMAGE":build-osm -f "docker/build-osm.Dockerfile" .
+    - docker push "$CI_REGISTRY_IMAGE":build-osm
   after_script:
     - docker logout $CI_REGISTRY
   when: manual
@@ -70,12 +88,12 @@ prepare:
     - make data/vicmap.geojson
   when: manual
 
-build:
+build vicmap:
   stage: build
-  image: "registry.gitlab.com/alantgeo/vicmap2osm:build"
+  image: "registry.gitlab.com/alantgeo/vicmap2osm:build-vicmap"
   cache:
     <<: *global_cache
-    policy: pull
+    policy: pull-push
   script:
     - yarn install
     - yarn run test
@@ -90,15 +108,17 @@ build:
     - make dist/vicmap-osm-uniq-flats-withinrange.geojson
   when: manual
   artifacts:
-    name: "build"
+    name: "build vicmap"
     paths:
       - dist
       - debug
 
 build osm:
   stage: build
-  image: "registry.gitlab.com/alantgeo/vicmap2osm:conflate"
-  cache: {}
+  image: "registry.gitlab.com/alantgeo/vicmap2osm:build-osm"
+  cache:
+    <<: *global_cache
+    policy: pull-push
   variables:
     QT_QPA_PLATFORM: "offscreen"
   script:
@@ -111,3 +131,19 @@ build osm:
     name: "build osm"
     paths:
       - dist
+
+conflate:
+  stage: conflate
+  image: "registry.gitlab.com/alantgeo/vicmap2osm:conflate"
+  cache:
+    <<: *global_cache
+    policy: pull
+  script:
+    - yarn install
+    - mkdir -p dist
+    - make dist/conflate
+  when: manual
+  artifacts:
+    name: "conflate"
+    paths:
+      - dist
diff --git a/Makefile b/Makefile
index 23a19e7..6fcca58 100644
--- a/Makefile
+++ b/Makefile
@@ -132,5 +132,5 @@ summariseBlocksByOSMAddr:
 	ogrinfo -dialect sqlite -sql 'select count(*), NUMPOINTS = 0 from blocksByOSMAddr group by (NUMPOINTS = 0)' data/blocksByOSMAddr.fgb
 
 # conflate processed vicmap data with osm data
-conflate:  dist/vicmap-osm-uniq-flats.geojson data/victoria-addr.osm.geojson dist/blocksByOSMAddr.geojson
+dist/conflate: dist/vicmap-osm-uniq-flats.geojson data/victoria-addr.osm.geojson dist/blocksByOSMAddr.geojson
 	./bin/conflate.js $^ $@
diff --git a/bin/conflate.js b/bin/conflate.js
index ff736cd..7b7f09b 100755
--- a/bin/conflate.js
+++ b/bin/conflate.js
@@ -10,21 +10,17 @@ const ndjson = require('ndjson')
 const PolygonLookup = require('polygon-lookup')
 
 const argv = require('yargs/yargs')(process.argv.slice(2))
-  .option('debug', {
-    type: 'boolean',
-    description: 'Dumps full debug logs'
-  })
   .argv
 
 if (argv._.length < 4) {
-  console.error("Usage: ./conflate.js vicmap.geojson osm.geojson blocksByOSMAddr.geojson output.geojson")
+  console.error("Usage: ./conflate.js vicmap.geojson osm.geojson blocksByOSMAddr.geojson dist/conflate")
   process.exit(1)
 }
 
 const vicmapFile = argv._[0]
 const osmFile = argv._[1]
 const blocksByOSMAddrFile = argv._[2]
-const outputFile = argv._[3]
+const outputPath = argv._[3]
 
 if (!fs.existsSync(vicmapFile)) {
   console.error(`${vicmapFile} not found`)
@@ -45,7 +41,6 @@ const blocksByOSMAddr = fs.readFileSync(blocksByOSMAddrFile, 'utf-8').toString()
     try {
       const feature = JSON.parse(line)
       feature.id = index + 1
-      // console.log(feature)
       return feature
     } catch {
       console.log(`Error parsing line ${index} of ${blocksByOSMAddrFile}: ${line}`)
@@ -75,7 +70,6 @@ const filterOSMAddrPoly = new Transform({
   transform(feature, encoding, callback) {
     osmAddrCount++
 
-    console.log(feature)
     if (process.stdout.isTTY && osmAddrCount % 10000 === 0) {
       process.stdout.write(` ${osmAddrCount / 1000}k\r`)
     }
@@ -103,14 +97,12 @@ const filterOSMAddrPoly = new Transform({
       }
     }
 
-    // pass through for further processing
-    this.push(feature)
-
     callback()
   }
 })
 
 // conflate vicmap addresses with OSM addresses
+let sourceCount = 0
 const conflate = new Transform({
   readableObjectMode: true,
   writableObjectMode: true,
@@ -176,7 +168,7 @@ const outputStreamOutputs = {}
 
 outputKeys.forEach(key => {
   outputStreams[key] = ndjson.stringify()
-  outputStreamOutputs[key] = outputStreams[key].pipe(fs.createWriteStream(`debug/conflate/${key}.geojson`))
+  outputStreamOutputs[key] = outputStreams[key].pipe(fs.createWriteStream(`${outputPath}/${key}.geojson`))
 })
 
 // first pass to index by geometry
@@ -215,7 +207,7 @@ pipeline(
             Promise.all(outputKeys.map(key => {
               return new Promise(resolve => {
                 outputStreamOutputs[key].on('finish', () => {
-                  console.log(`saved debug/conflate/${key}.geojson`)
+                  console.log(`saved ${outputPath}/${key}.geojson`)
                   resolve()
                 })
               })
diff --git a/docker/build-osm.Dockerfile b/docker/build-osm.Dockerfile
new file mode 100644
index 0000000..c557fe3
--- /dev/null
+++ b/docker/build-osm.Dockerfile
@@ -0,0 +1,10 @@
+FROM debian:unstable-slim
+RUN apt-get -y update && apt-get -y install curl gnupg make wget unzip gdal-bin osmium-tool software-properties-common
+RUN curl -fsSL https://deb.nodesource.com/setup_12.x | bash -
+RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+RUN apt-get -y update && apt-get -y install nodejs yarn
+RUN wget -qO - https://qgis.org/downloads/qgis-2020.gpg.key | gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/qgis-archive.gpg --import
+RUN chmod a+r /etc/apt/trusted.gpg.d/qgis-archive.gpg
+RUN add-apt-repository "deb https://qgis.org/debian unstable main"
+RUN apt-get -y update && apt-get -y install qgis
diff --git a/docker/build.Dockerfile b/docker/build-vicmap.Dockerfile
index 95b28f1..95b28f1 100644
--- a/docker/build.Dockerfile
+++ b/docker/build-vicmap.Dockerfile
diff --git a/docker/conflate.Dockerfile b/docker/conflate.Dockerfile
index c557fe3..95b28f1 100644
--- a/docker/conflate.Dockerfile
+++ b/docker/conflate.Dockerfile
@@ -1,10 +1,7 @@
-FROM debian:unstable-slim
-RUN apt-get -y update && apt-get -y install curl gnupg make wget unzip gdal-bin osmium-tool software-properties-common
+FROM debian:buster-slim
+RUN apt-get -y update && apt-get -y install curl gnupg make
 RUN curl -fsSL https://deb.nodesource.com/setup_12.x | bash -
 RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
 RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
 RUN apt-get -y update && apt-get -y install nodejs yarn
-RUN wget -qO - https://qgis.org/downloads/qgis-2020.gpg.key | gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/qgis-archive.gpg --import
-RUN chmod a+r /etc/apt/trusted.gpg.d/qgis-archive.gpg
-RUN add-apt-repository "deb https://qgis.org/debian unstable main"
-RUN apt-get -y update && apt-get -y install qgis
+