initial commit

7 files changed, 404 insertions, 0 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..b7aae12
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,21 @@
+image: "debian:buster-slim"
+
+stages:
+  - build
+
+build:
+  stage: build
+  script:
+    - apt-get update && apt-get install -y curl gnupg gdal-bin unzip wget
+    - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+    - echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+    - apt-get update && apt-get install -y yarn
+    - yarn install
+    - mkdir -p dist
+    - make downloadVicmap unzip
+    - make data/vicmap.geojson
+    - make dist/vicmap-osm.geojson
+  artifacts:
+    name: "build"
+    paths:
+      - dist
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..66de284
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,22 @@
+# download VicMap source data
+# the URL here usually gets manually updated weekly though no guarantees
+# it's a mirror of the upstream VICMAP data with split shp files reduced to a single shp file
+downloadVicmap:
+	mkdir -p data
+	wget --directory-prefix=data https://www.alantgeo.com.au/share/VICMAP_ADDRESS.zip
+
+unzip: data/VICMAP_ADDRESS.zip
+	mkdir -p data/vicmap
+	unzip -d data/vicmap $<
+
+data/vicmap.geojson: data/vicmap/ll_gda94/sde_shape/whole/VIC/VMADD/layer/address.shp
+	ogr2ogr -f GeoJSONSeq $@ $<
+
+dist/vicmap-osm.geojson: data/vicmap.geojson
+	./vicmap2osm.js $< $@
+
+data/vicmap.fgb: data/vicmap/ll_gda94/sde_shape/whole/VIC/VMADD/layer/address.shp
+	ogr2ogr -f FlatGeobuf $@ $<
+
+dist/vicmap-osm.fgb: dist/vicmap-osm.geojson
+	ogr2ogr -f FlatGeobuf $@ $<
diff --git a/filterOSM.js b/filterOSM.js
new file mode 100644
index 0000000..93db4a7
--- /dev/null
+++ b/filterOSM.js
@@ -0,0 +1,10 @@
+module.exports = (feature) => {
+
+  // skip any addresses without a housenumber
+  // eg PFI 53396626 has no housenumber
+  if (!('addr:housenumber' in feature.properties)) {
+    return false
+  }
+
+  return true
+}
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..5a1b265
--- /dev/null
+++ b/package.json
@@ -0,0 +1,14 @@
+{
+  "name": "vicmap-addresses2osm",
+  "version": "1.0.0",
+  "description": "Transform upstream VicMap Address data into OSM schema",
+  "main": "index.js",
+  "author": "Andrew Harvey <andrew@alantgeo.com.au>",
+  "license": "MIT",
+  "dependencies": {
+    "capital-case": "^1.0.4",
+    "ndjson": "^2.0.0",
+    "readable-stream": "^3.6.0",
+    "title-case": "^3.0.3"
+  }
+}
diff --git a/toOSM.js b/toOSM.js
new file mode 100644
index 0000000..dd993d0
--- /dev/null
+++ b/toOSM.js
@@ -0,0 +1,170 @@
+const { titleCase } = require('title-case')
+const { capitalCase } = require('capital-case')
+
+const buildingUnitType = {
+  ANT: 'ANTENNA',
+  APT: 'APARTMENT',
+  ATM: 'ATM',
+  BBOX: 'BATHING BOX',
+  BERT: 'BERTH',
+  BLDG: 'BUILDING',
+  BTSD: 'BOATSHED',
+  CARP: 'CARPARK',
+  CARS: 'CARSPACE',
+  CARW: 'CARWASH',
+  CHAL: 'CHALET',
+  CLUB: 'CLUB',
+  CTGE: 'COTTAGE',
+  CTYD: 'COURTYARD',
+  DUPL: 'DUPLEX',
+  FCTY: 'FACTORY',
+  FLAT: 'FLAT',
+  GATE: 'GARAGE',
+  GRGE: 'GATE',
+  HALL: 'HALL',
+  HELI: 'HELIPORT',
+  HNGR: 'HANGAR',
+  HOST: 'HOSTEL',
+  HSE: 'HOUSE',
+  KSK: 'KIOSK',
+  LOT: 'LOT',
+  MBTH: 'MAISONETTE',
+  OFFC: 'OFFICE',
+  PSWY: 'PASSAGEWAY',
+  PTHS: 'PENTHOUSE',
+  REST: 'RESTAURANT',
+  RESV: 'RESERVE',
+  ROOM: 'ROOM',
+  RPTN: 'RECPETION',
+  SAPT: 'STUDIO APARTMENT',
+  SE: 'SUITE',
+  SHCS: 'SHOWCASE',
+  SHED: 'SHED',
+  SHOP: 'SHOP',
+  SHRM: 'SHOWROOM',
+  SIGN: 'SIGN',
+  SITE: 'SITE',
+  STLL: 'STALL',
+  STOR: 'STORE',
+  STR: 'STRATA UNIT',
+  STU: 'STUDIO',
+  SUBS: 'SUBSTATION',
+  TNCY: 'TENANCY',
+  TNHS: 'TOWNHOUSE',
+  TWR: 'TOWER',
+  UNIT: 'UNIT',
+  VLLA: 'VILLA',
+  VLT: 'VAULT',
+  WHSE: 'WAREHOUSE',
+  WKSH: 'WORKSHOP'
+}
+
+// likely these are not proper names, so we will ignore them
+const emptyNames = [
+  'UNNAMED',
+  'NOT NAMED'
+]
+
+module.exports = (sourceFeature) => {
+
+  const outputFeature = Object.assign({}, sourceFeature)
+  const sourceProperties = sourceFeature.properties
+  const outputProperties = {}
+
+  // Building sub address type (eg UNIT OFFICE SHOP)
+  //
+  // bld_unit_*
+  const bld_unit_1 = [
+    sourceProperties.BUNIT_PRE1,
+    sourceProperties.BUNIT_ID1 || null, // 0 is used for an empty value in the source data, so convert 0 to null
+    sourceProperties.BUNIT_SUF1
+  ].join('') || null
+
+  const bld_unit_2 = [
+    sourceProperties.BUNIT_PRE2,
+    sourceProperties.BUNIT_ID2 || null, // 0 is used for an empty value in the source data, so convert 0 to null
+    sourceProperties.BUNIT_SUF2
+  ].join('') || null
+
+  // if both 1 and 2 defined, then use a range 1-2 otherwise just select the one which was defined
+  let bld_unit = null
+  if (sourceProperties.HSA_FLAG === 'Y') {
+    bld_unit = sourceProperties.HSAUNITID
+  } else {
+    if (bld_unit_1 && bld_unit_2) {
+      bld_unit = `${bld_unit_1}-${bld_unit_2}`
+    } else if (bld_unit_1) {
+      bld_unit = bld_unit_1
+    } else if (bld_unit_2) {
+      bld_unit = bld_unit_2
+    }
+  }
+
+  if (bld_unit) {
+    outputProperties['addr:unit'] = bld_unit
+  }
+
+  if (sourceProperties.BLGUNTTYP && sourceProperties.BLGUNTTYP in buildingUnitType) {
+    outputProperties['addr:unit:type'] = buildingUnitType[sourceProperties.BLGUNTTYP]
+  }
+
+  if (sourceProperties.BUILDING) {
+    outputProperties['addr:housename'] = sourceProperties.BUILDING
+  }
+
+  // house_*
+  const house_1 = [
+    sourceProperties.HSE_PREF1,
+    sourceProperties.HSE_NUM1 || null, // 0 is used for an empty value in the source data, so convert 0 to null
+    sourceProperties.HSE_SUF1
+  ].join('')
+
+  const house_2 = [
+    sourceProperties.HSE_PREF2,
+    sourceProperties.HSE_NUM2 || null, // 0 is used for an empty value in the source data, so convert 0 to null
+    sourceProperties.HSE_SUF2
+  ].join('')
+
+  let housenumber = null
+  if (house_1 && house_2) {
+    housenumber = `${house_1}-${house_2}`
+  } else if (house_1) {
+    housenumber = house_1
+  } else if (house_2) {
+    housenumber = house_2
+  }
+
+  if (housenumber) {
+    outputProperties['addr:housenumber'] = housenumber
+  }
+
+  // display numbers used predominately in the City of Melbourne CBD by large properties. Primarily to simplify an assigned number range. 
+  // so should map the assigned address or the signposted address?
+
+  // every record has at least ROAD_NAME populated
+  if (sourceProperties.ROAD_NAME && !emptyNames.includes(sourceProperties.ROAD_NAME)) {
+    outputProperties['addr:street'] = capitalCase([
+      sourceProperties.ROAD_NAME,
+      sourceProperties.ROAD_TYPE,
+      sourceProperties.RD_SUF
+    ].join(' '))
+  }
+
+  // every record has LOCALITY populated, however some values should be empty
+  if (sourceProperties.LOCALITY && !emptyNames.includes(sourceProperties.LOCALITY)) {
+    outputProperties['addr:suburb'] = capitalCase(sourceProperties.LOCALITY)
+  }
+
+  // every record has STATE populated
+  if (sourceProperties.STATE) {
+    outputProperties['addr:state'] = sourceProperties.STATE
+  }
+
+  // some records have no POSTCODE populated
+  if (sourceProperties.POSTCODE) {
+    outputProperties['addr:postcode'] = sourceProperties.POSTCODE
+  }
+
+  outputFeature.properties = outputProperties
+  return outputFeature
+}
diff --git a/vicmap2osm.js b/vicmap2osm.js
new file mode 100755
index 0000000..e579d98
--- /dev/null
+++ b/vicmap2osm.js
@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+
+const fs = require('fs')
+const { Transform, pipeline } = require('readable-stream')
+const ndjson = require('ndjson')
+const toOSM = require('./toOSM.js')
+const filterOSM = require('./filterOSM.js')
+
+const args = process.argv.slice(2)
+
+if (args.length < 2) {
+  console.error("Usage: ./vicmap2osm.js input.geojson output.geojson")
+  process.exit(1)
+}
+
+const inputFile = args[0]
+const outputFile = args[1]
+
+if (!fs.existsSync(inputFile)) {
+  console.error(`${inputFile} not found`)
+  process.exit(1)
+}
+
+const transform = new Transform({
+  readableObjectMode: true,
+  writableObjectMode: true,
+  transform(feature, encoding, callback) {
+    // convert source Feature into a Feature per the OSM schema
+    const osm = toOSM(feature)
+
+    // some addresses we skip importing into OSM
+    if (filterOSM(osm)) {
+      this.push(osm)
+    }
+
+    callback()
+  }
+})
+
+// stream in source ndjson, transfom and stream out
+pipeline(
+  fs.createReadStream(inputFile),
+  ndjson.parse(),
+  transform,
+  ndjson.stringify(),
+  fs.createWriteStream(outputFile),
+  (err) => {
+    if (err) {
+      console.log(err)
+      process.exit(1)
+    } else {
+      process.exit(0)
+    }
+  }
+)
diff --git a/yarn.lock b/yarn.lock
new file mode 100644
index 0000000..bb7a821
--- /dev/null
+++ b/yarn.lock
@@ -0,0 +1,112 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+capital-case@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/capital-case/-/capital-case-1.0.4.tgz#9d130292353c9249f6b00fa5852bee38a717e669"
+  integrity sha512-ds37W8CytHgwnhGGTi88pcPyR15qoNkOpYwmMMfnWqqWgESapLqvDx6huFjQ5vqWSn2Z06173XNA7LtMOeUh1A==
+  dependencies:
+    no-case "^3.0.4"
+    tslib "^2.0.3"
+    upper-case-first "^2.0.2"
+
+inherits@^2.0.3:
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
+  integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
+
+json-stringify-safe@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
+  integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=
+
+lower-case@^2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-2.0.2.tgz#6fa237c63dbdc4a82ca0fd882e4722dc5e634e28"
+  integrity sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==
+  dependencies:
+    tslib "^2.0.3"
+
+minimist@^1.2.5:
+  version "1.2.5"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
+  integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
+
+ndjson@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/ndjson/-/ndjson-2.0.0.tgz#320ac86f6fe53f5681897349b86ac6f43bfa3a19"
+  integrity sha512-nGl7LRGrzugTtaFcJMhLbpzJM6XdivmbkdlaGcrk/LXg2KL/YBC6z1g70xh0/al+oFuVFP8N8kiWRucmeEH/qQ==
+  dependencies:
+    json-stringify-safe "^5.0.1"
+    minimist "^1.2.5"
+    readable-stream "^3.6.0"
+    split2 "^3.0.0"
+    through2 "^4.0.0"
+
+no-case@^3.0.4:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/no-case/-/no-case-3.0.4.tgz#d361fd5c9800f558551a8369fc0dcd4662b6124d"
+  integrity sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==
+  dependencies:
+    lower-case "^2.0.2"
+    tslib "^2.0.3"
+
+readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.6.0:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
+  integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
+  dependencies:
+    inherits "^2.0.3"
+    string_decoder "^1.1.1"
+    util-deprecate "^1.0.1"
+
+safe-buffer@~5.2.0:
+  version "5.2.1"
+  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
+  integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
+
+split2@^3.0.0:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/split2/-/split2-3.2.2.tgz#bf2cf2a37d838312c249c89206fd7a17dd12365f"
+  integrity sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==
+  dependencies:
+    readable-stream "^3.0.0"
+
+string_decoder@^1.1.1:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
+  integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
+  dependencies:
+    safe-buffer "~5.2.0"
+
+through2@^4.0.0:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764"
+  integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==
+  dependencies:
+    readable-stream "3"
+
+title-case@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/title-case/-/title-case-3.0.3.tgz#bc689b46f02e411f1d1e1d081f7c3deca0489982"
+  integrity sha512-e1zGYRvbffpcHIrnuqT0Dh+gEJtDaxDSoG4JAIpq4oDFyooziLBIiYQv0GBT4FUAnUop5uZ1hiIAj7oAF6sOCA==
+  dependencies:
+    tslib "^2.0.3"
+
+tslib@^2.0.3:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.2.0.tgz#fb2c475977e35e241311ede2693cee1ec6698f5c"
+  integrity sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w==
+
+upper-case-first@^2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/upper-case-first/-/upper-case-first-2.0.2.tgz#992c3273f882abd19d1e02894cc147117f844324"
+  integrity sha512-514ppYHBaKwfJRK/pNC6c/OxfGa0obSnAl106u97Ed0I625Nin96KAjttZF6ZL3e1XLtphxnqrOi9iWgm+u+bg==
+  dependencies:
+    tslib "^2.0.3"
+
+util-deprecate@^1.0.1:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
+  integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=