diff options
| -rw-r--r-- | .gitlab-ci.yml | 25 | ||||
| -rw-r--r-- | Makefile | 13 | ||||
| -rwxr-xr-x | bin/compareSuburb.js | 101 |
3 files changed, 139 insertions, 0 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e63b3a..dbed9f3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -154,3 +154,28 @@ conflate: name: "conflate" paths: - dist + +build compareSuburb: + stage: build + image: "registry.gitlab.com/alantgeo/vicmap2osm:build-osm" + before_script: + - ls data dist + cache: + <<: *global_cache + policy: pull + script: + - yarn install + - mkdir -p dist data + - mkdir -p data/vicmap/ll_gda94/sde_shape/whole/VIC/VMADD/layer + - touch data/VICMAP_ADDRESS.zip + - touch data/vicmap/ll_gda94/sde_shape/whole/VIC/VMADD/layer/address.shp + - touch --no-create data/vicmap.geojson + - time make dist/vicmap-osm.geojson + - time make dist/vicmapSuburbDiffersWithOSM.geojson + - rm -f dist/vicmap-osm.geojson + when: manual + artifacts: + name: "build compareSuburb" + paths: + - dist + - debug @@ -149,3 +149,16 @@ convertConflationResultsToFGB: ogr2ogr -f FlatGeobuf dist/conflate/exactMatch.fgb dist/conflate/exactMatch.geojson ogr2ogr -f FlatGeobuf dist/conflate/noExactMatch.fgb dist/conflate/noExactMatch.geojson ogr2ogr -f FlatGeobuf dist/conflate/noOSMAddressWithinBlock.fgb dist/conflate/noOSMAddressWithinBlock.geojson + +# extract admin_level=10 from OSM +data/victoria-admin.osm.pbf: data/victoria.osm.pbf + osmium tags-filter --remove-tags --output=$@ $< r/boundary=administrative + +data/victoria-admin-level10.osm.pbf: data/victoria-admin.osm.pbf + osmium tags-filter --remove-tags --output=$@ $< r/admin_level=10 + +data/victoria-admin-level10.osm.geojson: data/victoria-admin-level10.osm.pbf + osmium export --overwrite --geometry-types=polygon --output-format=geojsonseq --format-option=print_record_separator=false --output $@ $< + +dist/vicmapSuburbDiffersWithOSM.geojson: data/vicmap-osm.geojson data/victoria-admin-level10.osm.geojson + ./bin/compareSuburb.geojson $^ $@ diff --git a/bin/compareSuburb.js b/bin/compareSuburb.js new file mode 100755 index 0000000..c849735 --- /dev/null +++ b/bin/compareSuburb.js @@ -0,0 +1,101 @@ +#!/usr/bin/env node + +/** + * Compare the addr:suburb reported from Vicmap which the corresponding suburb/locality boundary existing in OSM + */ + +const fs = require('fs') +const { Transform, pipeline } = require('stream') +const ndjson = require('ndjson') +const PolygonLookup = require('polygon-lookup') + +const argv = require('yargs/yargs')(process.argv.slice(2)) + .option('verbose', { + type: 'boolean', + description: 'Verbose logging' + }) + .argv + +if (argv._.length < 3) { + console.error("Usage: ./compareSuburb.js vicmap-osm.geojson osm_admin_level_10.geojson output.geojson") + process.exit(1) +} + +const vicmapFile = argv._[0] +const osmFile = argv._[1] +const outputFile = argv._[2] + +if (!fs.existsSync(vicmapFile)) { + console.error(`${vicmapFile} not found`) + process.exit(1) +} +if (!fs.existsSync(osmFile)) { + console.error(`${osmFile} not found`) + process.exit(1) +} + +const osmFeatures = fs.readFileSync(osmFile, 'utf-8').toString().split('\n') + .filter(line => line !== '') + .map((line, index) => { + try { + return JSON.parse(line) + } catch { + console.log(`Error parsing line ${index} of ${osmFile}: ${line}`) + } + }) + +console.log('Creating index for OSM Admin Boundaries lookup') +const lookup = new PolygonLookup({ + type: 'FeatureCollection', + features: osmFeatures +}) + +// conflate vicmap addresses with OSM addresses +let sourceCount = 0 +const compare = new Transform({ + readableObjectMode: true, + writableObjectMode: true, + transform(feature, encoding, callback) { + sourceCount++ + + if (process.stdout.isTTY && sourceCount % 10000 === 0) { + process.stdout.write(` ${sourceCount.toLocaleString()}\r`) + } + + // find which block this vicmap address is in + const results = lookup.search(...feature.geometry.coordinates.slice(0, 2), 1) + const osmFeature = results ? (results.type === 'FeatureCollection' ? (results.features ? results.features[0] : null) : results) : null + if (osmFeature) { + // address within an OSM suburb + if (feature.properties['addr:suburb'] !== osmFeature.properties['name']) { + // Vicmap suburb different to OSM admin_level=10 + // console.log('Suburb differs', feature.properties['addr:suburb'], osmFeature.properties['name']) + feature.properties._osmSuburb = osmFeature.properties['name'] + this.push(feature) + } + } else { + // address not found within any OSM suburb + // console.log('Not found within any OSM suburb', feature) + this.push(feature) + } + + callback() + } +}) + +console.log('Pass 1/1: Find Vicmap addresses where the suburb differs with OSM admin boundaries') +pipeline( + fs.createReadStream(vicmapFile), + ndjson.parse(), + compare, + ndjson.stringify(), + fs.createWriteStream(outputFile), + err => { + if (err) { + console.log(err) + process.exit(1) + } else { + process.exit(0) + } + } +) |