update conflate.js to match polygons where not strictly within

3 files changed, 144 insertions, 10 deletions

diff --git a/bin/conflate.js b/bin/conflate.js
index 3e0e321..f2b5328 100755
--- a/bin/conflate.js
+++ b/bin/conflate.js
@@ -8,8 +8,15 @@ const fs = require('fs')
 const { Transform, pipeline } = require('stream')
 const ndjson = require('ndjson')
 const PolygonLookup = require('polygon-lookup')
+const Flatbush = require('flatbush')
+const bbox = require('@turf/bbox').default
+const booleanIntersects = require('@turf/boolean-intersects').default
 
 const argv = require('yargs/yargs')(process.argv.slice(2))
+  .option('verbose', {
+    type: 'boolean',
+    description: 'Verbose logging'
+  })
   .argv
 
 if (argv._.length < 4) {
@@ -53,12 +60,17 @@ const lookupBlocks = new PolygonLookup({
   features: blocksByOSMAddr
 })
 let lookupOSMAddressPoly
-const osmAddrPoly = []
+const osmAddrPolygons = []
 const osmAddrLines = [] // address interpolation lines
+
 // indexed by block
 const osmAddrPoints = {
   0: [] // this one is for any points not within a block
 }
+const osmAddrPolygonsByBlock = {
+  0: [] // this one is for any polygons not within a block
+}
+
 
 // find OSM Addresses and store them
 // polygons go into a simple array, which later we create a point in polygon index for
@@ -76,7 +88,7 @@ const filterOSMAddrPoly = new Transform({
 
     if (feature && feature.geometry && feature.geometry.type) {
       if (feature.geometry.type === 'Polygon' || feature.geometry.type === 'MultiPolygon') {
-        osmAddrPoly.push(feature)
+        osmAddrPolygons.push(feature)
       } else if (feature.geometry.type === 'Point') {
         const results = lookupBlocks.search(...feature.geometry.coordinates.slice(0, 2), 1)
         const block = results ? (results.type === 'FeatureCollection' ? (results.features ? results.features[0] : null) : results) : null
@@ -113,9 +125,11 @@ const conflate = new Transform({
       process.stdout.write(` ${sourceCount / 1000}k\r`)
     }
 
+    // find which block this vicmap address is in
     const results = lookupBlocks.search(...feature.geometry.coordinates.slice(0, 2), 1)
     const block = results ? (results.type === 'FeatureCollection' ? (results.features ? results.features[0] : null) : results) : null
     if (block) {
+      // address within a block
       if (block.properties.NUMPOINTS === 0) {
         // no OSM addresses found within this block, so able to import without review
         outputStreams.noOSMAddressWithinBlock.write(feature)
@@ -132,8 +146,8 @@ const conflate = new Transform({
           // address not found within an existing OSM address polygon
           
           // see if any address with the same number and street in the same block
-          if (block.id in osmAddrPoints) {
-            const osmAddrWithinBlock = osmAddrPoints[block.id]
+          if (block.id in osmAddrPoints || block.id in osmAddrPolygonsByBlock) {
+            const osmAddrWithinBlock = [osmAddrPoints[block.id] || [], osmAddrPolygonsByBlock[block.id] || []].flat()
             const matches = osmAddrWithinBlock.filter(osmAddr => {
               return (feature.properties['addr:street'] === osmAddr.properties['addr:street'] &&
               feature.properties['addr:housenumber'] === osmAddr.properties['addr:housenumber'] )
@@ -147,9 +161,10 @@ const conflate = new Transform({
               outputStreams.noExactMatch.write(feature)
             }
           } else {
-            // block id not found in osmAddrPoints, meaning there are no osmAddress points in this block,
-            // however in this case NUMPOINTS should have been 0
-            console.log(`Block ID not found when expected`)
+            // block id not found in osmAddrPoints or osmAddrPolygonsByBlock, meaning there are no osmAddress points or polygons in this block,
+            // maybe there was an address as a linear way?
+            // we ignore address interpolation lines and only look at the endpoint nodes from the interpolation way
+            console.log(`Block ID ${block.id} not found when expected for `, JSON.stringify(feature), JSON.stringify(block))
           }
         }
       }
@@ -183,12 +198,36 @@ pipeline(
       console.log(err)
       process.exit(1)
     } else {
-      console.log(`  of ${osmAddrCount} OSM address features found ${osmAddrPoly.length} addresses represented as polygons, ${osmAddrLines.length} addresses represented as lines`)
+      console.log(`  of ${osmAddrCount} OSM address features found ${osmAddrPolygons.length} addresses represented as polygons, ${osmAddrLines.length} addresses represented as lines`)
+
       console.log('Creating index for OSM Address Polygon lookup')
       lookupOSMAddressPoly = new PolygonLookup({
         type: 'FeatureCollection',
-        features: osmAddrPoly
+        features: osmAddrPolygons
       })
+
+      // create an index of blocks
+      const blockIndex = new Flatbush(blocksByOSMAddr.length)
+      for (const block of blocksByOSMAddr) {
+        blockIndex.add(...bbox(block))
+      }
+      blockIndex.finish()
+
+      console.log(`Index OSM Address Polygons within each block`)
+      // for each OSM address polygon
+      for (const osmAddrPolygon of osmAddrPolygons) {
+        // find the blocks it might intersect
+        const candidateBlocks = blockIndex.search(...bbox(osmAddrPolygon))
+        // then test if it actually intersects
+        const intersectingBlocks = candidateBlocks.map(candidateBlock => booleanIntersects(osmAddrPolygon, blocksByOSMAddr[candidateBlock]))
+        for (const intersectingBlock of intersectingBlocks) {
+          if (!(intersectingBlock.id in osmAddrPolygonsByBlock)) {
+            osmAddrPolygonsByBlock[intersectingBlock.id] = []
+          }
+          osmAddrPolygonsByBlock[intersectingBlock.id].push(osmAddrPolygon)
+        }
+      }
+
       // second pass to conflate with existing OSM data
       console.log('Pass 2/2: Conflate with existing OSM data')
       pipeline(
diff --git a/package.json b/package.json
index 80c099a..cd684be 100644
--- a/package.json
+++ b/package.json
@@ -9,6 +9,8 @@
     "test": "./node_modules/.bin/tape test/toOSM.js test/cluster.js test/unitsToRanges.js test/withinRange.js test/valueLimits.js test/reduceRangeDuplicates.js"
   },
   "dependencies": {
+    "@turf/bbox": "^6.3.0",
+    "@turf/boolean-intersects": "^6.3.0",
     "capital-case": "^1.0.4",
     "cheap-ruler": "^3.0.1",
     "clone-deep": "^4.0.1",
diff --git a/yarn.lock b/yarn.lock
index 1055088..3aa5569 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2,6 +2,89 @@
 # yarn lockfile v1
 
 
+"@turf/bbox@*", "@turf/bbox@^6.3.0":
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/@turf/bbox/-/bbox-6.3.0.tgz#0e1a9b59f32d6a2a40c806f54cbaa73575bead25"
+  integrity sha512-N4ue5Xopu1qieSHP2MA/CJGWHPKaTrVXQJjzHRNcY1vtsO126xbSaJhWUrFc5x5vVkXp0dcucGryO0r5m4o/KA==
+  dependencies:
+    "@turf/helpers" "^6.3.0"
+    "@turf/meta" "^6.3.0"
+
+"@turf/boolean-disjoint@^6.3.0":
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/@turf/boolean-disjoint/-/boolean-disjoint-6.3.0.tgz#4b27fb3c3c9ff23f69fbcc8d2d03665e9f52c20e"
+  integrity sha512-bVAwAJF05QPH0tf+qjR3kUcCyqTgYcCbXSMgXl6LQF6mSGuOutzNq1gCyRLCOdOcZtw4Oh4dqeP3ykwv8kDibw==
+  dependencies:
+    "@turf/boolean-point-in-polygon" "^6.3.0"
+    "@turf/helpers" "^6.3.0"
+    "@turf/line-intersect" "^6.3.0"
+    "@turf/meta" "^6.3.0"
+    "@turf/polygon-to-line" "^6.3.0"
+
+"@turf/boolean-intersects@^6.3.0":
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/@turf/boolean-intersects/-/boolean-intersects-6.3.0.tgz#aaa096e6e346e6da673984ad397d4a96cb97cb89"
+  integrity sha512-2pHOYqHSKDo0rzHTiqwdAaxa+tHLwr4NaTAjOpuN2hipv9bErzGtv3e5IYceJBnT0u4akK17NTn6qAr7/7g2aQ==
+  dependencies:
+    "@turf/boolean-disjoint" "^6.3.0"
+    "@turf/helpers" "^6.3.0"
+    "@turf/meta" "^6.3.0"
+
+"@turf/boolean-point-in-polygon@^6.3.0":
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/@turf/boolean-point-in-polygon/-/boolean-point-in-polygon-6.3.0.tgz#784952a36c64119e90fbe94650245da62ecd8fc2"
+  integrity sha512-NqFSsoE6OwhDK19IllDQRhEQEkF7UVEOlqH9vgS1fGg4T6NcyKvACJs05c9457tL7QSbV9ZS53f2qiLneFL+qg==
+  dependencies:
+    "@turf/helpers" "^6.3.0"
+    "@turf/invariant" "^6.3.0"
+
+"@turf/helpers@6.x", "@turf/helpers@^6.3.0":
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/@turf/helpers/-/helpers-6.3.0.tgz#87f90f806c3f8ad6385ef8d2041d3662bf3c9fb1"
+  integrity sha512-kr6KuD4Z0GZ30tblTEvi90rvvVNlKieXuMC8CTzE/rVQb0/f/Cb29zCXxTD7giQTEQY/P2nRW23wEqqyNHulCg==
+
+"@turf/invariant@^6.3.0":
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/@turf/invariant/-/invariant-6.3.0.tgz#04a22b26c5503146c03fa6198176b72bd591eadf"
+  integrity sha512-2OFOi9p+QOrcIMySEnr+WlOiKaFZ1bY56jA98YyECewJHfhPFWUBZEhc4nWGRT0ahK08Vus9+gcuBX8QIpCIIw==
+  dependencies:
+    "@turf/helpers" "^6.3.0"
+
+"@turf/line-intersect@^6.3.0":
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/@turf/line-intersect/-/line-intersect-6.3.0.tgz#726a50edc66bb7b5e798b052b103fb0da4d1c4f4"
+  integrity sha512-3naxR7XpkPd2vst3Mw6DFry4C9m3o0/f2n/xu5UAyxb88Ie4m2k+1eqkhzMMx/0L+E6iThWpLx7DASM6q6o9ow==
+  dependencies:
+    "@turf/helpers" "^6.3.0"
+    "@turf/invariant" "^6.3.0"
+    "@turf/line-segment" "^6.3.0"
+    "@turf/meta" "^6.3.0"
+    geojson-rbush "3.x"
+
+"@turf/line-segment@^6.3.0":
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/@turf/line-segment/-/line-segment-6.3.0.tgz#b37d6877ee4425ebc12698860e7d355d1bf2ba9b"
+  integrity sha512-M+aDy83V+E7jYWNaf+b+A88yhnMrJhyg/lhAj6mU6UeB2PbruXB2qgSmmVDSE2dIknOvZZuIWNzEzUI07RO2kw==
+  dependencies:
+    "@turf/helpers" "^6.3.0"
+    "@turf/invariant" "^6.3.0"
+    "@turf/meta" "^6.3.0"
+
+"@turf/meta@6.x", "@turf/meta@^6.3.0":
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/@turf/meta/-/meta-6.3.0.tgz#f3e280ab29641f21e4f99310ce77f9c8394ae394"
+  integrity sha512-qBJjaAJS9H3ap0HlGXyF/Bzfl0qkA9suafX/jnDsZvWMfVLt+s+o6twKrXOGk5t7nnNON2NFRC8+czxpu104EQ==
+  dependencies:
+    "@turf/helpers" "^6.3.0"
+
+"@turf/polygon-to-line@^6.3.0":
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/@turf/polygon-to-line/-/polygon-to-line-6.3.0.tgz#aa3e1a0ea074479e5cb6c5c111a924bb80ec553e"
+  integrity sha512-KFGlQlGOBayBvELz+tip1zCa3eB8xyZePZUZ3I3OnU7mk0FFzJzvLTmPUc7MupgqORT4LkNGmyKSVWaz38NTig==
+  dependencies:
+    "@turf/helpers" "^6.3.0"
+    "@turf/invariant" "^6.3.0"
+
 ansi-regex@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75"
@@ -232,6 +315,16 @@ geoflatbush@^1.0.0:
   dependencies:
     flatqueue "^1.1.0"
 
+geojson-rbush@3.x:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/geojson-rbush/-/geojson-rbush-3.1.2.tgz#577d6ec70ba986d4e60b741f1df5147faeb82c97"
+  integrity sha512-grkfdg3HIeTjwTfiJe5FT8+fGU3fABCc+vRJDBwdQz9kkLF0Sbif2gs2JUzjewwgmnvLGy9fInySDeADoNuk7w==
+  dependencies:
+    "@turf/bbox" "*"
+    "@turf/helpers" "6.x"
+    "@turf/meta" "6.x"
+    rbush "^2.0.0"
+
 get-caller-file@^2.0.5:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
@@ -543,7 +636,7 @@ quickselect@^1.0.1:
   resolved "https://registry.yarnpkg.com/quickselect/-/quickselect-1.1.1.tgz#852e412ce418f237ad5b660d70cffac647ae94c2"
   integrity sha512-qN0Gqdw4c4KGPsBOQafj6yj/PA6c/L63f6CaZ/DCF/xF4Esu3jVmKLUDYxghFx8Kb/O7y9tI7x2RjTSXwdK1iQ==
 
-rbush@^2.0.2:
+rbush@^2.0.0, rbush@^2.0.2:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/rbush/-/rbush-2.0.2.tgz#bb6005c2731b7ba1d5a9a035772927d16a614605"
   integrity sha512-XBOuALcTm+O/H8G90b6pzu6nX6v2zCKiFG4BJho8a+bY6AER6t8uQUZdi5bomQc0AprCWhEGa7ncAbbRap0bRA==