1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
|
#!/usr/bin/env node
/**
* Remove duplicates (exact tags) at the same location or within a small proximity.
*/
const fs = require('fs')
const { Readable, Transform, pipeline } = require('stream')
const ndjson = require('ndjson')
const cluster = require('../lib/cluster.js')
const argv = require('yargs/yargs')(process.argv.slice(2))
.option('debug', {
type: 'boolean',
description: 'Dumps full debug logs'
})
.argv
if (argv._.length < 2) {
console.error("Usage: ./reduceDuplicates.js input.geojson output.geojson")
process.exit(1)
}
const inputFile = argv._[0]
const outputFile = argv._[1]
if (!fs.existsSync(inputFile)) {
console.error(`${inputFile} not found`)
process.exit(1)
}
let sourceCount = 0
const features = {}
const index = new Transform({
readableObjectMode: true,
writableObjectMode: true,
transform(feature, encoding, callback) {
sourceCount++
if (sourceCount % 10000 === 0) {
process.stdout.write(` ${sourceCount / 1000}k\r`)
}
const key = [
feature.properties['addr:unit'],
feature.properties['addr:housenumber'],
feature.properties['addr:street'],
feature.properties['addr:suburb'],
feature.properties['addr:state'],
feature.properties['addr:postcode']
].join('/')
if (!(key in features)) {
features[key] = []
}
features[key].push(feature)
callback()
}
})
let reduceIndex = 0
const reduce = new Transform({
readableObjectMode: true,
writableObjectMode: true,
transform(key, encoding, callback) {
reduceIndex++
if (reduceIndex % 10000 === 0) {
process.stdout.write(` ${reduceIndex / 1000}k / ${Math.round(sourceCount / 1000)}k (${Math.round(reduceIndex / sourceCount * 100)}%)\r`)
}
const groupedFeatures = features[key]
if (groupedFeatures.length === 1) {
// address not duplicated
this.push(groupedFeatures[0])
} else {
// address appears multiple times
const sameCoordinates = [...new Set(groupedFeatures.map(f => f.geometry.coordinates.join(',')))].length <= 1
if (sameCoordinates) {
// features have same properties and same geometry, so true duplicates can reduce to one
this.push(groupedFeatures[0])
} else {
// cluster features with a threshold of 25m
const clusters = cluster(groupedFeatures, 25)
// if clustered into a single cluster, then output a single average feature
if (clusters.length === 1) {
const averageCoordinates = [
groupedFeatures.map(f => f.geometry.coordinates[0]).reduce((acc, cur) => acc + cur) / groupedFeatures.length,
groupedFeatures.map(f => f.geometry.coordinates[1]).reduce((acc, cur) => acc + cur) / groupedFeatures.length
]
const averageFeature = groupedFeatures[0]
averageFeature.geometry.coordinates = averageCoordinates
this.push(averageFeature)
} else {
// more than one cluster, reduce those clustered into one, and then report all the results
const clusterAverages = clusters.map(cluster => {
if (cluster.length === 1) {
return cluster
} else {
const averageCoordinates = [
cluster.map(f => f.geometry.coordinates[0]).reduce((acc, cur) => acc + cur) / groupedFeatures.length,
cluster.map(f => f.geometry.coordinates[1]).reduce((acc, cur) => acc + cur) / groupedFeatures.length
]
const averageFeature = cluster[0]
averageFeature.geometry.coordinates = averageCoordinates
return averageFeature
}
})
// report these as address points with the same attributes but different locations beyond the threshold
if (debugDuplicateAddressStream) {
const webOfMatches = {
type: 'Feature',
properties: clusterAverages[0].properties,
geometry: {
type: 'LineString',
coordinates: averageClusters.map(p => p.geometry.coordinates)
}
}
debugDuplicateAddressStream.write(webOfMatches)
}
}
}
}
callback()
}
})
const debugDuplicateAddressesStream = argv.debug ?
ndjson.stringify()
.pipe(fs.createWriteStream('debug/reduceDuplicates/duplicateAddresses.geojson'))
: null
// first pass to index by geometry
console.log('First pass to index by address properties')
pipeline(
fs.createReadStream(inputFile),
ndjson.parse(),
index,
err => {
if (err) {
console.log(err)
process.exit(1)
} else {
console.log(` of ${sourceCount} features found ${Object.keys(features).length} unique addresses`)
// second pass to reduce overlapping features
pipeline(
Readable.from(Object.keys(features)),
reduce,
ndjson.stringify(),
fs.createWriteStream(outputFile),
err => {
if (err) {
console.log(err)
process.exit(1)
} else {
debugDuplicateAddressesStream.end()
process.exit(0)
}
}
)
}
}
)
|
