diff options
| -rw-r--r-- | Makefile | 3 | ||||
| -rw-r--r-- | README.md | 10 | ||||
| -rwxr-xr-x | bin/mr2osc.mjs | 438 | ||||
| -rw-r--r-- | package.json | 2 | ||||
| -rw-r--r-- | yarn.lock | 17 |
5 files changed, 467 insertions, 3 deletions
@@ -190,6 +190,9 @@ dist/conflate: mkdir -p $@ ./bin/conflate.js dist/vicmap-osm-uniq-flats.geojson data/victoria-addr.osm.geojson dist/blocksByOSMAddr.geojson $@ +dist/unitFromNumber.osc: dist/conflate/mr_explodeUnitFromNumber.geojson + ./bin/mr2osc.mjs --dry-run $< $@ + convertConflationResultsToFGB: ogr2ogr -f FlatGeobuf dist/conflate/withinExistingOSMAddressPoly.fgb dist/conflate/withinExistingOSMAddressPoly.geojson ogr2ogr -f FlatGeobuf dist/conflate/notFoundInBlocks.fgb dist/conflate/notFoundInBlocks.geojson @@ -306,15 +306,19 @@ Split into the following candidate categories, then again split into suburb/loca The dedicated import account used for the import is [`vicmap_import`](https://www.openstreetmap.org/user/vicmap_import). - ### Stage 1 - postal_code - For background see [Inclusion of `addr:suburb`, `addr:postcode` and `addr:state`](#inclusion-of-addrsuburb-addrpostcode-and-addrstate). Using JOSM RemoteControl commands [`postal_code`](https://wiki.openstreetmap.org/wiki/Key:postal_code) will be added to the existing Victorian `admin_level=10` boundaries using the postcode derived from Vicmap Addresses. Except for Melbourne suburb because there are two postal codes in use, and the `postal_code` boundaries are already mapped. +### Stage 2 - Set unit from housenumber +During the conflation stage, Vicmap addresses which were deemed to match OSM addresses where in OSM it was represented as `addr:housenumber=X/Y` whereas in Vicmap it was represented as `addr:unit=X`, `addr:housenumber=Y`, then an automated tag change to move the unit into `addr:unit` is performed. + +This will be a single Victoria wide changeset. + +`bin/mr2osc.mjs` is the script which creates the OsmChange and uploads it as a changeset from the tag changes outputted from the _conflate_ stage as a MapRoulette `setTags` operation. -### Stage 2 - +### Stage 3 - New addresses without conflicts `bin/upload.sh` is the script used to perform the actual uploads into OSM. For each import candidate (by candidate category by state) there is one OSM Changeset created. diff --git a/bin/mr2osc.mjs b/bin/mr2osc.mjs new file mode 100755 index 0000000..25f008e --- /dev/null +++ b/bin/mr2osc.mjs @@ -0,0 +1,438 @@ +#!/usr/bin/env node + +/** + * Given a MapRoulette Cooperative Challenge, upload all the changes directly without review + */ + +import fs from 'fs' +import { pipeline } from 'stream/promises' +import { Transform } from 'stream' +import ndjson from 'ndjson' +import _ from 'lodash' +import fetch from 'node-fetch' +import xml from 'xml-js' +import yargs from 'yargs' + +const argv = yargs(process.argv.slice(2)) + .option('dry-run', { + type: 'boolean', + description: 'Skip uploading to OSM, instead log what we would do' + }) + .option('changeset-comment', { + type: 'string', + description: 'OSM Changeset comment', + demandOption: true + }) + .argv + +if (argv._.length < 2) { + console.error("Usage: ./mr2osc.js mr.json change.osc") + process.exit(1) +} + +const mrFile = argv._[0] +const changeFile = argv._[1] + +if (!fs.existsSync(mrFile)) { + console.error(`${mrFile} not found`) + process.exit(1) +} + +if (argv.dryRun) { + console.log('Dry run enabled') +} + +// https://wiki.openstreetmap.org/wiki/API_v0.6#Multi_fetch:_GET_.2Fapi.2F0.6.2F.5Bnodes.7Cways.7Crelations.5D.3F.23parameters +const MAXIMUM_ELEMENTS_PER_GET_REQUEST = 725 + +const OSM_API_READ = 'https://api.openstreetmap.org' +const OSM_API_WRITE = (process.env.ENVIRONMENT === 'prod') ? 'https://api.openstreetmap.org' : 'https://master.apis.dev.openstreetmap.org' +const USER_AGENT = 'vicmap2osm/1.0 (+https://gitlab.com/alantgeo/vicmap2osm)' +const CREATED_BY = 'https://gitlab.com/alantgeo/vicmap2osm' +const AUTHORIZATION = `Basic ${Buffer.from(process.env.OSM_USERNAME + ':' + process.env.OSM_PASSWORD).toString('base64')}` +let MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST = 10000 + +console.log(`Retrieving capabilities from ${OSM_API_WRITE}`) +await fetch(`${OSM_API_WRITE}/api/capabilities`, { + headers: { + 'User-Agent': USER_AGENT + } +}) + .then(res => res.text()) + .then(text => { + const capabilities = JSON.parse(xml.xml2json(text, { + compact: true + })) + const apiMaximumElements = capabilities.osm.api.changesets._attributes.maximum_elements + MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST = Math.min(MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST, apiMaximumElements) + }) + +const operations = { + node: {}, + way: {}, + relation: {} +} + +// full OSM Elements live from OSM API +const elements = {} + +const changes = { + node: [], + way: [], + relation: [] +} + +let taskCount = 0 +/** + * Transform which receives MapRoulette tasks and + * stores operations by Element type by id in `operations`. + */ +const listElements = new Transform({ + readableObjectMode: true, + writableObjectMode: true, + transform(task, encoding, callback) { + taskCount++ + + if (process.stdout.isTTY && taskCount % 1000 === 0) { + process.stdout.write(` ${taskCount.toLocaleString()}\r`) + } + + if (task && task.features && task.features.length && task.cooperativeWork && task.cooperativeWork.operations && task.cooperativeWork.operations.length) { + const operation = task.cooperativeWork.operations[0] + + if (operation.operationType === 'modifyElement') { + const type = operation.data.id.split('/')[0] + const id = operation.data.id.split('/')[1] + operations[type][id] = operation.data.operations + } + } + + callback() + } +}) + +/** + * Fetch all Elements with operations from `operations` + * so we have the full Element to generate the OsmChange. + * Results stored in `elements`. + */ +async function fetchElements() { + /* + const nodeChunks = _.chunk(Object.keys(operations.nodes), MAXIMUM_ELEMENTS_PER_GET_REQUEST) + const wayChunks = _.chunk(Object.keys(operations.ways), MAXIMUM_ELEMENTS_PER_GET_REQUEST) + const relationChunks = _.chunk(Object.keys(operations.relations), MAXIMUM_ELEMENTS_PER_GET_REQUEST) + */ + + for (const type in operations) { + let index = 0 + const chunks = _.chunk(Object.keys(operations[type]), MAXIMUM_ELEMENTS_PER_GET_REQUEST) + for (const chunk of chunks) { + index++ + + process.stdout.write(` Fetch ${type}s (${index}/${chunks.length})\r`) + await fetch(`${OSM_API_READ}/api/0.6/${type}s?${type}s=${chunk.join(',')}`, { + headers: { + 'Accept': 'application/json', + 'User-Agent': USER_AGENT + } + }) + .then(res => res.json()) + .then(json => { + json.elements.forEach(element => { + elements[`${element.type}/${element.id}`] = element + }) + }) + .catch(err => { + console.error(err) + process.exit(1) + }) + } + process.stdout.write(`\r`) + process.stdout.write(` Fetched ${chunks.length} ${type}s\n`) + } + return Promise.resolve() + + /* + return [ + ...nodeChunks.map(nodeChunk => { + return fetch(`${OSM_API_READ}/api/0.6/nodes?nodes=${nodeChunk.join(',')}`, { + headers: { + 'Accept': 'application/json', + 'User-Agent': USER_AGENT + } + }) + .then(res => res.json()) + .then(json => { + json.elements.forEach(element => { + elements[`${element.type}/${element.id}`] = element + }) + }) + }), + ...wayChunks.map(wayChunk => { + return fetch(`${OSM_API_READ}/api/0.6/ways?ways=${wayChunk.join(',')}`, { + headers: { + 'Accept': 'application/json', + 'User-Agent': USER_AGENT + } + }) + .then(res => res.json()) + .then(json => { + json.elements.forEach(element => { + elements[`${element.type}/${element.id}`] = element + }) + }) + }), + ...relationChunks.map(relationChunk => { + return fetch(`${OSM_API_READ}/api/0.6/relations?relations=${relationChunk.join(',')}`, { + headers: { + 'Accept': 'application/json', + 'User-Agent': USER_AGENT + } + }) + .then(res => res.json()) + .then(json => { + json.elements.forEach(element => { + elements[`${element.type}/${element.id}`] = element + }) + }) + }) + ]) + */ +} + +function createChanges() { + for (const type in operations) { + for (const [id, ops] of Object.entries(operations[type])) { + const element = elements[`${type}/${id}`] + const tags = [] + + if (ops && ops.length) { + ops.forEach(operation => { + if (operation.operation === 'setTags') { + // first ensure that our assumptions about the current values of these tags hasn't changed + // TODO + + // tags to set + for (const [key, value] of Object.entries(operation.data)) { + // replace with new tag + element.tags[key] = value + } + } + }) + } + + // set nodeChanges for this node + for (const [key, value] of Object.entries(element.tags)) { + const tag = { + _attributes: { + k: key, + v: value + } + } + tags.push(tag) + } + switch (type) { + case 'node': + changes[type].push({ + _attributes: { + id: id, + version: element.version + 1, + lat: element.lat, + lon: element.lon + }, + tag: tags + }) + break + case 'way': + changes[type].push({ + _attributes: { + id: id, + version: element.version + 1, + }, + tag: tags, + nodes: element.nodes + }) + break + case 'relation': + changes[type].push({ + _attributes: { + id: id, + version: element.version + 1, + }, + tag: tags, + members: element.members + }) + break + } + } + } +} + +async function uploadChanges() { + // now we have the latest full elements, apply our changes as an OsmChange + // https://wiki.openstreetmap.org/wiki/API_v0.6#Diff_upload:_POST_.2Fapi.2F0.6.2Fchangeset.2F.23id.2Fupload + + const totalElements = Object.values(changes).flat().length + const totalChangesets = Math.ceil(totalElements / MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST) + if (totalChangesets > 1) { + console.log(`${totalElements} exceeds API maximum elements of ${MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST} splitting into ${totalChangesets} changesets`) + process.exit(1) + } + + for (let changesetIndex = 0; changesetIndex < totalChangesets; changesetIndex++) { + // create a changeset + let changesetId + const createBody = xml.json2xml({ + _declaration: { + _attributes: { + version: "1.0", + encoding: "UTF-8" + } + }, + osm: { + changeset: { + tag: [ + { + _attributes: { + k: 'created_by', + v: CREATED_BY + } + }, + { + _attributes: { + k: 'comment', + v: argv.changesetComment + } + } + ] + } + } + }, Object.assign({compact: true}, argv.dryRun ? { spaces: 2 } : {})) + if (argv.dryRun) { + console.log(`${OSM_API_WRITE}/api/0.6/changeset/create`) + console.log(createBody) + } else { + changesetId = await fetch(`${OSM_API_WRITE}/api/0.6/changeset/create`, { + method: 'PUT', + headers: { + 'Authorization': AUTHORIZATION, + 'User-Agent': USER_AGENT + }, + body: createBody + }) + .then(res => res.text()) + console.log(`Opened changeset ${changesetId}`) + } + + /* + const nodeChangesCount = changes.node.length + const wayChangesCount = changes.node.length + const relationChangesCount = changes.node.length + + const startIndex = changeIndex * MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST + const endIndex = ((changeIndex + 1) * MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST) - 1 + + if (startIndex < nodeChangesCount) { + changes.node.slice(startIndex, ) + } + */ + + // upload to the changeset + const uploadBody = xml.json2xml({ + _declaration: { + _attributes: { + version: "1.0", + encoding: "UTF-8" + } + }, + osmChange: { + _attributes: { + version: '0.6', + generator: CREATED_BY + }, + modify: { + node: changes.node.map(change => insertChangesetId(change, changesetId)), + way: changes.way.map(change => insertChangesetId(change, changesetId)), + relation: changes.relation.map(change => insertChangesetId(change, changesetId)) + } + } + }, Object.assign({ + compact: true, + attributeValueFn: value => { + // TODO need to test this via the dev api + return value.replace(/"/g, '"') // convert quote back before converting amp + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') + } + }, argv.dryRun ? { spaces: 2 } : {})) + + const filePath = totalChangesets > 1 ? path.join(path.dirname(changeFile), path.basename(changeFile, '.osc') + '-' + (changesetIndex + 1) + '.osc') : changeFile + fs.writeFileSync(filePath, uploadBody) + console.log(`Saved ${filePath}`) + + if (argv.dryRun) { + console.log(`${OSM_API_WRITE}/api/0.6/changeset/${changesetId}/upload`) + } else { + await fetch(`${OSM_API_WRITE}/api/0.6/changeset/${changesetId}/upload`, { + method: 'POST', + headers: { + 'Authorization': AUTHORIZATION, + 'User-Agent': USER_AGENT + }, + body: uploadBody + }) + console.log(`Uploaded contents to ${changesetId}`) + } + + // close the changeset + if (argv.dryRun) { + console.log(`${OSM_API_WRITE}/api/0.6/changeset/${changesetId}/close`) + } else { + await fetch(`${OSM_API_WRITE}/api/0.6/changeset/${changesetId}/close`, { + method: 'PUT', + headers: { + 'Authorization': AUTHORIZATION, + 'User-Agent': USER_AGENT + } + }) + console.log(`Closed changeset ${changesetId}`) + } + } + + return +} + +function insertChangesetId(change, changesetId) { + change._attributes.changeset = changesetId + return change +} + +console.log('Step 1/4: Find Elements to modify') +async function run() { + await pipeline( + fs.createReadStream(mrFile), + ndjson.parse(), + listElements, + ) + + for (const [type, value] of Object.entries(operations)) { + console.log(` ${type}s ${Object.keys(value).length}`) + } + + console.log('Step 2/4: Fetch latest Elements to modify from OSM API') + await fetchElements() + + console.log('Step 3/4: Create changes') + createChanges() + console.log(changes) + + console.log('Step 4/4: Upload changes') + await uploadChanges() + + console.log('Success') +} + +run().catch(console.error) diff --git a/package.json b/package.json index 85c91ea..726a9cb 100644 --- a/package.json +++ b/package.json @@ -22,12 +22,14 @@ "geojsontoosm": "^0.0.3", "mktemp": "^1.0.0", "ndjson": "^2.0.0", + "node-fetch": "^2.6.1", "object.omit": "^3.0.0", "osm-geojson": "^0.8.4", "polygon-lookup": "^2.6.0", "readable-stream": "^3.6.0", "string-comparison": "^1.0.9", "tape": "^5.2.2", + "xml-js": "^1.6.11", "yargs": "^17.0.1" }, "engines": { @@ -851,6 +851,11 @@ no-case@^3.0.4: lower-case "^2.0.2" tslib "^2.0.3" +node-fetch@^2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" + integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== + oauth-sign@~0.9.0: version "0.9.0" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" @@ -1100,6 +1105,11 @@ safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +sax@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" + integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== + shallow-clone@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3" @@ -1369,6 +1379,13 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= +xml-js@^1.6.11: + version "1.6.11" + resolved "https://registry.yarnpkg.com/xml-js/-/xml-js-1.6.11.tgz#927d2f6947f7f1c19a316dd8eea3614e8b18f8e9" + integrity sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g== + dependencies: + sax "^1.2.4" + xmldom@0.1.19: version "0.1.19" resolved "https://registry.yarnpkg.com/xmldom/-/xmldom-0.1.19.tgz#631fc07776efd84118bf25171b37ed4d075a0abc" |