unit from number create changeset from maproulette setTags operations

5 files changed, 467 insertions, 3 deletions

diff --git a/Makefile b/Makefile
index b9604ba..3acd27b 100644
--- a/Makefile
+++ b/Makefile
@@ -190,6 +190,9 @@ dist/conflate:
 	mkdir -p $@
 	./bin/conflate.js dist/vicmap-osm-uniq-flats.geojson data/victoria-addr.osm.geojson dist/blocksByOSMAddr.geojson $@
 
+dist/unitFromNumber.osc: dist/conflate/mr_explodeUnitFromNumber.geojson
+	./bin/mr2osc.mjs --dry-run $< $@
+
 convertConflationResultsToFGB:
 	ogr2ogr -f FlatGeobuf dist/conflate/withinExistingOSMAddressPoly.fgb dist/conflate/withinExistingOSMAddressPoly.geojson
 	ogr2ogr -f FlatGeobuf dist/conflate/notFoundInBlocks.fgb dist/conflate/notFoundInBlocks.geojson
diff --git a/README.md b/README.md
index ab2febe..ccc336d 100644
--- a/README.md
+++ b/README.md
@@ -306,15 +306,19 @@ Split into the following candidate categories, then again split into suburb/loca
 
 The dedicated import account used for the import is [`vicmap_import`](https://www.openstreetmap.org/user/vicmap_import).
 
-
 ### Stage 1 - postal_code
-
 For background see [Inclusion of `addr:suburb`, `addr:postcode` and `addr:state`](#inclusion-of-addrsuburb-addrpostcode-and-addrstate).
 
 Using JOSM RemoteControl commands [`postal_code`](https://wiki.openstreetmap.org/wiki/Key:postal_code) will be added to the existing Victorian `admin_level=10` boundaries using the postcode derived from Vicmap Addresses. Except for Melbourne suburb because there are two postal codes in use, and the `postal_code` boundaries are already mapped.
 
+### Stage 2 - Set unit from housenumber
+During the conflation stage, Vicmap addresses which were deemed to match OSM addresses where in OSM it was represented as `addr:housenumber=X/Y` whereas in Vicmap it was represented as `addr:unit=X`, `addr:housenumber=Y`, then an automated tag change to move the unit into `addr:unit` is performed.
+
+This will be a single Victoria wide changeset.
+
+`bin/mr2osc.mjs` is the script which creates the OsmChange and uploads it as a changeset from the tag changes outputted from the _conflate_ stage as a MapRoulette `setTags` operation.
 
-### Stage 2 - 
+### Stage 3 - New addresses without conflicts
 
 `bin/upload.sh` is the script used to perform the actual uploads into OSM. For each import candidate (by candidate category by state) there is one OSM Changeset created.
 
diff --git a/bin/mr2osc.mjs b/bin/mr2osc.mjs
new file mode 100755
index 0000000..25f008e
--- /dev/null
+++ b/bin/mr2osc.mjs
@@ -0,0 +1,438 @@
+#!/usr/bin/env node
+
+/**
+ * Given a MapRoulette Cooperative Challenge, upload all the changes directly without review
+ */
+
+import fs from 'fs'
+import { pipeline } from 'stream/promises'
+import { Transform } from 'stream'
+import ndjson from 'ndjson'
+import _ from 'lodash'
+import fetch from 'node-fetch'
+import xml from 'xml-js'
+import yargs from 'yargs'
+
+const argv = yargs(process.argv.slice(2))
+  .option('dry-run', {
+    type: 'boolean',
+    description: 'Skip uploading to OSM, instead log what we would do'
+  })
+  .option('changeset-comment', {
+    type: 'string',
+    description: 'OSM Changeset comment',
+    demandOption: true
+  })
+  .argv
+
+if (argv._.length < 2) {
+  console.error("Usage: ./mr2osc.js mr.json change.osc")
+  process.exit(1)
+}
+
+const mrFile = argv._[0]
+const changeFile = argv._[1]
+
+if (!fs.existsSync(mrFile)) {
+  console.error(`${mrFile} not found`)
+  process.exit(1)
+}
+
+if (argv.dryRun) {
+  console.log('Dry run enabled')
+}
+
+// https://wiki.openstreetmap.org/wiki/API_v0.6#Multi_fetch:_GET_.2Fapi.2F0.6.2F.5Bnodes.7Cways.7Crelations.5D.3F.23parameters
+const MAXIMUM_ELEMENTS_PER_GET_REQUEST = 725
+
+const OSM_API_READ = 'https://api.openstreetmap.org'
+const OSM_API_WRITE = (process.env.ENVIRONMENT === 'prod') ? 'https://api.openstreetmap.org' : 'https://master.apis.dev.openstreetmap.org'
+const USER_AGENT = 'vicmap2osm/1.0 (+https://gitlab.com/alantgeo/vicmap2osm)'
+const CREATED_BY = 'https://gitlab.com/alantgeo/vicmap2osm'
+const AUTHORIZATION = `Basic ${Buffer.from(process.env.OSM_USERNAME + ':' + process.env.OSM_PASSWORD).toString('base64')}`
+let MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST = 10000
+
+console.log(`Retrieving capabilities from ${OSM_API_WRITE}`)
+await fetch(`${OSM_API_WRITE}/api/capabilities`, {
+  headers: {
+    'User-Agent': USER_AGENT
+  }
+})
+  .then(res => res.text())
+  .then(text => {
+    const capabilities = JSON.parse(xml.xml2json(text, {
+      compact: true
+    }))
+    const apiMaximumElements = capabilities.osm.api.changesets._attributes.maximum_elements
+    MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST = Math.min(MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST, apiMaximumElements)
+  })
+
+const operations = {
+  node: {},
+  way: {},
+  relation: {}
+}
+
+// full OSM Elements live from OSM API
+const elements = {}
+
+const changes = {
+  node: [],
+  way: [],
+  relation: []
+}
+
+let taskCount = 0
+/**
+ * Transform which receives MapRoulette tasks and
+ * stores operations by Element type by id in `operations`.
+ */
+const listElements = new Transform({
+  readableObjectMode: true,
+  writableObjectMode: true,
+  transform(task, encoding, callback) {
+    taskCount++
+
+    if (process.stdout.isTTY && taskCount % 1000 === 0) {
+      process.stdout.write(` ${taskCount.toLocaleString()}\r`)
+    }
+
+    if (task && task.features && task.features.length && task.cooperativeWork && task.cooperativeWork.operations && task.cooperativeWork.operations.length) {
+      const operation = task.cooperativeWork.operations[0]
+
+      if (operation.operationType === 'modifyElement') {
+        const type = operation.data.id.split('/')[0]
+        const id = operation.data.id.split('/')[1]
+        operations[type][id] = operation.data.operations
+      }
+    }
+
+    callback()
+  }
+})
+
+/**
+ * Fetch all Elements with operations from `operations`
+ * so we have the full Element to generate the OsmChange.
+ * Results stored in `elements`.
+ */
+async function fetchElements() {
+  /*
+  const nodeChunks = _.chunk(Object.keys(operations.nodes), MAXIMUM_ELEMENTS_PER_GET_REQUEST)
+  const wayChunks = _.chunk(Object.keys(operations.ways), MAXIMUM_ELEMENTS_PER_GET_REQUEST)
+  const relationChunks = _.chunk(Object.keys(operations.relations), MAXIMUM_ELEMENTS_PER_GET_REQUEST)
+  */
+
+  for (const type in operations) {
+    let index = 0
+    const chunks = _.chunk(Object.keys(operations[type]), MAXIMUM_ELEMENTS_PER_GET_REQUEST)
+    for (const chunk of chunks) {
+      index++
+
+      process.stdout.write(`   Fetch ${type}s (${index}/${chunks.length})\r`)
+      await fetch(`${OSM_API_READ}/api/0.6/${type}s?${type}s=${chunk.join(',')}`, {
+        headers: {
+          'Accept': 'application/json',
+          'User-Agent': USER_AGENT
+        }
+      })
+        .then(res => res.json())
+        .then(json => {
+          json.elements.forEach(element => {
+            elements[`${element.type}/${element.id}`] = element
+          })
+        })
+        .catch(err => {
+          console.error(err)
+          process.exit(1)
+        })
+    }
+    process.stdout.write(`\r`)
+    process.stdout.write(`   Fetched ${chunks.length} ${type}s\n`)
+  }
+  return Promise.resolve()
+
+  /*
+  return [
+    ...nodeChunks.map(nodeChunk => {
+      return fetch(`${OSM_API_READ}/api/0.6/nodes?nodes=${nodeChunk.join(',')}`, {
+        headers: {
+          'Accept': 'application/json',
+          'User-Agent': USER_AGENT
+        }
+      })
+        .then(res => res.json())
+        .then(json => {
+          json.elements.forEach(element => {
+            elements[`${element.type}/${element.id}`] = element
+          })
+        })
+    }),
+    ...wayChunks.map(wayChunk => {
+      return fetch(`${OSM_API_READ}/api/0.6/ways?ways=${wayChunk.join(',')}`, {
+        headers: {
+          'Accept': 'application/json',
+          'User-Agent': USER_AGENT
+        }
+      })
+        .then(res => res.json())
+        .then(json => {
+          json.elements.forEach(element => {
+            elements[`${element.type}/${element.id}`] = element
+          })
+        })
+    }),
+    ...relationChunks.map(relationChunk => {
+      return fetch(`${OSM_API_READ}/api/0.6/relations?relations=${relationChunk.join(',')}`, {
+        headers: {
+          'Accept': 'application/json',
+          'User-Agent': USER_AGENT
+        }
+      })
+        .then(res => res.json())
+        .then(json => {
+          json.elements.forEach(element => {
+            elements[`${element.type}/${element.id}`] = element
+          })
+        })
+    })
+  ])
+  */
+}
+
+function createChanges() {
+  for (const type in operations) {
+    for (const [id, ops] of Object.entries(operations[type])) {
+      const element = elements[`${type}/${id}`]
+      const tags = []
+
+      if (ops && ops.length) {
+        ops.forEach(operation => {
+          if (operation.operation === 'setTags') {
+            // first ensure that our assumptions about the current values of these tags hasn't changed
+            // TODO
+
+            // tags to set
+            for (const [key, value] of Object.entries(operation.data)) {
+              // replace with new tag
+              element.tags[key] = value
+            }
+          }
+        })
+      }
+
+      // set nodeChanges for this node
+      for (const [key, value] of Object.entries(element.tags)) {
+        const tag = {
+          _attributes: {
+            k: key,
+            v: value
+          }
+        }
+        tags.push(tag)
+      }
+      switch (type) {
+        case 'node':
+          changes[type].push({
+            _attributes: {
+              id: id,
+              version: element.version + 1,
+              lat: element.lat,
+              lon: element.lon
+            },
+            tag: tags
+          })
+          break
+        case 'way':
+          changes[type].push({
+            _attributes: {
+              id: id,
+              version: element.version + 1,
+            },
+            tag: tags,
+            nodes: element.nodes
+          })
+          break
+        case 'relation':
+          changes[type].push({
+            _attributes: {
+              id: id,
+              version: element.version + 1,
+            },
+            tag: tags,
+            members: element.members
+          })
+          break
+      }
+    }
+  }
+}
+
+async function uploadChanges() {
+  // now we have the latest full elements, apply our changes as an OsmChange
+  // https://wiki.openstreetmap.org/wiki/API_v0.6#Diff_upload:_POST_.2Fapi.2F0.6.2Fchangeset.2F.23id.2Fupload
+
+  const totalElements = Object.values(changes).flat().length
+  const totalChangesets = Math.ceil(totalElements / MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST)
+  if (totalChangesets > 1) {
+    console.log(`${totalElements} exceeds API maximum elements of ${MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST} splitting into ${totalChangesets} changesets`)
+    process.exit(1)
+  }
+
+  for (let changesetIndex = 0; changesetIndex < totalChangesets; changesetIndex++) {
+    // create a changeset
+    let changesetId
+    const createBody = xml.json2xml({
+      _declaration: {
+        _attributes: {
+          version: "1.0",
+          encoding: "UTF-8"
+        }
+      },
+      osm: {
+        changeset: {
+          tag: [
+            {
+              _attributes: {
+                k: 'created_by',
+                v: CREATED_BY
+              }
+            },
+            {
+              _attributes: {
+                k: 'comment',
+                v: argv.changesetComment
+              }
+            }
+          ]
+        }
+      }
+    }, Object.assign({compact: true}, argv.dryRun ? { spaces: 2 } : {}))
+    if (argv.dryRun) {
+      console.log(`${OSM_API_WRITE}/api/0.6/changeset/create`)
+      console.log(createBody)
+    } else {
+      changesetId = await fetch(`${OSM_API_WRITE}/api/0.6/changeset/create`, {
+        method: 'PUT',
+        headers: {
+          'Authorization': AUTHORIZATION,
+          'User-Agent': USER_AGENT
+        },
+        body: createBody
+      })
+        .then(res => res.text())
+      console.log(`Opened changeset ${changesetId}`)
+    }
+
+    /*
+    const nodeChangesCount = changes.node.length
+    const wayChangesCount = changes.node.length
+    const relationChangesCount = changes.node.length
+
+    const startIndex = changeIndex * MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST
+    const endIndex = ((changeIndex + 1) * MAXIMUM_ELEMENTS_PER_UPLOAD_REQUEST) - 1
+
+    if (startIndex < nodeChangesCount) {
+      changes.node.slice(startIndex, )
+    }
+    */
+
+    // upload to the changeset
+    const uploadBody = xml.json2xml({
+      _declaration: {
+        _attributes: {
+          version: "1.0",
+          encoding: "UTF-8"
+        }
+      },
+      osmChange: {
+        _attributes: {
+          version: '0.6',
+          generator: CREATED_BY
+        },
+        modify: {
+          node: changes.node.map(change => insertChangesetId(change, changesetId)),
+          way: changes.way.map(change => insertChangesetId(change, changesetId)),
+          relation: changes.relation.map(change => insertChangesetId(change, changesetId))
+        }
+      }
+    }, Object.assign({
+      compact: true,
+      attributeValueFn: value => {
+        // TODO need to test this via the dev api
+        return value.replace(/&quot;/g, '"')  // convert quote back before converting amp
+          .replace(/&/g, '&amp;')
+          .replace(/</g, '&lt;')
+          .replace(/>/g, '&gt;')
+          .replace(/"/g, '&quot;')
+          .replace(/'/g, '&apos;')
+      }
+    }, argv.dryRun ? { spaces: 2 } : {}))
+
+    const filePath = totalChangesets > 1 ? path.join(path.dirname(changeFile), path.basename(changeFile, '.osc') + '-' + (changesetIndex + 1) + '.osc') : changeFile
+    fs.writeFileSync(filePath, uploadBody)
+    console.log(`Saved ${filePath}`)
+
+    if (argv.dryRun) {
+      console.log(`${OSM_API_WRITE}/api/0.6/changeset/${changesetId}/upload`)
+    } else {
+      await fetch(`${OSM_API_WRITE}/api/0.6/changeset/${changesetId}/upload`, {
+        method: 'POST',
+        headers: {
+          'Authorization': AUTHORIZATION,
+          'User-Agent': USER_AGENT
+        },
+        body: uploadBody
+      })
+      console.log(`Uploaded contents to ${changesetId}`)
+    }
+
+    // close the changeset
+    if (argv.dryRun) {
+      console.log(`${OSM_API_WRITE}/api/0.6/changeset/${changesetId}/close`)
+    } else {
+      await fetch(`${OSM_API_WRITE}/api/0.6/changeset/${changesetId}/close`, {
+        method: 'PUT',
+        headers: {
+          'Authorization': AUTHORIZATION,
+          'User-Agent': USER_AGENT
+        }
+      })
+      console.log(`Closed changeset ${changesetId}`)
+    }
+  }
+
+  return
+}
+
+function insertChangesetId(change, changesetId) {
+  change._attributes.changeset = changesetId
+  return change
+}
+
+console.log('Step 1/4: Find Elements to modify')
+async function run() {
+  await pipeline(
+    fs.createReadStream(mrFile),
+    ndjson.parse(),
+    listElements,
+  )
+
+  for (const [type, value] of Object.entries(operations)) {
+    console.log(`   ${type}s ${Object.keys(value).length}`)
+  }
+
+  console.log('Step 2/4: Fetch latest Elements to modify from OSM API')
+  await fetchElements()
+
+  console.log('Step 3/4: Create changes')
+  createChanges()
+  console.log(changes)
+
+  console.log('Step 4/4: Upload changes')
+  await uploadChanges()
+
+  console.log('Success')
+}
+
+run().catch(console.error)
diff --git a/package.json b/package.json
index 85c91ea..726a9cb 100644
--- a/package.json
+++ b/package.json
@@ -22,12 +22,14 @@
     "geojsontoosm": "^0.0.3",
     "mktemp": "^1.0.0",
     "ndjson": "^2.0.0",
+    "node-fetch": "^2.6.1",
     "object.omit": "^3.0.0",
     "osm-geojson": "^0.8.4",
     "polygon-lookup": "^2.6.0",
     "readable-stream": "^3.6.0",
     "string-comparison": "^1.0.9",
     "tape": "^5.2.2",
+    "xml-js": "^1.6.11",
     "yargs": "^17.0.1"
   },
   "engines": {
diff --git a/yarn.lock b/yarn.lock
index ee62a0b..558f67a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -851,6 +851,11 @@ no-case@^3.0.4:
     lower-case "^2.0.2"
     tslib "^2.0.3"
 
+node-fetch@^2.6.1:
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
+  integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
+
 oauth-sign@~0.9.0:
   version "0.9.0"
   resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455"
@@ -1100,6 +1105,11 @@ safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
   integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==
 
+sax@^1.2.4:
+  version "1.2.4"
+  resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
+  integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
+
 shallow-clone@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3"
@@ -1369,6 +1379,13 @@ wrappy@1:
   resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
   integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
 
+xml-js@^1.6.11:
+  version "1.6.11"
+  resolved "https://registry.yarnpkg.com/xml-js/-/xml-js-1.6.11.tgz#927d2f6947f7f1c19a316dd8eea3614e8b18f8e9"
+  integrity sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==
+  dependencies:
+    sax "^1.2.4"
+
 xmldom@0.1.19:
   version "0.1.19"
   resolved "https://registry.yarnpkg.com/xmldom/-/xmldom-0.1.19.tgz#631fc07776efd84118bf25171b37ed4d075a0abc"