diff options
Diffstat (limited to 'upload')
| -rw-r--r-- | upload/README.md | 7 | ||||
| -rwxr-xr-x | upload/change2diff.py | 107 | ||||
| -rw-r--r-- | upload/change2diff2.py | 121 | ||||
| -rwxr-xr-x | upload/close.py | 225 | ||||
| -rwxr-xr-x | upload/conflict-view.py | 78 | ||||
| -rw-r--r-- | upload/diffpatch | 13 | ||||
| -rwxr-xr-x | upload/diffpatch.py | 76 | ||||
| -rw-r--r-- | upload/osm-merge | 16 | ||||
| -rw-r--r-- | upload/osm2change-reorder.patch | 38 | ||||
| -rwxr-xr-x | upload/osm2change.py | 85 | ||||
| -rwxr-xr-x | upload/osmpatch.py | 144 | ||||
| -rwxr-xr-x | upload/set-changeset-tag.py | 252 | ||||
| -rwxr-xr-x | upload/smarter-sort.py | 309 | ||||
| -rwxr-xr-x | upload/split.py | 117 | ||||
| -rwxr-xr-x | upload/upload.py | 432 |
15 files changed, 2020 insertions, 0 deletions
diff --git a/upload/README.md b/upload/README.md new file mode 100644 index 0000000..6cafa2d --- /dev/null +++ b/upload/README.md @@ -0,0 +1,7 @@ +## Clone/improvements for Openstreetmap CLI upload tools + +This is a copy of Openstreetmap tools documented at https://wiki.openstreetmap.org/wiki/Upload.py +I plan to support them working good enough for my own needs, first by fixing problems with deprecation, then by +error fixing. Maybe new features will come later. + +The scripts need Python 3, and should use HTTPS API access method. diff --git a/upload/change2diff.py b/upload/change2diff.py new file mode 100755 index 0000000..3ec11dc --- /dev/null +++ b/upload/change2diff.py @@ -0,0 +1,107 @@ +#! /usr/bin/python +# vim: fileencoding=utf-8 encoding=utf-8 et sw=4 + +# Copyright (C) 2009 Andrzej Zaborowski <balrogg@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +""" +Generate a .diff.xml file (the response from the server after a diff upload) +from an uploaded changeset file (downloadable through +https://www.openstreetmap.org/api/0.6/changeset/<id>/download) -- this is +useful if the network connection broke after uploading the changeset but +before receiving the server response. +""" + +__version__ = "$Revision: 21 $" + +import os +import sys +import traceback +import codecs +import locale +import subprocess + +import httplib + +import xml.etree.cElementTree as ElementTree + +import locale, codecs +locale.setlocale(locale.LC_ALL, "en_US.UTF-8") +encoding = locale.getlocale()[1] +sys.stdout = codecs.getwriter(encoding)(sys.stdout, errors = "replace") +sys.stderr = codecs.getwriter(encoding)(sys.stderr, errors = "replace") + +try: + this_dir = os.path.dirname(__file__) + version = subprocess.Popen(["svnversion", this_dir], stdout = subprocess.PIPE).communicate()[0].strip() + if len(sys.argv) != 2: + print >>sys.stderr, u"Synopsis:" + print >>sys.stderr, u" %s <file_name>" + sys.exit(1) + + filename = sys.argv[1] + if not os.path.exists(filename): + print >>sys.stderr, u"File %r doesn't exist!" % (filename,) + sys.exit(1) + if filename.endswith(".osc"): + filename_base = filename[:-4] + else: + filename_base = filename + + tree = ElementTree.parse(filename) + root = tree.getroot() + if root.tag != "osmChange" or (root.attrib.get("version") != "0.3" and + root.attrib.get("version") != "0.6"): + print >>sys.stderr, u"File %s is not a v0.3 osmChange file!" % (filename,) + sys.exit(1) + + diff_attr = {"version": "0.6", "generator": root.attrib.get("generator")} + diff_root = ElementTree.Element("diffResult", diff_attr) + diff_tree = ElementTree.ElementTree(diff_root) + + # Note this is broken, it assumes the nodes in the resulting osmChange + # are in the same order they were in the osmChange sent to the server + # and that the negative IDs there started at -1 and were increasing by + # -1 with each new element. + # A better idea (but still wrong) would be to parse the input osmChange + # xml at the same time and assume that the elements in input and output + # come in the same order, possibly with additional checks (lat/lon..) + old_id = -1 + for operation in root: + for element in operation: + attr = {} + if operation.tag == "create": + attr["old_id"] = str(old_id) + attr["new_id"] = element.attrib.get("id") + attr["new_version"] = element.attrib.get("version") + old_id -= 1 + elif operation.tag == "modify": + attr["old_id"] = element.attrib.get("id") + attr["new_id"] = element.attrib.get("id") + attr["new_version"] = element.attrib.get("version") + elif operation.tag == "delete": + attr["old_id"] = element.attrib.get("id") + else: + print "unknown operation", operation.tag + sys.exit(-1) + diff = ElementTree.SubElement(diff_root, element.tag, attr) + + diff_tree.write(filename_base + ".diff.xml", "utf-8") + +except Exception,err: + print >>sys.stderr, repr(err) + traceback.print_exc(file=sys.stderr) + sys.exit(1) diff --git a/upload/change2diff2.py b/upload/change2diff2.py new file mode 100644 index 0000000..e6643d4 --- /dev/null +++ b/upload/change2diff2.py @@ -0,0 +1,121 @@ +#! /usr/bin/python2 +# vim: fileencoding=utf-8 encoding=utf-8 et sw=4 + +# Copyright (C) 2009 Andrzej Zaborowski <balrogg@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +""" +Generate a .diff.xml file (the response from the server after a diff upload) +from an uploaded changeset file (downloadable through +https://www.openstreetmap.org/api/0.6/changeset/<id>/download) -- this is +useful if the network connection broke after uploading the changeset but +before receiving the server response. +""" + +__version__ = "$Revision: 21 $" + +import os +import sys +import traceback +import codecs +import locale +import subprocess + +import httplib + +import xml.etree.cElementTree as ElementTree + +import locale, codecs +locale.setlocale(locale.LC_ALL, "en_US.UTF-8") +encoding = locale.getlocale()[1] +sys.stdout = codecs.getwriter(encoding)(sys.stdout, errors = "replace") +sys.stderr = codecs.getwriter(encoding)(sys.stderr, errors = "replace") + +try: + this_dir = os.path.dirname(__file__) + version = subprocess.Popen(["svnversion", this_dir], stdout = subprocess.PIPE).communicate()[0].strip() + if len(sys.argv) != 3: + print >>sys.stderr, u"Synopsis:" + print >>sys.stderr, u" %s <old-file-name.osc> <new-file-name.osc>" + sys.exit(1) + + oldtree = ElementTree.parse(sys.argv[1]) + oldroot = oldtree.getroot() + if oldroot.tag != "osmChange" or (oldroot.attrib.get("version") != "0.3" and + oldroot.attrib.get("version") != "0.6"): + print >>sys.stderr, u"File %s is not a v0.3 osmChange file!" % (sys.argv[1],) + sys.exit(1) + + old_ids = [] + for operation in oldroot: + if operation.tag == "create": + for element in operation: + old_ids.append(int(element.attrib.get("id"))) + + filename = sys.argv[2] + if not os.path.exists(filename): + print >>sys.stderr, u"File %r doesn't exist!" % (filename,) + sys.exit(1) + if filename.endswith(".osc"): + filename_base = filename[:-4] + else: + filename_base = filename + + tree = ElementTree.parse(filename) + root = tree.getroot() + if root.tag != "osmChange" or (root.attrib.get("version") != "0.3" and + root.attrib.get("version") != "0.6"): + print >>sys.stderr, u"File %s is not a v0.3 osmChange file!" % (filename,) + sys.exit(1) + + diff_attr = {"version": "0.6", "generator": root.attrib.get("generator")} + diff_root = ElementTree.Element("diffResult", diff_attr) + diff_tree = ElementTree.ElementTree(diff_root) + + # Note this is broken, it assumes the nodes in the resulting osmChange + # are in the same order they were in the osmChange sent to the server + # and that the negative IDs there started at -1 and were increasing by + # -1 with each new element. + # A better idea (but still wrong) would be to parse the input osmChange + # xml at the same time and assume that the elements in input and output + # come in the same order, possibly with additional checks (lat/lon..) + old_id = 0 + for operation in root: + for element in operation: + attr = {} + # TODO: at least make sure the element type matches!! + if operation.tag == "create": + attr["old_id"] = str(old_ids[old_id]) + attr["new_id"] = element.attrib.get("id") + attr["new_version"] = element.attrib.get("version") + old_id += 1 + elif operation.tag == "modify": + attr["old_id"] = element.attrib.get("id") + attr["new_id"] = element.attrib.get("id") + attr["new_version"] = element.attrib.get("version") + elif operation.tag == "delete": + attr["old_id"] = element.attrib.get("id") + else: + print "unknown operation", operation.tag + sys.exit(-1) + diff = ElementTree.SubElement(diff_root, element.tag, attr) + + diff_tree.write(filename_base + ".diff.xml", "utf-8") + +except Exception,err: + print >>sys.stderr, repr(err) + traceback.print_exc(file=sys.stderr) + sys.exit(1) diff --git a/upload/close.py b/upload/close.py new file mode 100755 index 0000000..c3ee3e3 --- /dev/null +++ b/upload/close.py @@ -0,0 +1,225 @@ +#! /usr/bin/python3 +# vim: fileencoding=utf-8 encoding=utf-8 et sw=4 + +# Copyright (C) 2009 Jacek Konieczny <jajcus@jajcus.net> +# Copyright (C) 2009 Andrzej Zaborowski <balrogg@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +""" +Closes a changeset, given id. +""" + +__version__ = "$Revision: 21 $" + +import os +import subprocess +import sys +import traceback +import base64 + +import http.client as httplib +import xml.etree.cElementTree as ElementTree +import urllib.parse as urlparse + +class HTTPError(Exception): + pass + +class OSM_API(object): + url = 'https://master.apis.dev.openstreetmap.org/' + def __init__(self, username = None, password = None): + if username and password: + self.username = username + self.password = password + else: + self.username = "" + self.password = "" + self.changeset = None + self.progress_msg = None + + def __del__(self): + if self.changeset is not None: + self.close_changeset() + + def msg(self, mesg): + sys.stderr.write("\r%s… " % (self.progress_msg)) + sys.stderr.write("\r%s… %s" % (self.progress_msg, mesg)) + sys.stderr.flush() + + def request(self, conn, method, url, body, headers, progress): + if progress: + self.msg("making request") + conn.putrequest(method, url) + self.msg("sending headers") + if body: + conn.putheader('Content-Length', str(len(body))) + for hdr, value in headers.iteritems(): + conn.putheader(hdr, value) + self.msg("end of headers") + conn.endheaders() + self.msg(" 0%") + if body: + start = 0 + size = len(body) + chunk = size / 100 + if chunk < 16384: + chunk = 16384 + while start < size: + end = min(size, start + chunk) + conn.send(body[start:end]) + start = end + self.msg("%2i%%" % (start * 100 / size)) + else: + self.msg(" ") + conn.request(method, url, body, headers) + + def _run_request(self, method, url, body = None, progress = 0, content_type = "text/xml"): + url = urlparse.urljoin(self.url, url) + purl = urlparse.urlparse(url) + if purl.scheme != "https": + raise ValueError("Unsupported url scheme: %r" % (purl.scheme,)) + if ":" in purl.netloc: + host, port = purl.netloc.split(":", 1) + port = int(port) + else: + host = purl.netloc + port = None + url = purl.path + if purl.query: + url += "?" + query + headers = {} + if body: + headers["Content-Type"] = content_type + + try_no_auth = 0 + + if not try_no_auth and not self.username: + raise HTTPError("Need a username") + + try: + self.msg("connecting") + conn = httplib.HTTPSConnection(host, port) +# conn.set_debuglevel(10) + + if try_no_auth: + self.request(conn, method, url, body, headers, progress) + self.msg("waiting for status") + response = conn.getresponse() + + if not try_no_auth or (response.status == httplib.UNAUTHORIZED and + self.username): + if try_no_auth: + conn.close() + self.msg("re-connecting") + conn = httplib.HTTPSConnection(host, port) +# conn.set_debuglevel(10) + + creds = self.username + ":" + self.password + headers["Authorization"] = "Basic " + \ + base64.b64encode(bytes(creds, "utf8")).decode("utf8") + # ^ Seems to be broken in python3 (even the raw + # documentation examples don't run for base64) + self.request(conn, method, url, body, headers, progress) + self.msg("waiting for status") + response = conn.getresponse() + + if response.status == httplib.OK: + self.msg("reading response") + sys.stderr.flush() + response_body = response.read() + else: + raise HTTPError("%02i: %s (%s)" % (response.status, + response.reason, response.read())) + finally: + conn.close() + return response_body + + def create_changeset(self, created_by, comment): + if self.changeset is not None: + raise RuntimeError("Changeset already opened") + self.progress_msg = "I'm creating the changeset" + self.msg("") + sys.stderr.flush() + root = ElementTree.Element("osm") + tree = ElementTree.ElementTree(root) + element = ElementTree.SubElement(root, "changeset") + ElementTree.SubElement(element, "tag", {"k": "created_by", "v": created_by}) + ElementTree.SubElement(element, "tag", {"k": "comment", "v": comment}) + body = ElementTree.tostring(root, "utf-8") + reply = self._run_request("PUT", "/api/0.6/changeset/create", body) + changeset = int(reply.strip()) + self.msg("done. Id: %i" % (changeset,)) + sys.stderr("\n") + self.changeset = changeset + + def upload(self, change): + if self.changeset is None: + raise RuntimeError("Changeset not opened") + self.progress_msg = "Now I'm sending changes" + self.msg("") + sys.stderr.flush() + for operation in change: + if operation.tag not in ("create", "modify", "delete"): + continue + for element in operation: + element.attrib["changeset"] = str(self.changeset) + body = ElementTree.tostring(change, "utf-8") + reply = self._run_request("POST", "/api/0.6/changeset/%i/upload" + % (self.changeset,), body, 1) + self.msg("done.") + sys.stderr.write("\n") + return reply + + def close_changeset(self): + if self.changeset is None: + raise RuntimeError("Changeset not opened") + self.progress_msg = "Closing" + self.msg("") + sys.stderr.flush() + sys.stderr.flush() + reply = self._run_request("PUT", "/api/0.6/changeset/%i/close" + % (self.changeset,)) + self.changeset = None + self.msg("done, too.") + sys.stderr.write("\n") + +try: + this_dir = os.path.dirname(__file__) + try: + version = int(subprocess.Popen(["svnversion", this_dir], stdout = subprocess.PIPE).communicate()[0].strip()) + except: + version = 1 + if len(sys.argv) != 2: + sys.stderr.write("Synopsis:\n") + sys.stderr.write(" %s <changeset-id>\n", sys.argv[0]) + sys.exit(1) + + login = input("OSM login: ") + if not login: + sys.exit(1) + password = input("OSM password: ") + if not login: + sys.exit(1) + + api = OSM_API(login, password) + api.changeset = int(sys.argv[1]) + api.close_changeset() +except HTTPError as err: + sys.stderr.write(str(err) + "\n") + sys.exit(1) +except Exception as err: + sys.stderr.write(str(err) + "\n") + traceback.print_exc(file=sys.stderr) + sys.exit(1) diff --git a/upload/conflict-view.py b/upload/conflict-view.py new file mode 100755 index 0000000..cda91ad --- /dev/null +++ b/upload/conflict-view.py @@ -0,0 +1,78 @@ +#! /usr/bin/python +# vim: fileencoding=utf-8 encoding=utf-8 et sw=4 + +# Copyright (C) 2009 Andrzej Zaborowski <balrogg@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +import os +import sys +import math +import codecs +import locale + +import httplib + +import xml.etree.cElementTree as ElementTree + +import locale, codecs +locale.setlocale(locale.LC_ALL, "en_US.UTF-8") +encoding = locale.getlocale()[1] +sys.stdout = codecs.getwriter(encoding)(sys.stdout, errors = "replace") +sys.stderr = codecs.getwriter(encoding)(sys.stderr, errors = "replace") + +def osmparse(filename): + tree = ElementTree.parse(filename).getroot() + elems = {} + if tree.tag == "osm" and tree.attrib.get("version") == "0.6": + for element in tree: + if not "id" in element.attrib: + continue + id = element.attrib["id"] + if "version" in element.attrib: + v = element.attrib["version"] + else: + v = "0" + if "action" in element.attrib: + elems[id] = (element.attrib["action"], v, element) + elif tree.tag == "osmChange" and \ + tree.attrib.get("version") in [ "0.3", "0.6" ]: + for op in tree: + for element in op: + id = element.attrib["id"] + if "version" in element.attrib: + v = element.attrib["version"] + else: + v = "0" + elems[id] = (op.tag, v, element) + else: + print >>sys.stderr, u"File %s is in unknown format %s v%s!" % \ + (filename, tree.tag, tree.attrib["version"]) + sys.exit(1) + return elems + +if len(sys.argv) != 3: + print >>sys.stderr, u"Synopsis:" + print >>sys.stderr, u" %s <osm-or-osc-file> <osm-or-osc-file>" + sys.exit(1) + +a = osmparse(sys.argv[1]) +b = osmparse(sys.argv[2]) + +for id in a: + if id in b: + print a[id][2].tag + " " + id + ":", \ + "A:", a[id][0], "(v" + a[id][1] + ")", \ + "B:", b[id][0], "(v" + b[id][1] + ")" diff --git a/upload/diffpatch b/upload/diffpatch new file mode 100644 index 0000000..1538a31 --- /dev/null +++ b/upload/diffpatch @@ -0,0 +1,13 @@ +#! /bin/bash +[ $# -ge 2 ] || exit -1 + +fn="$1" +shift +grep old_id "$fn" | \ +while read line; do + old=`echo "$line" | grep -o 'old_id="[0-9-]*'` + new=`echo "$line" | grep -o 'new_id="[0-9-]*'` + old=${old:8} + new=${new:8} + sed -i "s/ref=\"${old}\"/ref=\"${new}\"/g" $* +done diff --git a/upload/diffpatch.py b/upload/diffpatch.py new file mode 100755 index 0000000..e95e7dd --- /dev/null +++ b/upload/diffpatch.py @@ -0,0 +1,76 @@ +#! /usr/bin/python2 +# vim: fileencoding=utf-8 encoding=utf-8 et sw=4 + +# Copyright (C) 2009 Jacek Konieczny <jajcus@jajcus.net> +# Copyright (C) 2009 Andrzej Zaborowski <balrogg@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +""" +Patches .osc files with .diff.xml files resulting from an upload of +a previous chunk of a multipart upload. +""" + +__version__ = "$Revision: 21 $" + +import os +import subprocess +import sys +import traceback +import codecs +import locale + +import locale, codecs +locale.setlocale(locale.LC_ALL, "en_US.UTF-8") +encoding = locale.getlocale()[1] +sys.stdout = codecs.getwriter(encoding)(sys.stdout, errors = "replace") +sys.stderr = codecs.getwriter(encoding)(sys.stderr, errors = "replace") + +if len(sys.argv) < 2 or sys.argv[1] == "--help": + print >>sys.stderr, u"Synopsis:" + print >>sys.stderr, u" %s <file.diff.xml> [osm-files-to-patch...]" + sys.exit(1) + +dd = {} + +diff = open(sys.argv[1], "r") +sys.stdout.write("Parsing diff\n") +for line in diff: + oldpos = line.find("old_id=\"") + newpos = line.find("new_id=\"") + if oldpos < 0 or newpos < 0: + continue + + # For the moment assume every element is operated on only + # once in a changeset (TODO) + old = line[oldpos + 8:] + new = line[newpos + 8:] + old = old[:old.find("\"")] + new = new[:new.find("\"")] + dd[old] = new + +for f in sys.argv[2:]: + sys.stdout.write("Parsing " + f + "\n") + change = open(f, "r") + newchange = open(f + ".diffed", "w") + for line in change: + refpos = line.find("ref=\"") + if refpos > -1: + ref = line[refpos + 5:] + ref = ref[:ref.find("\"")] + if ref in dd: + line = line.replace("ref=\"" + ref + "\"", "ref=\"" + dd[ref] + "\"") + newchange.write(line) + newchange.close() diff --git a/upload/osm-merge b/upload/osm-merge new file mode 100644 index 0000000..b3e3720 --- /dev/null +++ b/upload/osm-merge @@ -0,0 +1,16 @@ +#! /bin/bash +# Copyright (C) 2009 Andrzej Zaborowski +# Merge two .osm files without applying fancy logic (JOSM merge layers +# operation tries to be too smart and corrupts data - see bug #2245) +if [ $# != 2 ]; then + echo Usage: $0 a.osm b.osm \> a+b.osm >&2 + exit +fi + +echo "<?xml version='1.0' encoding='UTF-8'?>" +echo "<osm version='0.6' generator='$0'>" +cat "$1" | grep -v "<osm" | grep -v "<\\?xml" | grep -v "</osm" +cat "$2" | grep -v "<osm" | grep -v "<\\?xml" | grep -v "</osm" | \ + sed "s/id='-/id='-1000000/" | \ + sed "s/ref='-/ref='-1000000/" +echo "</osm>" diff --git a/upload/osm2change-reorder.patch b/upload/osm2change-reorder.patch new file mode 100644 index 0000000..d97a879 --- /dev/null +++ b/upload/osm2change-reorder.patch @@ -0,0 +1,38 @@ +--- osm2change.py 2009-09-18 00:14:45.000000000 +0200 ++++ osm2change-modified.py 2009-09-18 00:27:30.000000000 +0200 +@@ -73,7 +73,18 @@ try: + + operation = {} + for opname in [ "create", "modify", "delete" ]: +- operation[opname] = ElementTree.SubElement(output_root, ++ operation[opname] = {} ++ for opname, elname in [ ++ ("create", "node"), ++ ("modify", "node"), ++ ("create", "way"), ++ ("modify", "way"), ++ ("create", "relation"), ++ ("modify", "relation"), ++ ("delete", "relation"), ++ ("delete", "way"), ++ ("delete", "node") ]: ++ operation[opname][elname] = ElementTree.SubElement(output_root, + opname, output_attr) + + for element in root: +@@ -83,12 +94,12 @@ try: + opname = element.attrib.pop("action") + else: + continue +- operation[opname].append(element) ++ operation[opname][element.tag].append(element) + + # Does this account for all cases? Also, is it needed? + # (cases like relations containing relations... is that allowed?) +- osmsort(operation["create"], [ "node", "way", "relation" ]) +- osmsort(operation["delete"], [ "relation", "way", "node" ]) ++ #osmsort(operation["create"], [ "node", "way", "relation" ]) ++ #osmsort(operation["delete"], [ "relation", "way", "node" ]) + + output_tree.write(filename_base + ".osc", "utf-8") + except Exception,err: diff --git a/upload/osm2change.py b/upload/osm2change.py new file mode 100755 index 0000000..8a836d2 --- /dev/null +++ b/upload/osm2change.py @@ -0,0 +1,85 @@ +#! /usr/bin/env python3 + +# Copyright (C) 2009 Jacek Konieczny <jajcus@jajcus.net> +# Copyright (C) 2009 Andrzej Zaborowski <balrogg@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +""" +Convert .osm files to osmChange 0.6 +""" + +import os +import sys +import traceback + +import http + +import xml.etree.cElementTree as ElementTree + +def osmsort(tree, order): + list = tree[0:len(tree)] + list.sort(lambda x, y: order.index(x.tag) - order.index(y.tag)) + tree[0:len(tree)] = list + +try: + if len(sys.argv) != 2: + sys.stderr.write("Synopsis:\n") + sys.stderr.write(" %s <file-name.osm>\n" % (sys.argv[0],)) + sys.exit(1) + + filename = sys.argv[1] + if not os.path.exists(filename): + sys.stderr.write("File %r doesn't exist!\n" % (filename,)) + sys.exit(1) + if filename.endswith(".osm"): + filename_base = filename[:-4] + else: + filename_base = filename + + tree = ElementTree.parse(filename) + root = tree.getroot() + if root.tag != "osm" or root.attrib.get("version") != "0.6": + sys.stderr.write("File %s is not a v0.6 osm file!\n" % (filename,)) + sys.exit(1) + + output_attr = {"version": "0.6", "generator": root.attrib.get("generator")} + output_root = ElementTree.Element("osmChange", output_attr) + output_tree = ElementTree.ElementTree(output_root) + + operation = {} + for opname in [ "create", "modify", "delete" ]: + operation[opname] = ElementTree.SubElement(output_root, + opname, output_attr) + + for element in root: + if "id" in element.attrib and int(element.attrib["id"]) < 0: + opname = "create" + elif "action" in element.attrib: + opname = element.attrib.pop("action") + else: + continue + operation[opname].append(element) + + # Does this account for all cases? Also, is it needed? + # (cases like relations containing relations... is that allowed?) + #osmsort(operation["create"], [ "node", "way", "relation" ]) + #osmsort(operation["delete"], [ "relation", "way", "node" ]) + + output_tree.write(filename_base + ".osc", "utf-8") +except Exception as err: + sys.stderr.write(repr(err) + "\n") + traceback.print_exc(file = sys.stderr) + sys.exit(1) diff --git a/upload/osmpatch.py b/upload/osmpatch.py new file mode 100755 index 0000000..a516c22 --- /dev/null +++ b/upload/osmpatch.py @@ -0,0 +1,144 @@ +#! /usr/bin/python +# vim: fileencoding=utf-8 encoding=utf-8 et sw=4 + +# Copyright (C) 2009 Jacek Konieczny <jajcus@jajcus.net> +# Copyright (C) 2009 Andrzej Zaborowski <balrogg@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +""" +Patches .osm files with .diff.xml files resulting from an upload of +an .osc file, for use when uploading a .osc file produced by osm2change.py +Note this removes deleted elements from the .osm file. +""" + +__version__ = "$Revision: 21 $" + +import os +import subprocess +import sys +import traceback +import codecs +import locale + +import xml.etree.cElementTree as ElementTree + +import locale, codecs +locale.setlocale(locale.LC_ALL, "en_US.UTF-8") +encoding = locale.getlocale()[1] +sys.stdout = codecs.getwriter(encoding)(sys.stdout, errors = "replace") +sys.stderr = codecs.getwriter(encoding)(sys.stderr, errors = "replace") + +if len(sys.argv) < 2 or sys.argv[1] == "--help": + print >>sys.stderr, u"Synopsis:" + print >>sys.stderr, u" %s <file.diff.xml> [osm-files-to-patch...]" + sys.exit(1) + +dd = [ {}, {}, {} ] +ddv = [ {}, {}, {} ] + +# TODO: use ElementTree +# TODO: take multiple diff arguments +diff = open(sys.argv[1], "r") +sys.stdout.write("Parsing diff\n") +for line in diff: + oldpos = line.find("old_id=\"") + newpos = line.find("new_id=\"") + newvpos = line.find("new_version=\"") + if oldpos < 0: + continue + if line.find("node") >= 0: + idx = 0 + elif line.find("way") >= 0: + idx = 1 + elif line.find("relation") >= 0: + idx = 2 + else: + continue + + old = line[oldpos + 8:] + old = old[:old.find("\"")] + if newpos >= 0 and newvpos >= 0: + new = line[newpos + 8:] + newv = line[newvpos + 13:] + new = new[:new.find("\"")] + newv = newv[:newv.find("\"")] + else: + new = 0 + newv = 0 + dd[idx][old] = new + ddv[idx][old] = newv + +for filename in sys.argv[2:]: + sys.stdout.write("Parsing " + filename + "\n") + + if not os.path.exists(filename): + print >>sys.stderr, u"File %r doesn't exist!" % (filename,) + sys.exit(1) + if filename.endswith(".osm"): + filename_base = filename[:-4] + else: + filename_base = filename + + tree = ElementTree.parse(filename) + root = tree.getroot() + if root.tag != "osm" or root.attrib.get("version") != "0.6": + print >>sys.stderr, u"File %s is not a v0.6 osm file!" % (filename,) + sys.exit(1) + + output_attr = {"version": "0.6", "generator": root.attrib.get("generator")} + output_root = ElementTree.Element("osm", output_attr) + output_tree = ElementTree.ElementTree(output_root) + + for element in root: + copy = 1 + + if "id" in element.attrib: + old = element.attrib["id"] + idx = [ "node", "way", "relation" ].index(element.tag) + if old in dd[idx]: + if dd[idx][old] and ddv[idx][old]: + element.attrib["id"] = dd[idx][old] + element.attrib["version"] = ddv[idx][old] + else: + copy = 0 + + if "action" in element.attrib: + action = element.attrib.pop("action") + + if action in [ "delete" ]: + if copy: + print "Bad delete on id " + old + elif action in [ "create", "modify" ]: + if not copy: + print "Bad create/modify on id " + old + else: + print "Bad action on id " + old + + for member in element: + if member.tag in [ "nd", "member" ]: + idx = 0; + if member.tag == "member": + idx = [ "node", "way", "relation" ].index( + member.attrib["type"]) + + ref = member.attrib["ref"] + if ref in dd[idx]: + member.attrib["ref"] = dd[idx][ref] + + if copy: + output_root.append(element) + + output_tree.write(filename_base + ".osm.diffed", "utf-8") diff --git a/upload/set-changeset-tag.py b/upload/set-changeset-tag.py new file mode 100755 index 0000000..9c9787a --- /dev/null +++ b/upload/set-changeset-tag.py @@ -0,0 +1,252 @@ +#! /usr/bin/python2 +# vim: fileencoding=utf-8 encoding=utf-8 et sw=4 + +# Copyright (C) 2009 Jacek Konieczny <jajcus@jajcus.net> +# Copyright (C) 2009 Andrzej Zaborowski <balrogg@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +""" +Set a tag on an open changeset +""" + +__version__ = "$Revision: 21 $" + +import os +import subprocess +import sys +import traceback +import codecs +import locale + +import httplib + +import xml.etree.cElementTree as ElementTree +import urlparse + +import locale, codecs +try: + locale.setlocale(locale.LC_ALL, "en_US.UTF-8") + encoding = locale.getlocale()[1] + sys.stdout = codecs.getwriter(encoding)(sys.stdout, errors = "replace") + sys.stderr = codecs.getwriter(encoding)(sys.stderr, errors = "replace") +except locale.Error: + pass + +class HTTPError(Exception): + pass + +class OSM_API(object): + url = 'https://master.apis.dev.openstreetmap.org/' + def __init__(self, username = None, password = None): + if username and password: + self.username = username + self.password = password + else: + self.username = "" + self.password = "" + self.changeset = None + self.tags = {} + self.progress_msg = None + + def __del__(self): + #if self.changeset is not None: + # self.close_changeset() + pass + + def msg(self, mesg): + sys.stderr.write(u"\r%s… " % (self.progress_msg)) + sys.stderr.write(u"\r%s… %s" % (self.progress_msg, mesg)) + sys.stderr.flush() + + def request(self, conn, method, url, body, headers, progress): + if progress: + self.msg(u"making request") + conn.putrequest(method, url) + self.msg(u"sending headers") + if body: + conn.putheader('Content-Length', str(len(body))) + for hdr, value in headers.iteritems(): + conn.putheader(hdr, value) + self.msg(u"end of headers") + conn.endheaders() + self.msg(u" 0%") + if body: + start = 0 + size = len(body) + chunk = size / 100 + if chunk < 16384: + chunk = 16384 + while start < size: + end = min(size, start + chunk) + conn.send(body[start:end]) + start = end + self.msg(u"%2i%%" % (start * 100 / size)) + else: + self.msg(u" ") + conn.request(method, url, body, headers) + + def _run_request(self, method, url, body = None, progress = 0, content_type = "text/xml"): + url = urlparse.urljoin(self.url, url) + purl = urlparse.urlparse(url) + if purl.scheme != "https": + raise ValueError, "Unsupported url scheme: %r" % (purl.scheme,) + if ":" in purl.netloc: + host, port = purl.netloc.split(":", 1) + port = int(port) + else: + host = purl.netloc + port = None + url = purl.path + if purl.query: + url += "?" + query + headers = {} + if body: + headers["Content-Type"] = content_type + + try_no_auth = 0 + + if not try_no_auth and not self.username: + raise HTTPError, "Need a username" + + try: + self.msg(u"connecting") + conn = httplib.HTTPSConnection(host, port) +# conn.set_debuglevel(10) + + if try_no_auth: + self.request(conn, method, url, body, headers, progress) + self.msg(u"waiting for status") + response = conn.getresponse() + + if not try_no_auth or (response.status == httplib.UNAUTHORIZED and + self.username): + if try_no_auth: + conn.close() + self.msg(u"re-connecting") + conn = httplib.HTTPSConnection(host, port) +# conn.set_debuglevel(10) + + creds = self.username + ":" + self.password + headers["Authorization"] = "Basic " + \ + creds.encode("base64").strip() + self.request(conn, method, url, body, headers, progress) + self.msg(u"waiting for status") + response = conn.getresponse() + + if response.status == httplib.OK: + self.msg(u"reading response") + sys.stderr.flush() + response_body = response.read() + else: + raise HTTPError( "%02i: %s (%s)" % (response.status, + response.reason, response.read())) + finally: + conn.close() + return response_body + + def get_changeset_tags(self): + if self.changeset is None: + raise RuntimeError, "Changeset not opened" + self.progress_msg = u"Getting changeset tags" + self.msg(u"") + reply = self._run_request("GET", "/api/0.6/changeset/" + + str(self.changeset), None) + root = ElementTree.XML(reply) + if root.tag != "osm" or root[0].tag != "changeset": + print >>sys.stderr, u"API returned unexpected XML!" + sys.exit(1) + + for element in root[0]: + if element.tag == "tag" and "k" in element.attrib and \ + "v" in element.attrib: + self.tags[element.attrib["k"]] = element.attrib["v"] + + self.msg(u"done.") + print >>sys.stderr, u"" + + def set_changeset_tags(self): + self.progress_msg = u"Setting new changeset tags" + self.msg(u"") + + root = ElementTree.Element("osm") + tree = ElementTree.ElementTree(root) + element = ElementTree.SubElement(root, "changeset") + for key in self.tags: + ElementTree.SubElement(element, "tag", + { "k": key, "v": self.tags[key] }) + + self._run_request("PUT", "/api/0.6/changeset/" + + str(self.changeset), ElementTree.tostring(root, "utf-8")) + + self.msg(u"done, too.") + print >>sys.stderr, u"" + +try: + this_dir = os.path.dirname(__file__) + try: + version = int(subprocess.Popen(["svnversion", this_dir], stdout = subprocess.PIPE).communicate()[0].strip()) + except: + version = 1 + if len(sys.argv) < 3 or (len(sys.argv) & 2): + print >>sys.stderr, u"Synopsis:" + print >>sys.stderr, u" %s <changeset> <key> <value> [...]" + sys.exit(1) + + args = [] + param = {} + num = 0 + skip = 0 + for arg in sys.argv[1:]: + num += 1 + if skip: + skip -= 1 + continue + + if arg == "-u": + param['user'] = sys.argv[num + 1] + skip = 1 + elif arg == "-p": + param['pass'] = sys.argv[num + 1] + skip = 1 + else: + args.append(arg) + + if 'user' in param: + login = param['user'] + else: + login = raw_input("OSM login: ") + if not login: + sys.exit(1) + if 'pass' in param: + password = param['pass'] + else: + password = raw_input("OSM password: ") + if not password: + sys.exit(1) + + api = OSM_API(login, password) + api.changeset = int(args[0]) + + api.get_changeset_tags() + api.tags.update(zip(args[1::2], args[2::2])) + api.set_changeset_tags() +except HTTPError as err: + print >>sys.stderr, err + sys.exit(1) +except Exception,err: + print >>sys.stderr, repr(err) + traceback.print_exc(file=sys.stderr) + sys.exit(1) diff --git a/upload/smarter-sort.py b/upload/smarter-sort.py new file mode 100755 index 0000000..634c53e --- /dev/null +++ b/upload/smarter-sort.py @@ -0,0 +1,309 @@ +#! /usr/bin/python2 +# vim: fileencoding=utf-8 encoding=utf-8 et sw=4 + +# Copyright (C) 2009 Andrzej Zaborowski <balrogg@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +""" +Re-order changes in a changeset in a "logical" way (most autonomous +items first). Useful thing to do before splitting a changeset and +uploading in pieces. +""" + +__version__ = "$Revision: 21 $" + +import os +import sys +import traceback +import codecs +import locale +import subprocess + +import xml.etree.cElementTree as ElementTree + +import locale, codecs +locale.setlocale(locale.LC_ALL, "en_US.UTF-8") +encoding = locale.getlocale()[1] +sys.stdout = codecs.getwriter(encoding)(sys.stdout, errors = "replace") +sys.stderr = codecs.getwriter(encoding)(sys.stderr, errors = "replace") + +def makename(element): + return element.tag + element.attrib.get("id") + +opers = {} + +def calcdepends(element): + deps = {} + for sub in element: + if sub.tag == "nd": + name = "node" + sub.attrib.get("ref") + elif sub.tag == "member": + name = sub.attrib.get("type") + sub.attrib.get("ref") + else: + continue + + if name in opers: + # Technically we only need to append if + # opers[name][1] == "create", but the effect will look better + # if we do always. + deps[name] = 1 + + return deps + +def calcdownscore(deps, scale): + score = 0 + for dep in deps: + if opers[dep]["downscore"] + scale > score: + score = opers[dep]["downscore"] + scale + return score + +def calcdepnum(deps): + depnum = 0 + for dep in deps: + depnum += opers[dep]["depnum"] + 1 + return depnum + +globalbbox = ([360.0, 360.0], [-360.0, -360.0]) +globalscale = 0.0 +def calcbbox(element, deps): + if element.tag == "node": + lat = float(element.attrib.get("lat")) + lon = float(element.attrib.get("lon")) + if lat < globalbbox[0][0]: + globalbbox[0][0] = lat + if lon < globalbbox[0][1]: + globalbbox[0][1] = lon + if lat > globalbbox[1][0]: + globalbbox[1][0] = lat + if lon > globalbbox[1][1]: + globalbbox[1][1] = lon + return ((lat, lon), (lat, lon)) + + bbox = ([360.0, 360.0], [-360.0, -360.0]) + for dep in deps: + if opers[dep]["bbox"][0][0] < bbox[0][0]: + bbox[0][0] = opers[dep]["bbox"][0][0] + if opers[dep]["bbox"][0][1] < bbox[0][1]: + bbox[0][1] = opers[dep]["bbox"][0][1] + if opers[dep]["bbox"][1][0] > bbox[1][0]: + bbox[1][0] = opers[dep]["bbox"][1][0] + if opers[dep]["bbox"][1][1] > bbox[1][1]: + bbox[1][1] = opers[dep]["bbox"][1][1] + return ((bbox[0][0], bbox[0][1]), (bbox[1][0], bbox[1][1])) + +def update_refs(name, scale): + for dep in opers[name]["depends"]: + if opers[dep]["upscore"] < opers[name]["upscore"] + scale: + opers[dep]["upscore"] = opers[name]["upscore"] + scale + update_refs(dep, scale) +def update_only_some_refs_instead(name, scale): + for dep in opers[name]["depends"]: + opers[dep]["depended"][name] = 1 + +def recursiveusefulness(op, depth): + v = len(op["depended"]) + if depth < 3: + for dep in op["depends"]: + v += recursiveusefulness(opers[dep], depth + 1) - 1 + return v + +queue = [] +geo = None + +def queueup(names): + global geo + global queue + global globalscale + levelnames = None + while names or levelnames: + if not levelnames: + names = [ x for x in names if x in opers ] + if not names: + return + minscore = min([ opers[x]["upscore"] for x in names ]) + levelnames = {} + newnames = {} + for name in names: + if opers[name]["upscore"] == minscore: + levelnames[name] = 1 + else: + newnames[name] = 1 + names = newnames + maxscore = -1 + max = None + delete = [] + for name in levelnames: + if name in opers: + op = opers[name] + centre = ((op["bbox"][0][0] + op["bbox"][1][0]) * 0.5, + (op["bbox"][0][1] + op["bbox"][1][1]) * 0.5) + #distance = math.hypot(centre[0] - geo[0], centre[1] - geo[1]) + distance = abs(centre[0] - geo[0]) + abs(centre[1] - geo[1]) + + # This is the decision maker (possibly very wrong) + score = 10.0 - op["upscore"] + \ + 1.0 / (distance / globalscale + 0.3) - \ + op["depnum"] / (op["orig-depnum"] + 1) + #op["downscore"] * 0.1 + \ + #recursiveusefulness(op, 0) * 0.01 + \ + #(len(op["depended"]) - len(op["depends"])) * 0.00001 + + if score > maxscore: + maxscore = score + max = name + else: + delete.append(name) + for name in delete: + del levelnames[name] + if not levelnames: + continue + + if opers[max]["depends"]: + queueup(opers[max]["depends"]) + + op = opers.pop(max) + queue.append((op["element"], op["operation"])) + + for dep in op["depended"]: + del opers[dep]["depends"][max] + opers[dep]["depnum"] -= op["orig-depnum"] + 1 + + centre = ((op["bbox"][0][0] + op["bbox"][1][0]) * 0.5, + (op["bbox"][0][1] + op["bbox"][1][1]) * 0.5) + geo = ((geo[0] * 2 + centre[0]) / 3, + (geo[1] * 2 + centre[1]) / 3) + +try: + this_dir = os.path.dirname(__file__) + version = subprocess.Popen(["svnversion", this_dir], stdout = subprocess.PIPE).communicate()[0].strip() + if len(sys.argv) not in (2,): + print >>sys.stderr, u"Synopsis:" + print >>sys.stderr, u" %s <file_name>" + sys.exit(1) + + filename = sys.argv[1] + if len(sys.argv) > 2: + num_parts = int(sys.argv[2]) + else: + num_parts = 2 + if not os.path.exists(filename): + print >>sys.stderr, u"File %r doesn't exist!" % (filename,) + sys.exit(1) + if filename.endswith(".osc"): + filename_base = filename[:-4] + else: + filename_base = filename + + print >>sys.stderr, u"Parsing osmChange..." + tree = ElementTree.parse(filename) + root = tree.getroot() + if root.tag != "osmChange" or (root.attrib.get("version") != "0.3" and + root.attrib.get("version") != "0.6"): + print >>sys.stderr, u"File %s is not a v0.3 osmChange file!" % (filename,) + sys.exit(1) + + print >>sys.stderr, u"Building dependency trees..." + # Note: assumes each element appearing only once - easy to work around + # (we should really detect those and compress all operations on any given + # item into 0 (creation + deletion) or 1 operation (any other scenario).) + # (perhaps this should be done as a separate step before running this + # program.) + deldeps = {"node": {}, "way": {}, "relation": {}} + ops = [] + for operation in root: + ops.append(operation) + for element in operation: + name = makename(element) + + if operation.tag == "delete": + depends = deldeps[element.tag].copy() + scale = 0.01 + else: + depends = calcdepends(element) + scale = 1 + + depnum = calcdepnum(depends) + opers[name] = { + "element": element, + "operation": operation.tag, + "scale": scale, + "depends": depends, + "depended": {}, + #"downscore": calcdownscore(depends, scale), # unused now + "depnum": depnum, + "orig-depnum": depnum, + "upscore": 0, + "bbox": calcbbox(element, depends), + } + + # Update references + #update_refs(name, scale) # We now update them only once, at the end + update_only_some_refs_instead(name, scale) + + # Assume that we don't delete objects we've just created, then + # a delete operation depends on all the modify and delete + # operations that appear before it. We could calculate the + # dependencies of a delete operation with more accuracy with + # access to the current state but not with only the contents + # of the current diff. + if operation.tag in [ "modify", "delete" ]: + if element.tag == "way": + deldeps["node"][name] = 1 + if element.tag == "relation": + for el in deldeps: + deldeps[el][name] = 1 + + for name in opers: + if not opers[name]["depended"]: + update_refs(name, opers[name]["scale"]) + + print >>sys.stderr, u"Sorting references..." + for operation in ops: + root.remove(operation) + if opers: # Take a random starting point + geo = opers[opers.keys()[0]]["bbox"][0] + geo = (-1000, -1000) + geo = globalbbox[0] + globalscale = (globalbbox[1][0] - globalbbox[0][0] + + globalbbox[1][1] - globalbbox[0][1]) + queueup(opers) + + print >>sys.stderr, u"Writing osmChange..." + opattrs = { "generator": "smarter-sort.py", "version": "0.3" } + popname = "desert storm" + + for element, opname in queue: + if opname != popname: + op = ElementTree.SubElement(root, opname, opattrs) + popname = opname + + op.append(element) + + tree.write(filename_base + "-sorted.osc", "utf-8") + + comment_fn = filename_base + ".comment" + if os.path.exists(comment_fn): + comment_file = codecs.open(comment_fn, "r", "utf-8") + comment = comment_file.read().strip() + comment_file.close() + comment_fn = filename_base + "-sorted.comment" + comment_file = codecs.open(comment_fn, "w", "utf-8") + print >> comment_file, comment + comment_file.close() +except Exception,err: + print >>sys.stderr, repr(err) + traceback.print_exc(file=sys.stderr) + sys.exit(1) diff --git a/upload/split.py b/upload/split.py new file mode 100755 index 0000000..86d65e3 --- /dev/null +++ b/upload/split.py @@ -0,0 +1,117 @@ +#! /usr/bin/python3 +# vim: fileencoding=utf-8 encoding=utf-8 et sw=4 + +# Copyright (C) 2009 Jacek Konieczny <jajcus@jajcus.net> +# Copyright (C) 2009 Andrzej Zaborowski <balrogg@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +""" +Split large osmChange files. +""" + +__version__ = "$Revision: 21 $" + +import os +import sys +import traceback +import codecs +import locale +import subprocess + +import http + +import xml.etree.cElementTree as ElementTree + +#locale.setlocale(locale.LC_ALL, "en_US.UTF-8") +#encoding = locale.getlocale()[1] +#sys.stdout = codecs.getwriter(encoding)(sys.stdout, errors = "replace") +#sys.stderr = codecs.getwriter(encoding)(sys.stderr, errors = "replace") + +try: + this_dir = os.path.dirname(__file__) + version = subprocess.Popen(["svnversion", this_dir], stdout = subprocess.PIPE).communicate()[0].strip() + if len(sys.argv) not in (2, 3): + sys.stderr.write("Synopsis:\n") + sys.stderr.write(" %s <file_name> [<num_of_pieces>\n]" % (sys.argv[0],)) + sys.exit(1) + + filename = sys.argv[1] + if len(sys.argv) > 2: + num_parts = int(sys.argv[2]) + else: + num_parts = 2 + if not os.path.exists(filename): + sys.stderr.write("File %r doesn't exist!\n" % (filename,)) + sys.exit(1) + if filename.endswith(".osc"): + filename_base = filename[:-4] + else: + filename_base = filename + + tree = ElementTree.parse(filename) + root = tree.getroot() + if root.tag != "osmChange" or (root.attrib.get("version") != "0.3" and + root.attrib.get("version") != "0.6"): + sys.stderr.write("File %s is not a v0.3 osmChange file!\n" % (filename,)) + sys.exit(1) + + element_count = 0 + for operation in root: + element_count += len(operation) + + sys.stderr.write("Number of parts: %r\n" % (element_count,)) + part_size = int((element_count + num_parts - 1) / num_parts) + + part = 1 + operation_iter = iter(root) + operation = next(operation_iter) + elements = list(operation) + while elements and operation: + filename = "%s-part%i.osc" % (filename_base, part) + part_root = ElementTree.Element(root.tag, root.attrib) + part_tree = ElementTree.ElementTree(part_root) + current_size = 0 + while operation and current_size < part_size: + part_op = ElementTree.SubElement(part_root, operation.tag, operation.attrib) + this_part_elements = elements[:(part_size-current_size)] + elements = elements[(part_size-current_size):] + for element in this_part_elements: + part_op.append(element) + current_size += 1 + if not elements: + try: + while not elements: + operation = next(operation_iter) + elements = list(operation) + except StopIteration: + operation = None + elements = [] + part_tree.write(filename, "utf-8") + part += 1 + comment_fn = filename_base + ".comment" + if os.path.exists(comment_fn): + comment_file = codecs.open(comment_fn, "r", "utf-8") + comment = comment_file.read().strip() + comment_file.close() + for part in range(1, num_parts + 1): + comment_fn = "%s-part%i.comment" % (filename_base, part) + comment_file = codecs.open(comment_fn, "w", "utf-8") + comment_file.write("%s, part %i/%i" % (comment, part, num_parts)) + comment_file.close() +except Exception as err: + sys.stderr.write(repr(err)) + traceback.print_exc(file=sys.stderr) + sys.exit(1) diff --git a/upload/upload.py b/upload/upload.py new file mode 100755 index 0000000..934bc7b --- /dev/null +++ b/upload/upload.py @@ -0,0 +1,432 @@ +#! /usr/bin/python3 +# vim: fileencoding=utf-8 encoding=utf-8 et sw=4 + +# Copyright (C) 2009 Jacek Konieczny <jajcus@jajcus.net> +# Copyright (C) 2009 Andrzej Zaborowski <balrogg@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +""" +Uploads complete osmChange 0.3 files. Use your login (not email) as username. +""" + +__version__ = "$Revision: 21 $" + +import os +import subprocess +import sys +import traceback +import base64 +import codecs + +import http.client as httplib +import xml.etree.cElementTree as ElementTree +import urllib.parse as urlparse + +class HTTPError(Exception): + pass + +class OSM_API(object): + url = 'https://master.apis.dev.openstreetmap.org/' + def __init__(self, username = None, password = None): + if username and password: + self.username = username + self.password = password + else: + self.username = "" + self.password = "" + self.changeset = None + self.progress_msg = None + + def __del__(self): + #if self.changeset is not None: + # self.close_changeset() + pass + + def msg(self, mesg): + sys.stderr.write("\r%s… " % (self.progress_msg)) + sys.stderr.write("\r%s… %s" % (self.progress_msg, mesg)) + sys.stderr.flush() + + def request(self, conn, method, url, body, headers, progress): + if progress: + self.msg("making request") + conn.putrequest(method, url) + self.msg("sending headers") + if body: + conn.putheader('Content-Length', str(len(body))) + for hdr, value in headers.items(): + conn.putheader(hdr, value) + self.msg("end of headers") + conn.endheaders() + self.msg(" 0%") + if body: + start = 0 + size = len(body) + chunk = size / 100 + if chunk < 16384: + chunk = 16384 + while start < size: + end = min(size, int(start + chunk)) + conn.send(body[start:end]) + start = end + self.msg("%2i%%" % (int(start * 100 / size),)) + else: + self.msg(" ") + conn.request(method, url, body, headers) + + def _run_request(self, method, url, body = None, progress = 0, content_type = "text/xml"): + url = urlparse.urljoin(self.url, url) + purl = urlparse.urlparse(url) + if purl.scheme != "https": + raise ValueError("Unsupported url scheme: %r" % (purl.scheme,)) + if ":" in purl.netloc: + host, port = purl.netloc.split(":", 1) + port = int(port) + else: + host = purl.netloc + port = None + url = purl.path + if purl.query: + url += "?" + query + headers = {} + if body: + headers["Content-Type"] = content_type + + try_no_auth = 0 + + if not try_no_auth and not self.username: + raise HTTPError(0, "Need a username") + + try: + self.msg("connecting") + conn = httplib.HTTPSConnection(host, port) +# conn.set_debuglevel(10) + + if try_no_auth: + self.request(conn, method, url, body, headers, progress) + self.msg("waiting for status") + response = conn.getresponse() + + if not try_no_auth or (response.status == httplib.UNAUTHORIZED and + self.username): + if try_no_auth: + conn.close() + self.msg("re-connecting") + conn = httplib.HTTPSConnection(host, port) +# conn.set_debuglevel(10) + + creds = self.username + ":" + self.password + headers["Authorization"] = "Basic " + \ + base64.b64encode(bytes(creds, "utf8")).decode("utf8") + # ^ Seems to be broken in python3 (even the raw + # documentation examples don't run for base64) + self.request(conn, method, url, body, headers, progress) + self.msg("waiting for status") + response = conn.getresponse() + + if response.status == httplib.OK: + self.msg("reading response") + sys.stderr.flush() + response_body = response.read() + else: + err = response.read() + raise HTTPError(response.status, "%03i: %s (%s)" % ( + response.status, response.reason, err), err) + finally: + conn.close() + return response_body + + def create_changeset(self, created_by, comment, source): + if self.changeset is not None: + raise RuntimeError("Changeset already opened") + self.progress_msg = "I'm creating the changeset" + self.msg("") + root = ElementTree.Element("osm") + tree = ElementTree.ElementTree(root) + element = ElementTree.SubElement(root, "changeset") + ElementTree.SubElement(element, "tag", {"k": "comment", "v": comment}) + ElementTree.SubElement(element, "tag", {"k": "source", "v": source}) + ElementTree.SubElement(element, "tag", {"k": "source:ref", "v": "https://www.land.vic.gov.au/maps-and-spatial/spatial-data/vicmap-catalogue/vicmap-address"}) + ElementTree.SubElement(element, "tag", {"k": "created_by", "v": created_by}) + body = ElementTree.tostring(root, "utf-8") + reply = self._run_request("PUT", "/api/0.6/changeset/create", body) + changeset = int(reply.strip()) + self.msg("done.\nChangeset ID: %i" % (changeset)) + sys.stderr.write("\n") + self.changeset = changeset + + def upload(self, change): + if self.changeset is None: + raise RuntimeError("Changeset not opened") + self.progress_msg = "Now I'm sending changes" + self.msg("") + for operation in change: + if operation.tag not in ("create", "modify", "delete"): + continue + for element in operation: + element.attrib["changeset"] = str(self.changeset) + body = ElementTree.tostring(change, "utf-8") + reply = self._run_request("POST", "/api/0.6/changeset/%i/upload" + % (self.changeset,), body, 1) + self.msg("done.") + sys.stderr.write("\n") + return reply + + def close_changeset(self): + if self.changeset is None: + raise RuntimeError("Changeset not opened") + self.progress_msg = "Closing" + self.msg("") + reply = self._run_request("PUT", "/api/0.6/changeset/%i/close" + % (self.changeset,)) + self.changeset = None + self.msg("done, too.") + sys.stderr.write("\n") + +try: + this_dir = os.path.dirname(__file__) + try: + version = int(subprocess.Popen(["svnversion", this_dir], stdout = subprocess.PIPE).communicate()[0].strip()) + except: + version = 1 + if len(sys.argv) < 2: + sys.stderr.write("Synopsis:\n") + sys.stderr.write(" %s <file-name.osc> [<file-name.osc>...]\n" % (sys.argv[0],)) + sys.exit(1) + + filenames = [] + param = {} + num = 0 + skip = 0 + for arg in sys.argv[1:]: + num += 1 + if skip: + skip -= 1 + continue + + if arg == "-u": + param['user'] = sys.argv[num + 1] + skip = 1 + elif arg == "-p": + param['pass'] = sys.argv[num + 1] + skip = 1 + elif arg == "-c": + param['confirm'] = sys.argv[num + 1] + skip = 1 + elif arg == "-m": + param['comment'] = sys.argv[num + 1] + skip = 1 + elif arg == "-s": + param['changeset'] = sys.argv[num + 1] + skip = 1 + elif arg == "-n": + param['start'] = 1 + skip = 0 + elif arg == "-t": + param['try'] = 1 + skip = 0 + elif arg == "-x": + param['created_by'] = sys.argv[num + 1] + skip = 1 + elif arg == "-y": + param['source'] = sys.argv[num + 1] + skip = 1 + elif arg == "-z": + param['url'] = sys.argv[num + 1] + skip = 1 + else: + filenames.append(arg) + + if 'user' in param: + login = param['user'] + else: + login = input("OSM login: ") + if not login: + sys.exit(1) + if 'pass' in param: + password = param['pass'] + else: + password = input("OSM password: ") + if not password: + sys.exit(1) + + api = OSM_API(login, password) + + changes = [] + for filename in filenames: + if not os.path.exists(filename): + sys.stderr.write("File %r doesn't exist!\n" % (filename,)) + sys.exit(1) + if 'start' not in param: + # Should still check validity, but let's save time + + tree = ElementTree.parse(filename) + root = tree.getroot() + if root.tag != "osmChange" or (root.attrib.get("version") != "0.3" and + root.attrib.get("version") != "0.6"): + sys.stderr.write("File %s is not a v0.3 osmChange file!\n" % (filename,)) + sys.exit(1) + + if filename.endswith(".osc"): + diff_fn = filename[:-4] + ".diff.xml" + else: + diff_fn = filename + ".diff.xml" + if os.path.exists(diff_fn): + sys.stderr.write("Diff file %r already exists, delete it " \ + "if you're sure you want to re-upload\n" % (diff_fn,)) + sys.exit(1) + + if filename.endswith(".osc"): + comment_fn = filename[:-4] + ".comment" + else: + comment_fn = filename + ".comment" + try: + comment_file = codecs.open(comment_fn, "r", "utf-8") + comment = comment_file.read().strip() + comment_file.close() + except IOError: + comment = None + if not comment: + if 'comment' in param: + comment = param['comment'] + else: + comment = input("Your comment to %r: " % (filename,)) + if not comment: + sys.exit(1) + #try: + # comment = comment.decode(locale.getlocale()[1]) + #except TypeError: + # comment = comment.decode("UTF-8") + + sys.stderr.write(" File: %r\n" % (filename,)) + sys.stderr.write(" Comment: %s\n" % (comment,)) + + if 'confirm' in param: + sure = param['confirm'] + else: + sys.stderr.write("Are you sure you want to send these changes?") + sure = input() + if sure.lower() not in ("y", "yes"): + sys.stderr.write("Skipping...\n\n") + continue + sys.stderr.write("\n") + created_by = param.get("created_by", "osm-bulk-upload/upload.py v. %s" % (version,)) + source = param.get("source", "") + url = param.get("url", "") + if 'changeset' in param: + api.changeset = int(param['changeset']) + else: + api.create_changeset(created_by, comment, source, url) + if 'start' in param: + print(api.changeset) + sys.exit(0) + while 1: + try: + diff_file = codecs.open(diff_fn, "w", "utf-8") + diff = api.upload(root) + diff_file.write(diff.decode("utf8")) + diff_file.close() + except HTTPError as e: + sys.stderr.write("\n" + e.args[1] + "\n") + if e.args[0] in [ 404, 409, 412 ]: # Merge conflict + # TODO: also unlink when not the whole file has been uploaded + # because then likely the server will not be able to parse + # it and nothing gets committed + os.unlink(diff_fn) + errstr = e.args[2].decode("utf8") + if 'try' in param and e.args[0] == 409 and \ + errstr.find("Version mismatch") > -1: + id = errstr.split(" ")[-1] + found = 0 + for oper in root: + todel = [] + for elem in oper: + if elem.attrib.get("id") != id: + continue + todel.append(elem) + found = 1 + for elem in todel: + oper.remove(elem) + if not found: + sys.stderr.write("\nElement " + id + " not found\n") + if 'changeset' not in param: + api.close_changeset() + sys.exit(1) + sys.stderr.write("\nRetrying upload without element " + + id + "\n") + continue + if 'try' in param and e.args[0] == 400 and \ + errstr.find("Placeholder Way not found") > -1: + id = errstr.replace(".", "").split(" ")[-1] + found = 0 + for oper in root: + todel = [] + for elem in oper: + if elem.attrib.get("id") != id: + continue + todel.append(elem) + found = 1 + for elem in todel: + oper.remove(elem) + if not found: + sys.stderr.write("\nElement " + id + " not found\n") + if 'changeset' not in param: + api.close_changeset() + sys.exit(1) + sys.stderr.write("\nRetrying upload without element " + + id + "\n") + continue + if 'try' in param and e.args[0] == 412 and \ + errstr.find(" requires ") > -1: + idlist = errstr.split("id in (")[1].split(")")[0].split(",") + found = 0 + delids = [] + for oper in root: + todel = [] + for elem in oper: + for nd in elem: + if nd.tag not in [ "nd", "member" ]: + continue + if nd.attrib.get("ref") not in idlist: + continue + found = 1 + delids.append(elem.attrib.get("id")) + todel.append(elem) + break + for elem in todel: + oper.remove(elem) + if not found: + sys.stderr.write("\nElement " + str(idlist) + + " not found\n") + if 'changeset' not in param: + api.close_changeset() + sys.exit(1) + sys.stderr.write("\nRetrying upload without elements " + + str(delids) + "\n") + continue + if 'changeset' not in param: + api.close_changeset() + sys.exit(1) + break + if 'changeset' not in param: + api.close_changeset() +except HTTPError as err: + sys.stderr.write(err.args[1]) + sys.exit(1) +except Exception as err: + sys.stderr.write(repr(err) + "\n") + traceback.print_exc(file=sys.stderr) + sys.exit(1) |