diff options
Diffstat (limited to 'upload/upload.py')
| -rwxr-xr-x | upload/upload.py | 432 |
1 files changed, 432 insertions, 0 deletions
diff --git a/upload/upload.py b/upload/upload.py new file mode 100755 index 0000000..934bc7b --- /dev/null +++ b/upload/upload.py @@ -0,0 +1,432 @@ +#! /usr/bin/python3 +# vim: fileencoding=utf-8 encoding=utf-8 et sw=4 + +# Copyright (C) 2009 Jacek Konieczny <jajcus@jajcus.net> +# Copyright (C) 2009 Andrzej Zaborowski <balrogg@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +""" +Uploads complete osmChange 0.3 files. Use your login (not email) as username. +""" + +__version__ = "$Revision: 21 $" + +import os +import subprocess +import sys +import traceback +import base64 +import codecs + +import http.client as httplib +import xml.etree.cElementTree as ElementTree +import urllib.parse as urlparse + +class HTTPError(Exception): + pass + +class OSM_API(object): + url = 'https://master.apis.dev.openstreetmap.org/' + def __init__(self, username = None, password = None): + if username and password: + self.username = username + self.password = password + else: + self.username = "" + self.password = "" + self.changeset = None + self.progress_msg = None + + def __del__(self): + #if self.changeset is not None: + # self.close_changeset() + pass + + def msg(self, mesg): + sys.stderr.write("\r%s… " % (self.progress_msg)) + sys.stderr.write("\r%s… %s" % (self.progress_msg, mesg)) + sys.stderr.flush() + + def request(self, conn, method, url, body, headers, progress): + if progress: + self.msg("making request") + conn.putrequest(method, url) + self.msg("sending headers") + if body: + conn.putheader('Content-Length', str(len(body))) + for hdr, value in headers.items(): + conn.putheader(hdr, value) + self.msg("end of headers") + conn.endheaders() + self.msg(" 0%") + if body: + start = 0 + size = len(body) + chunk = size / 100 + if chunk < 16384: + chunk = 16384 + while start < size: + end = min(size, int(start + chunk)) + conn.send(body[start:end]) + start = end + self.msg("%2i%%" % (int(start * 100 / size),)) + else: + self.msg(" ") + conn.request(method, url, body, headers) + + def _run_request(self, method, url, body = None, progress = 0, content_type = "text/xml"): + url = urlparse.urljoin(self.url, url) + purl = urlparse.urlparse(url) + if purl.scheme != "https": + raise ValueError("Unsupported url scheme: %r" % (purl.scheme,)) + if ":" in purl.netloc: + host, port = purl.netloc.split(":", 1) + port = int(port) + else: + host = purl.netloc + port = None + url = purl.path + if purl.query: + url += "?" + query + headers = {} + if body: + headers["Content-Type"] = content_type + + try_no_auth = 0 + + if not try_no_auth and not self.username: + raise HTTPError(0, "Need a username") + + try: + self.msg("connecting") + conn = httplib.HTTPSConnection(host, port) +# conn.set_debuglevel(10) + + if try_no_auth: + self.request(conn, method, url, body, headers, progress) + self.msg("waiting for status") + response = conn.getresponse() + + if not try_no_auth or (response.status == httplib.UNAUTHORIZED and + self.username): + if try_no_auth: + conn.close() + self.msg("re-connecting") + conn = httplib.HTTPSConnection(host, port) +# conn.set_debuglevel(10) + + creds = self.username + ":" + self.password + headers["Authorization"] = "Basic " + \ + base64.b64encode(bytes(creds, "utf8")).decode("utf8") + # ^ Seems to be broken in python3 (even the raw + # documentation examples don't run for base64) + self.request(conn, method, url, body, headers, progress) + self.msg("waiting for status") + response = conn.getresponse() + + if response.status == httplib.OK: + self.msg("reading response") + sys.stderr.flush() + response_body = response.read() + else: + err = response.read() + raise HTTPError(response.status, "%03i: %s (%s)" % ( + response.status, response.reason, err), err) + finally: + conn.close() + return response_body + + def create_changeset(self, created_by, comment, source): + if self.changeset is not None: + raise RuntimeError("Changeset already opened") + self.progress_msg = "I'm creating the changeset" + self.msg("") + root = ElementTree.Element("osm") + tree = ElementTree.ElementTree(root) + element = ElementTree.SubElement(root, "changeset") + ElementTree.SubElement(element, "tag", {"k": "comment", "v": comment}) + ElementTree.SubElement(element, "tag", {"k": "source", "v": source}) + ElementTree.SubElement(element, "tag", {"k": "source:ref", "v": "https://www.land.vic.gov.au/maps-and-spatial/spatial-data/vicmap-catalogue/vicmap-address"}) + ElementTree.SubElement(element, "tag", {"k": "created_by", "v": created_by}) + body = ElementTree.tostring(root, "utf-8") + reply = self._run_request("PUT", "/api/0.6/changeset/create", body) + changeset = int(reply.strip()) + self.msg("done.\nChangeset ID: %i" % (changeset)) + sys.stderr.write("\n") + self.changeset = changeset + + def upload(self, change): + if self.changeset is None: + raise RuntimeError("Changeset not opened") + self.progress_msg = "Now I'm sending changes" + self.msg("") + for operation in change: + if operation.tag not in ("create", "modify", "delete"): + continue + for element in operation: + element.attrib["changeset"] = str(self.changeset) + body = ElementTree.tostring(change, "utf-8") + reply = self._run_request("POST", "/api/0.6/changeset/%i/upload" + % (self.changeset,), body, 1) + self.msg("done.") + sys.stderr.write("\n") + return reply + + def close_changeset(self): + if self.changeset is None: + raise RuntimeError("Changeset not opened") + self.progress_msg = "Closing" + self.msg("") + reply = self._run_request("PUT", "/api/0.6/changeset/%i/close" + % (self.changeset,)) + self.changeset = None + self.msg("done, too.") + sys.stderr.write("\n") + +try: + this_dir = os.path.dirname(__file__) + try: + version = int(subprocess.Popen(["svnversion", this_dir], stdout = subprocess.PIPE).communicate()[0].strip()) + except: + version = 1 + if len(sys.argv) < 2: + sys.stderr.write("Synopsis:\n") + sys.stderr.write(" %s <file-name.osc> [<file-name.osc>...]\n" % (sys.argv[0],)) + sys.exit(1) + + filenames = [] + param = {} + num = 0 + skip = 0 + for arg in sys.argv[1:]: + num += 1 + if skip: + skip -= 1 + continue + + if arg == "-u": + param['user'] = sys.argv[num + 1] + skip = 1 + elif arg == "-p": + param['pass'] = sys.argv[num + 1] + skip = 1 + elif arg == "-c": + param['confirm'] = sys.argv[num + 1] + skip = 1 + elif arg == "-m": + param['comment'] = sys.argv[num + 1] + skip = 1 + elif arg == "-s": + param['changeset'] = sys.argv[num + 1] + skip = 1 + elif arg == "-n": + param['start'] = 1 + skip = 0 + elif arg == "-t": + param['try'] = 1 + skip = 0 + elif arg == "-x": + param['created_by'] = sys.argv[num + 1] + skip = 1 + elif arg == "-y": + param['source'] = sys.argv[num + 1] + skip = 1 + elif arg == "-z": + param['url'] = sys.argv[num + 1] + skip = 1 + else: + filenames.append(arg) + + if 'user' in param: + login = param['user'] + else: + login = input("OSM login: ") + if not login: + sys.exit(1) + if 'pass' in param: + password = param['pass'] + else: + password = input("OSM password: ") + if not password: + sys.exit(1) + + api = OSM_API(login, password) + + changes = [] + for filename in filenames: + if not os.path.exists(filename): + sys.stderr.write("File %r doesn't exist!\n" % (filename,)) + sys.exit(1) + if 'start' not in param: + # Should still check validity, but let's save time + + tree = ElementTree.parse(filename) + root = tree.getroot() + if root.tag != "osmChange" or (root.attrib.get("version") != "0.3" and + root.attrib.get("version") != "0.6"): + sys.stderr.write("File %s is not a v0.3 osmChange file!\n" % (filename,)) + sys.exit(1) + + if filename.endswith(".osc"): + diff_fn = filename[:-4] + ".diff.xml" + else: + diff_fn = filename + ".diff.xml" + if os.path.exists(diff_fn): + sys.stderr.write("Diff file %r already exists, delete it " \ + "if you're sure you want to re-upload\n" % (diff_fn,)) + sys.exit(1) + + if filename.endswith(".osc"): + comment_fn = filename[:-4] + ".comment" + else: + comment_fn = filename + ".comment" + try: + comment_file = codecs.open(comment_fn, "r", "utf-8") + comment = comment_file.read().strip() + comment_file.close() + except IOError: + comment = None + if not comment: + if 'comment' in param: + comment = param['comment'] + else: + comment = input("Your comment to %r: " % (filename,)) + if not comment: + sys.exit(1) + #try: + # comment = comment.decode(locale.getlocale()[1]) + #except TypeError: + # comment = comment.decode("UTF-8") + + sys.stderr.write(" File: %r\n" % (filename,)) + sys.stderr.write(" Comment: %s\n" % (comment,)) + + if 'confirm' in param: + sure = param['confirm'] + else: + sys.stderr.write("Are you sure you want to send these changes?") + sure = input() + if sure.lower() not in ("y", "yes"): + sys.stderr.write("Skipping...\n\n") + continue + sys.stderr.write("\n") + created_by = param.get("created_by", "osm-bulk-upload/upload.py v. %s" % (version,)) + source = param.get("source", "") + url = param.get("url", "") + if 'changeset' in param: + api.changeset = int(param['changeset']) + else: + api.create_changeset(created_by, comment, source, url) + if 'start' in param: + print(api.changeset) + sys.exit(0) + while 1: + try: + diff_file = codecs.open(diff_fn, "w", "utf-8") + diff = api.upload(root) + diff_file.write(diff.decode("utf8")) + diff_file.close() + except HTTPError as e: + sys.stderr.write("\n" + e.args[1] + "\n") + if e.args[0] in [ 404, 409, 412 ]: # Merge conflict + # TODO: also unlink when not the whole file has been uploaded + # because then likely the server will not be able to parse + # it and nothing gets committed + os.unlink(diff_fn) + errstr = e.args[2].decode("utf8") + if 'try' in param and e.args[0] == 409 and \ + errstr.find("Version mismatch") > -1: + id = errstr.split(" ")[-1] + found = 0 + for oper in root: + todel = [] + for elem in oper: + if elem.attrib.get("id") != id: + continue + todel.append(elem) + found = 1 + for elem in todel: + oper.remove(elem) + if not found: + sys.stderr.write("\nElement " + id + " not found\n") + if 'changeset' not in param: + api.close_changeset() + sys.exit(1) + sys.stderr.write("\nRetrying upload without element " + + id + "\n") + continue + if 'try' in param and e.args[0] == 400 and \ + errstr.find("Placeholder Way not found") > -1: + id = errstr.replace(".", "").split(" ")[-1] + found = 0 + for oper in root: + todel = [] + for elem in oper: + if elem.attrib.get("id") != id: + continue + todel.append(elem) + found = 1 + for elem in todel: + oper.remove(elem) + if not found: + sys.stderr.write("\nElement " + id + " not found\n") + if 'changeset' not in param: + api.close_changeset() + sys.exit(1) + sys.stderr.write("\nRetrying upload without element " + + id + "\n") + continue + if 'try' in param and e.args[0] == 412 and \ + errstr.find(" requires ") > -1: + idlist = errstr.split("id in (")[1].split(")")[0].split(",") + found = 0 + delids = [] + for oper in root: + todel = [] + for elem in oper: + for nd in elem: + if nd.tag not in [ "nd", "member" ]: + continue + if nd.attrib.get("ref") not in idlist: + continue + found = 1 + delids.append(elem.attrib.get("id")) + todel.append(elem) + break + for elem in todel: + oper.remove(elem) + if not found: + sys.stderr.write("\nElement " + str(idlist) + + " not found\n") + if 'changeset' not in param: + api.close_changeset() + sys.exit(1) + sys.stderr.write("\nRetrying upload without elements " + + str(delids) + "\n") + continue + if 'changeset' not in param: + api.close_changeset() + sys.exit(1) + break + if 'changeset' not in param: + api.close_changeset() +except HTTPError as err: + sys.stderr.write(err.args[1]) + sys.exit(1) +except Exception as err: + sys.stderr.write(repr(err) + "\n") + traceback.print_exc(file=sys.stderr) + sys.exit(1) |