001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.InputStream;
007import java.io.InputStreamReader;
008import java.text.MessageFormat;
009import java.util.ArrayList;
010import java.util.Collection;
011import java.util.List;
012import java.util.regex.Matcher;
013import java.util.regex.Pattern;
014
015import javax.xml.stream.Location;
016import javax.xml.stream.XMLInputFactory;
017import javax.xml.stream.XMLStreamConstants;
018import javax.xml.stream.XMLStreamException;
019import javax.xml.stream.XMLStreamReader;
020
021import org.openstreetmap.josm.Main;
022import org.openstreetmap.josm.data.Bounds;
023import org.openstreetmap.josm.data.DataSource;
024import org.openstreetmap.josm.data.coor.LatLon;
025import org.openstreetmap.josm.data.osm.Changeset;
026import org.openstreetmap.josm.data.osm.DataSet;
027import org.openstreetmap.josm.data.osm.Node;
028import org.openstreetmap.josm.data.osm.NodeData;
029import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
030import org.openstreetmap.josm.data.osm.PrimitiveData;
031import org.openstreetmap.josm.data.osm.Relation;
032import org.openstreetmap.josm.data.osm.RelationData;
033import org.openstreetmap.josm.data.osm.RelationMemberData;
034import org.openstreetmap.josm.data.osm.Tagged;
035import org.openstreetmap.josm.data.osm.User;
036import org.openstreetmap.josm.data.osm.Way;
037import org.openstreetmap.josm.data.osm.WayData;
038import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
039import org.openstreetmap.josm.gui.progress.ProgressMonitor;
040import org.openstreetmap.josm.tools.CheckParameterUtil;
041import org.openstreetmap.josm.tools.date.DateUtils;
042
043/**
044 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
045 *
046 * For each xml element, there is a dedicated method.
047 * The XMLStreamReader cursor points to the start of the element, when the method is
048 * entered, and it must point to the end of the same element, when it is exited.
049 */
050public class OsmReader extends AbstractReader {
051
052    protected XMLStreamReader parser;
053
054    protected boolean cancel;
055
056    /** Used by plugins to register themselves as data postprocessors. */
057    private static volatile List<OsmServerReadPostprocessor> postprocessors;
058
059    /** Register a new postprocessor.
060     * @param pp postprocessor
061     * @see #deregisterPostprocessor
062     */
063    public static void registerPostprocessor(OsmServerReadPostprocessor pp) {
064        if (postprocessors == null) {
065            postprocessors = new ArrayList<>();
066        }
067        postprocessors.add(pp);
068    }
069
070    /**
071     * Deregister a postprocessor previously registered with {@link #registerPostprocessor}.
072     * @param pp postprocessor
073     * @see #registerPostprocessor
074     */
075    public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) {
076        if (postprocessors != null) {
077            postprocessors.remove(pp);
078        }
079    }
080
081    /**
082     * constructor (for private and subclasses use only)
083     *
084     * @see #parseDataSet(InputStream, ProgressMonitor)
085     */
086    protected OsmReader() {
087        // Restricts visibility
088    }
089
090    protected void setParser(XMLStreamReader parser) {
091        this.parser = parser;
092    }
093
094    protected void throwException(String msg, Throwable th) throws XMLStreamException {
095        throw new OsmParsingException(msg, parser.getLocation(), th);
096    }
097
098    protected void throwException(String msg) throws XMLStreamException {
099        throw new OsmParsingException(msg, parser.getLocation());
100    }
101
102    protected void parse() throws XMLStreamException {
103        int event = parser.getEventType();
104        while (true) {
105            if (event == XMLStreamConstants.START_ELEMENT) {
106                parseRoot();
107            } else if (event == XMLStreamConstants.END_ELEMENT)
108                return;
109            if (parser.hasNext()) {
110                event = parser.next();
111            } else {
112                break;
113            }
114        }
115        parser.close();
116    }
117
118    protected void parseRoot() throws XMLStreamException {
119        if ("osm".equals(parser.getLocalName())) {
120            parseOsm();
121        } else {
122            parseUnknown();
123        }
124    }
125
126    private void parseOsm() throws XMLStreamException {
127        String v = parser.getAttributeValue(null, "version");
128        if (v == null) {
129            throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
130        }
131        if (!"0.6".equals(v)) {
132            throwException(tr("Unsupported version: {0}", v));
133        }
134        ds.setVersion(v);
135        String upload = parser.getAttributeValue(null, "upload");
136        if (upload != null) {
137            ds.setUploadDiscouraged(!Boolean.parseBoolean(upload));
138        }
139        String generator = parser.getAttributeValue(null, "generator");
140        Long uploadChangesetId = null;
141        if (parser.getAttributeValue(null, "upload-changeset") != null) {
142            uploadChangesetId = getLong("upload-changeset");
143        }
144        while (true) {
145            int event = parser.next();
146
147            if (cancel) {
148                cancel = false;
149                throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation());
150            }
151
152            if (event == XMLStreamConstants.START_ELEMENT) {
153                switch (parser.getLocalName()) {
154                case "bounds":
155                    parseBounds(generator);
156                    break;
157                case "node":
158                    parseNode();
159                    break;
160                case "way":
161                    parseWay();
162                    break;
163                case "relation":
164                    parseRelation();
165                    break;
166                case "changeset":
167                    parseChangeset(uploadChangesetId);
168                    break;
169                default:
170                    parseUnknown();
171                }
172            } else if (event == XMLStreamConstants.END_ELEMENT)
173                return;
174        }
175    }
176
177    private void parseBounds(String generator) throws XMLStreamException {
178        String minlon = parser.getAttributeValue(null, "minlon");
179        String minlat = parser.getAttributeValue(null, "minlat");
180        String maxlon = parser.getAttributeValue(null, "maxlon");
181        String maxlat = parser.getAttributeValue(null, "maxlat");
182        String origin = parser.getAttributeValue(null, "origin");
183        if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
184            if (origin == null) {
185                origin = generator;
186            }
187            Bounds bounds = new Bounds(
188                    Double.parseDouble(minlat), Double.parseDouble(minlon),
189                    Double.parseDouble(maxlat), Double.parseDouble(maxlon));
190            if (bounds.isOutOfTheWorld()) {
191                Bounds copy = new Bounds(bounds);
192                bounds.normalize();
193                Main.info("Bbox " + copy + " is out of the world, normalized to " + bounds);
194            }
195            DataSource src = new DataSource(bounds, origin);
196            ds.dataSources.add(src);
197        } else {
198            throwException(tr("Missing mandatory attributes on element ''bounds''. " +
199                    "Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
200                    minlon, minlat, maxlon, maxlat, origin
201            ));
202        }
203        jumpToEnd();
204    }
205
206    protected Node parseNode() throws XMLStreamException {
207        NodeData nd = new NodeData();
208        String lat = parser.getAttributeValue(null, "lat");
209        String lon = parser.getAttributeValue(null, "lon");
210        if (lat != null && lon != null) {
211            nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon)));
212        }
213        readCommon(nd);
214        Node n = new Node(nd.getId(), nd.getVersion());
215        n.setVisible(nd.isVisible());
216        n.load(nd);
217        externalIdMap.put(nd.getPrimitiveId(), n);
218        while (true) {
219            int event = parser.next();
220            if (event == XMLStreamConstants.START_ELEMENT) {
221                if ("tag".equals(parser.getLocalName())) {
222                    parseTag(n);
223                } else {
224                    parseUnknown();
225                }
226            } else if (event == XMLStreamConstants.END_ELEMENT)
227                return n;
228        }
229    }
230
231    protected Way parseWay() throws XMLStreamException {
232        WayData wd = new WayData();
233        readCommon(wd);
234        Way w = new Way(wd.getId(), wd.getVersion());
235        w.setVisible(wd.isVisible());
236        w.load(wd);
237        externalIdMap.put(wd.getPrimitiveId(), w);
238
239        Collection<Long> nodeIds = new ArrayList<>();
240        while (true) {
241            int event = parser.next();
242            if (event == XMLStreamConstants.START_ELEMENT) {
243                switch (parser.getLocalName()) {
244                case "nd":
245                    nodeIds.add(parseWayNode(w));
246                    break;
247                case "tag":
248                    parseTag(w);
249                    break;
250                default:
251                    parseUnknown();
252                }
253            } else if (event == XMLStreamConstants.END_ELEMENT) {
254                break;
255            }
256        }
257        if (w.isDeleted() && !nodeIds.isEmpty()) {
258            Main.info(tr("Deleted way {0} contains nodes", w.getUniqueId()));
259            nodeIds = new ArrayList<>();
260        }
261        ways.put(wd.getUniqueId(), nodeIds);
262        return w;
263    }
264
265    private long parseWayNode(Way w) throws XMLStreamException {
266        if (parser.getAttributeValue(null, "ref") == null) {
267            throwException(
268                    tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId())
269            );
270        }
271        long id = getLong("ref");
272        if (id == 0) {
273            throwException(
274                    tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
275            );
276        }
277        jumpToEnd();
278        return id;
279    }
280
281    protected Relation parseRelation() throws XMLStreamException {
282        RelationData rd = new RelationData();
283        readCommon(rd);
284        Relation r = new Relation(rd.getId(), rd.getVersion());
285        r.setVisible(rd.isVisible());
286        r.load(rd);
287        externalIdMap.put(rd.getPrimitiveId(), r);
288
289        Collection<RelationMemberData> members = new ArrayList<>();
290        while (true) {
291            int event = parser.next();
292            if (event == XMLStreamConstants.START_ELEMENT) {
293                switch (parser.getLocalName()) {
294                case "member":
295                    members.add(parseRelationMember(r));
296                    break;
297                case "tag":
298                    parseTag(r);
299                    break;
300                default:
301                    parseUnknown();
302                }
303            } else if (event == XMLStreamConstants.END_ELEMENT) {
304                break;
305            }
306        }
307        if (r.isDeleted() && !members.isEmpty()) {
308            Main.info(tr("Deleted relation {0} contains members", r.getUniqueId()));
309            members = new ArrayList<>();
310        }
311        relations.put(rd.getUniqueId(), members);
312        return r;
313    }
314
315    private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
316        String role = null;
317        OsmPrimitiveType type = null;
318        long id = 0;
319        String value = parser.getAttributeValue(null, "ref");
320        if (value == null) {
321            throwException(tr("Missing attribute ''ref'' on member in relation {0}.", r.getUniqueId()));
322        }
323        try {
324            id = Long.parseLong(value);
325        } catch (NumberFormatException e) {
326            throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),
327                    value), e);
328        }
329        value = parser.getAttributeValue(null, "type");
330        if (value == null) {
331            throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId())));
332        }
333        try {
334            type = OsmPrimitiveType.fromApiTypeName(value);
335        } catch (IllegalArgumentException e) {
336            throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.",
337                    Long.toString(id), Long.toString(r.getUniqueId()), value), e);
338        }
339        value = parser.getAttributeValue(null, "role");
340        role = value;
341
342        if (id == 0) {
343            throwException(tr("Incomplete <member> specification with ref=0"));
344        }
345        jumpToEnd();
346        return new RelationMemberData(role, type, id);
347    }
348
349    private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
350
351        Long id = null;
352        if (parser.getAttributeValue(null, "id") != null) {
353            id = getLong("id");
354        }
355        // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value
356        if (id == uploadChangesetId || (id != null && id.equals(uploadChangesetId))) {
357            uploadChangeset = new Changeset(id != null ? id.intValue() : 0);
358            while (true) {
359                int event = parser.next();
360                if (event == XMLStreamConstants.START_ELEMENT) {
361                    if ("tag".equals(parser.getLocalName())) {
362                        parseTag(uploadChangeset);
363                    } else {
364                        parseUnknown();
365                    }
366                } else if (event == XMLStreamConstants.END_ELEMENT)
367                    return;
368            }
369        } else {
370            jumpToEnd(false);
371        }
372    }
373
374    private void parseTag(Tagged t) throws XMLStreamException {
375        String key = parser.getAttributeValue(null, "k");
376        String value = parser.getAttributeValue(null, "v");
377        if (key == null || value == null) {
378            throwException(tr("Missing key or value attribute in tag."));
379        } else {
380            t.put(key.intern(), value.intern());
381        }
382        jumpToEnd();
383    }
384
385    protected void parseUnknown(boolean printWarning) throws XMLStreamException {
386        final String element = parser.getLocalName();
387        if (printWarning && ("note".equals(element) || "meta".equals(element))) {
388            // we know that Overpass API returns those elements
389            Main.debug(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
390        } else if (printWarning) {
391            Main.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
392        }
393        while (true) {
394            int event = parser.next();
395            if (event == XMLStreamConstants.START_ELEMENT) {
396                parseUnknown(false); /* no more warning for inner elements */
397            } else if (event == XMLStreamConstants.END_ELEMENT)
398                return;
399        }
400    }
401
402    protected void parseUnknown() throws XMLStreamException {
403        parseUnknown(true);
404    }
405
406    /**
407     * When cursor is at the start of an element, moves it to the end tag of that element.
408     * Nested content is skipped.
409     *
410     * This is basically the same code as parseUnknown(), except for the warnings, which
411     * are displayed for inner elements and not at top level.
412     * @param printWarning if {@code true}, a warning message will be printed if an unknown element is met
413     * @throws XMLStreamException if there is an error processing the underlying XML source
414     */
415    private void jumpToEnd(boolean printWarning) throws XMLStreamException {
416        while (true) {
417            int event = parser.next();
418            if (event == XMLStreamConstants.START_ELEMENT) {
419                parseUnknown(printWarning);
420            } else if (event == XMLStreamConstants.END_ELEMENT)
421                return;
422        }
423    }
424
425    private void jumpToEnd() throws XMLStreamException {
426        jumpToEnd(true);
427    }
428
429    private User createUser(String uid, String name) throws XMLStreamException {
430        if (uid == null) {
431            if (name == null)
432                return null;
433            return User.createLocalUser(name);
434        }
435        try {
436            long id = Long.parseLong(uid);
437            return User.createOsmUser(id, name);
438        } catch (NumberFormatException e) {
439            throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e);
440        }
441        return null;
442    }
443
444    /**
445     * Read out the common attributes and put them into current OsmPrimitive.
446     * @param current primitive to update
447     * @throws XMLStreamException if there is an error processing the underlying XML source
448     */
449    private void readCommon(PrimitiveData current) throws XMLStreamException {
450        current.setId(getLong("id"));
451        if (current.getUniqueId() == 0) {
452            throwException(tr("Illegal object with ID=0."));
453        }
454
455        String time = parser.getAttributeValue(null, "timestamp");
456        if (time != null && !time.isEmpty()) {
457            current.setRawTimestamp((int) (DateUtils.tsFromString(time)/1000));
458        }
459
460        String user = parser.getAttributeValue(null, "user");
461        String uid = parser.getAttributeValue(null, "uid");
462        current.setUser(createUser(uid, user));
463
464        String visible = parser.getAttributeValue(null, "visible");
465        if (visible != null) {
466            current.setVisible(Boolean.parseBoolean(visible));
467        }
468
469        String versionString = parser.getAttributeValue(null, "version");
470        int version = 0;
471        if (versionString != null) {
472            try {
473                version = Integer.parseInt(versionString);
474            } catch (NumberFormatException e) {
475                throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
476                        Long.toString(current.getUniqueId()), versionString), e);
477            }
478            switch (ds.getVersion()) {
479            case "0.6":
480                if (version <= 0 && !current.isNew()) {
481                    throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
482                            Long.toString(current.getUniqueId()), versionString));
483                } else if (version < 0 && current.isNew()) {
484                    Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.",
485                            current.getUniqueId(), version, 0, "0.6"));
486                    version = 0;
487                }
488                break;
489            default:
490                // should not happen. API version has been checked before
491                throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
492            }
493        } else {
494            // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
495            if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) {
496                throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
497            }
498        }
499        current.setVersion(version);
500
501        String action = parser.getAttributeValue(null, "action");
502        if (action == null) {
503            // do nothing
504        } else if ("delete".equals(action)) {
505            current.setDeleted(true);
506            current.setModified(current.isVisible());
507        } else if ("modify".equals(action)) {
508            current.setModified(true);
509        }
510
511        String v = parser.getAttributeValue(null, "changeset");
512        if (v == null) {
513            current.setChangesetId(0);
514        } else {
515            try {
516                current.setChangesetId(Integer.parseInt(v));
517            } catch (IllegalArgumentException e) {
518                Main.debug(e.getMessage());
519                if (current.isNew()) {
520                    // for a new primitive we just log a warning
521                    Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
522                            v, current.getUniqueId()));
523                    current.setChangesetId(0);
524                } else {
525                    // for an existing primitive this is a problem
526                    throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e);
527                }
528            } catch (IllegalStateException e) {
529                // thrown for positive changeset id on new primitives
530                Main.info(e.getMessage());
531                current.setChangesetId(0);
532            }
533            if (current.getChangesetId() <= 0) {
534                if (current.isNew()) {
535                    // for a new primitive we just log a warning
536                    Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
537                            v, current.getUniqueId()));
538                    current.setChangesetId(0);
539                } else {
540                    // for an existing primitive this is a problem
541                    throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
542                }
543            }
544        }
545    }
546
547    private long getLong(String name) throws XMLStreamException {
548        String value = parser.getAttributeValue(null, name);
549        if (value == null) {
550            throwException(tr("Missing required attribute ''{0}''.", name));
551        }
552        try {
553            return Long.parseLong(value);
554        } catch (NumberFormatException e) {
555            throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.", name, value), e);
556        }
557        return 0; // should not happen
558    }
559
560    private static class OsmParsingException extends XMLStreamException {
561
562        OsmParsingException(String msg, Location location) {
563            super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */
564            this.location = location;
565        }
566
567        OsmParsingException(String msg, Location location, Throwable th) {
568            super(msg, th);
569            this.location = location;
570        }
571
572        @Override
573        public String getMessage() {
574            String msg = super.getMessage();
575            if (msg == null) {
576                msg = getClass().getName();
577            }
578            if (getLocation() == null)
579                return msg;
580            msg += ' ' + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber());
581            int offset = getLocation().getCharacterOffset();
582            if (offset > -1) {
583                msg += ". "+ tr("{0} bytes have been read", offset);
584            }
585            return msg;
586        }
587    }
588
589    /**
590     * Exception thrown after user cancelation.
591     */
592    private static final class OsmParsingCanceledException extends OsmParsingException implements ImportCancelException {
593        /**
594         * Constructs a new {@code OsmParsingCanceledException}.
595         * @param msg The error message
596         * @param location The parser location
597         */
598        OsmParsingCanceledException(String msg, Location location) {
599            super(msg, location);
600        }
601    }
602
603    protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
604        if (progressMonitor == null) {
605            progressMonitor = NullProgressMonitor.INSTANCE;
606        }
607        ProgressMonitor.CancelListener cancelListener = new ProgressMonitor.CancelListener() {
608            @Override public void operationCanceled() {
609                cancel = true;
610            }
611        };
612        progressMonitor.addCancelListener(cancelListener);
613        CheckParameterUtil.ensureParameterNotNull(source, "source");
614        try {
615            progressMonitor.beginTask(tr("Prepare OSM data...", 2));
616            progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
617
618            try (InputStreamReader ir = UTFInputStreamReader.create(source)) {
619                XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir);
620                setParser(parser);
621                parse();
622            }
623            progressMonitor.worked(1);
624
625            progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
626            prepareDataSet();
627            progressMonitor.worked(1);
628
629            // iterate over registered postprocessors and give them each a chance
630            // to modify the dataset we have just loaded.
631            if (postprocessors != null) {
632                for (OsmServerReadPostprocessor pp : postprocessors) {
633                    pp.postprocessDataSet(getDataSet(), progressMonitor);
634                }
635            }
636            return getDataSet();
637        } catch (IllegalDataException e) {
638            throw e;
639        } catch (OsmParsingException e) {
640            throw new IllegalDataException(e.getMessage(), e);
641        } catch (XMLStreamException e) {
642            String msg = e.getMessage();
643            Pattern p = Pattern.compile("Message: (.+)");
644            Matcher m = p.matcher(msg);
645            if (m.find()) {
646                msg = m.group(1);
647            }
648            if (e.getLocation() != null)
649                throw new IllegalDataException(tr("Line {0} column {1}: ",
650                        e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
651            else
652                throw new IllegalDataException(msg, e);
653        } catch (Exception e) {
654            throw new IllegalDataException(e);
655        } finally {
656            progressMonitor.finishTask();
657            progressMonitor.removeCancelListener(cancelListener);
658        }
659    }
660
661    /**
662     * Parse the given input source and return the dataset.
663     *
664     * @param source the source input stream. Must not be null.
665     * @param progressMonitor  the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
666     *
667     * @return the dataset with the parsed data
668     * @throws IllegalDataException if an error was found while parsing the data from the source
669     * @throws IllegalArgumentException if source is null
670     */
671    public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
672        return new OsmReader().doParseDataSet(source, progressMonitor);
673    }
674}