001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005import static org.openstreetmap.josm.tools.I18n.trn;
006
007import java.io.IOException;
008import java.io.InputStream;
009import java.net.HttpURLConnection;
010import java.util.ArrayList;
011import java.util.Collection;
012import java.util.HashSet;
013import java.util.Iterator;
014import java.util.LinkedHashSet;
015import java.util.List;
016import java.util.NoSuchElementException;
017import java.util.Set;
018import java.util.concurrent.Callable;
019import java.util.concurrent.CompletionService;
020import java.util.concurrent.ExecutionException;
021import java.util.concurrent.Executor;
022import java.util.concurrent.ExecutorCompletionService;
023import java.util.concurrent.Executors;
024import java.util.concurrent.Future;
025
026import org.openstreetmap.josm.Main;
027import org.openstreetmap.josm.data.osm.DataSet;
028import org.openstreetmap.josm.data.osm.DataSetMerger;
029import org.openstreetmap.josm.data.osm.Node;
030import org.openstreetmap.josm.data.osm.OsmPrimitive;
031import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
032import org.openstreetmap.josm.data.osm.PrimitiveId;
033import org.openstreetmap.josm.data.osm.Relation;
034import org.openstreetmap.josm.data.osm.RelationMember;
035import org.openstreetmap.josm.data.osm.SimplePrimitiveId;
036import org.openstreetmap.josm.data.osm.Way;
037import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
038import org.openstreetmap.josm.gui.progress.ProgressMonitor;
039import org.openstreetmap.josm.tools.CheckParameterUtil;
040
041/**
042 * Retrieves a set of {@link OsmPrimitive}s from an OSM server using the so called
043 * Multi Fetch API.
044 *
045 * Usage:
046 * <pre>
047 *    MultiFetchServerObjectReader reader = MultiFetchServerObjectReader()
048 *         .append(2345,2334,4444)
049 *         .append(new Node(72343));
050 *    reader.parseOsm();
051 *    if (!reader.getMissingPrimitives().isEmpty()) {
052 *        Main.info("There are missing primitives: " + reader.getMissingPrimitives());
053 *    }
054 *    if (!reader.getSkippedWays().isEmpty()) {
055 *       Main.info("There are skipped ways: " + reader.getMissingPrimitives());
056 *    }
057 * </pre>
058 */
059public class MultiFetchServerObjectReader extends OsmServerReader{
060    /**
061     * the max. number of primitives retrieved in one step. Assuming IDs with 7 digits,
062     * this leads to a max. request URL of ~ 1600 Bytes ((7 digits +  1 Separator) * 200),
063     * which should be safe according to the
064     * <a href="http://www.boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>.
065     */
066    private static final int MAX_IDS_PER_REQUEST = 200;
067
068    private Set<Long> nodes;
069    private Set<Long> ways;
070    private Set<Long> relations;
071    private Set<PrimitiveId> missingPrimitives;
072    private DataSet outputDataSet;
073
074    /**
075     * Constructs a {@code MultiFetchServerObjectReader}.
076     */
077    public MultiFetchServerObjectReader() {
078        nodes = new LinkedHashSet<>();
079        ways = new LinkedHashSet<>();
080        relations = new LinkedHashSet<>();
081        this.outputDataSet = new DataSet();
082        this.missingPrimitives = new LinkedHashSet<>();
083    }
084
085    /**
086     * Remembers an {@link OsmPrimitive}'s id. The id will
087     * later be fetched as part of a Multi Get request.
088     *
089     * Ignore the id if it represents a new primitives.
090     *
091     * @param id  the id
092     */
093    protected void remember(PrimitiveId id) {
094        if (id.isNew()) return;
095        switch(id.getType()) {
096        case NODE: nodes.add(id.getUniqueId()); break;
097        case WAY: ways.add(id.getUniqueId()); break;
098        case RELATION: relations.add(id.getUniqueId()); break;
099        }
100    }
101
102    /**
103     * remembers an {@link OsmPrimitive}'s id. <code>ds</code> must include
104     * an {@link OsmPrimitive} with id=<code>id</code>. The id will
105     * later we fetched as part of a Multi Get request.
106     *
107     * Ignore the id if it id &lt;= 0.
108     *
109     * @param ds  the dataset (must not be null)
110     * @param id  the primitive id
111     * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
112     * @throws IllegalArgumentException if ds is null
113     * @throws NoSuchElementException if ds does not include an {@link OsmPrimitive} with id=<code>id</code>
114     */
115    protected void remember(DataSet ds, long id, OsmPrimitiveType type) throws IllegalArgumentException, NoSuchElementException{
116        CheckParameterUtil.ensureParameterNotNull(ds, "ds");
117        if (id <= 0) return;
118        OsmPrimitive primitive = ds.getPrimitiveById(id, type);
119        if (primitive == null)
120            throw new NoSuchElementException(tr("No primitive with id {0} in local dataset. Cannot infer primitive type.", id));
121        remember(primitive.getPrimitiveId());
122        return;
123    }
124
125    /**
126     * appends a {@link OsmPrimitive} id to the list of ids which will be fetched from the server.
127     *
128     * @param ds the {@link DataSet} to which the primitive belongs
129     * @param id the primitive id
130     * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
131     * @return this
132     */
133    public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) {
134        OsmPrimitive p = ds.getPrimitiveById(id,type);
135        switch(type) {
136        case NODE:
137            return appendNode((Node)p);
138        case WAY:
139            return appendWay((Way)p);
140        case RELATION:
141            return appendRelation((Relation)p);
142        }
143        return this;
144    }
145
146    /**
147     * appends a {@link Node} id to the list of ids which will be fetched from the server.
148     *
149     * @param node  the node (ignored, if null)
150     * @return this
151     */
152    public MultiFetchServerObjectReader appendNode(Node node) {
153        if (node == null) return this;
154        remember(node.getPrimitiveId());
155        return this;
156    }
157
158    /**
159     * appends a {@link Way} id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server.
160     *
161     * @param way the way (ignored, if null)
162     * @return this
163     */
164    public MultiFetchServerObjectReader appendWay(Way way) {
165        if (way == null) return this;
166        if (way.isNew()) return this;
167        for (Node node: way.getNodes()) {
168            if (!node.isNew()) {
169                remember(node.getPrimitiveId());
170            }
171        }
172        remember(way.getPrimitiveId());
173        return this;
174    }
175
176    /**
177     * appends a {@link Relation} id to the list of ids which will be fetched from the server.
178     *
179     * @param relation  the relation (ignored, if null)
180     * @return this
181     */
182    protected MultiFetchServerObjectReader appendRelation(Relation relation) {
183        if (relation == null) return this;
184        if (relation.isNew()) return this;
185        remember(relation.getPrimitiveId());
186        for (RelationMember member : relation.getMembers()) {
187            if (OsmPrimitiveType.from(member.getMember()).equals(OsmPrimitiveType.RELATION)) {
188                // avoid infinite recursion in case of cyclic dependencies in relations
189                //
190                if (relations.contains(member.getMember().getId())) {
191                    continue;
192                }
193            }
194            if (!member.getMember().isIncomplete()) {
195                append(member.getMember());
196            }
197        }
198        return this;
199    }
200
201    /**
202     * appends an {@link OsmPrimitive} to the list of ids which will be fetched from the server.
203     * @param primitive the primitive
204     * @return this
205     */
206    public MultiFetchServerObjectReader append(OsmPrimitive primitive) {
207        if (primitive != null) {
208            switch (OsmPrimitiveType.from(primitive)) {
209                case NODE: return appendNode((Node)primitive);
210                case WAY: return appendWay((Way)primitive);
211                case RELATION: return appendRelation((Relation)primitive);
212            }
213        }
214        return this;
215    }
216
217    /**
218     * appends a list of {@link OsmPrimitive} to the list of ids which will be fetched from the server.
219     *
220     * @param primitives  the list of primitives (ignored, if null)
221     * @return this
222     *
223     * @see #append(OsmPrimitive)
224     */
225    public MultiFetchServerObjectReader append(Collection<? extends OsmPrimitive> primitives) {
226        if (primitives == null) return this;
227        for (OsmPrimitive primitive : primitives) {
228            append(primitive);
229        }
230        return this;
231    }
232
233    /**
234     * extracts a subset of max {@link #MAX_IDS_PER_REQUEST} ids from <code>ids</code> and
235     * replies the subset. The extracted subset is removed from <code>ids</code>.
236     *
237     * @param ids a set of ids
238     * @return the subset of ids
239     */
240    protected Set<Long> extractIdPackage(Set<Long> ids) {
241        HashSet<Long> pkg = new HashSet<>();
242        if (ids.isEmpty())
243            return pkg;
244        if (ids.size() > MAX_IDS_PER_REQUEST) {
245            Iterator<Long> it = ids.iterator();
246            for (int i=0; i<MAX_IDS_PER_REQUEST; i++) {
247                pkg.add(it.next());
248            }
249            ids.removeAll(pkg);
250        } else {
251            pkg.addAll(ids);
252            ids.clear();
253        }
254        return pkg;
255    }
256
257    /**
258     * builds the Multi Get request string for a set of ids and a given
259     * {@link OsmPrimitiveType}.
260     *
261     * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
262     * @param idPackage  the package of ids
263     * @return the request string
264     */
265    protected static String buildRequestString(OsmPrimitiveType type, Set<Long> idPackage) {
266        StringBuilder sb = new StringBuilder();
267        sb.append(type.getAPIName()).append("s?")
268        .append(type.getAPIName()).append("s=");
269
270        Iterator<Long> it = idPackage.iterator();
271        for (int i=0; i<idPackage.size(); i++) {
272            sb.append(it.next());
273            if (i < idPackage.size()-1) {
274                sb.append(",");
275            }
276        }
277        return sb.toString();
278    }
279
280    /**
281     * builds the Multi Get request string for a single id and a given
282     * {@link OsmPrimitiveType}.
283     *
284     * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
285     * @param id the id
286     * @return the request string
287     */
288    protected static String buildRequestString(OsmPrimitiveType type, long id) {
289        StringBuilder sb = new StringBuilder();
290        sb.append(type.getAPIName()).append("s?")
291        .append(type.getAPIName()).append("s=")
292        .append(id);
293        return sb.toString();
294    }
295
296    protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) {
297        for (Way w: from.getWays()) {
298            if (w.hasIncompleteNodes()) {
299                for (Node n: w.getNodes()) {
300                    if (n.isIncomplete()) {
301                        nodes.add(n.getId());
302                    }
303                }
304            }
305        }
306    }
307
308    /**
309     * merges the dataset <code>from</code> to {@link #outputDataSet}.
310     *
311     * @param from the other dataset
312     */
313    protected void merge(DataSet from) {
314        final DataSetMerger visitor = new DataSetMerger(outputDataSet,from);
315        visitor.merge();
316    }
317
318    /**
319     * fetches a set of ids of a given {@link OsmPrimitiveType} from the server
320     *
321     * @param ids the set of ids
322     * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
323     * @throws OsmTransferException if an error occurs while communicating with the API server
324     */
325    protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException {
326        String msg = "";
327        String baseUrl = OsmApi.getOsmApi().getBaseUrl();
328        switch (type) {
329            case NODE:     msg = tr("Fetching a package of nodes from ''{0}''",     baseUrl); break;
330            case WAY:      msg = tr("Fetching a package of ways from ''{0}''",      baseUrl); break;
331            case RELATION: msg = tr("Fetching a package of relations from ''{0}''", baseUrl); break;
332        }
333        progressMonitor.setTicksCount(ids.size());
334        progressMonitor.setTicks(0);
335        // The complete set containg all primitives to fetch
336        Set<Long> toFetch = new HashSet<>(ids);
337        // Build a list of fetchers that will  download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each.
338        // we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers.
339        int threadsNumber = Main.pref.getInteger("osm.download.threads", OsmApi.MAX_DOWNLOAD_THREADS);
340        threadsNumber = Math.min(Math.max(threadsNumber, 1), OsmApi.MAX_DOWNLOAD_THREADS);
341        Executor exec = Executors.newFixedThreadPool(threadsNumber);
342        CompletionService<FetchResult> ecs = new ExecutorCompletionService<>(exec);
343        List<Future<FetchResult>> jobs = new ArrayList<>();
344        while (!toFetch.isEmpty()) {
345            jobs.add(ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor)));
346        }
347        // Run the fetchers
348        for (int i = 0; i < jobs.size() && !isCanceled(); i++) {
349            progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + "/" + progressMonitor.getTicksCount());
350            try {
351                FetchResult result = ecs.take().get();
352                if (result.missingPrimitives != null) {
353                    missingPrimitives.addAll(result.missingPrimitives);
354                }
355                if (result.dataSet != null && !isCanceled()) {
356                    rememberNodesOfIncompleteWaysToLoad(result.dataSet);
357                    merge(result.dataSet);
358                }
359            } catch (InterruptedException | ExecutionException e) {
360                Main.error(e);
361            }
362        }
363        // Cancel requests if the user choosed to
364        if (isCanceled()) {
365            for (Future<FetchResult> job : jobs) {
366                job.cancel(true);
367            }
368        }
369    }
370
371    /**
372     * invokes one or more Multi Gets to fetch the {@link OsmPrimitive}s and replies
373     * the dataset of retrieved primitives. Note that the dataset includes non visible primitives too!
374     * In contrast to a simple Get for a node, a way, or a relation, a Multi Get always replies
375     * the latest version of the primitive (if any), even if the primitive is not visible (i.e. if
376     * visible==false).
377     *
378     * Invoke {@link #getMissingPrimitives()} to get a list of primitives which have not been
379     * found on  the server (the server response code was 404)
380     *
381     * @return the parsed data
382     * @throws OsmTransferException if an error occurs while communicating with the API server
383     * @see #getMissingPrimitives()
384     *
385     */
386    @Override
387    public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
388        int n = nodes.size() + ways.size() + relations.size();
389        progressMonitor.beginTask(trn("Downloading {0} object from ''{1}''",
390                "Downloading {0} objects from ''{1}''", n, n, OsmApi.getOsmApi().getBaseUrl()));
391        try {
392            missingPrimitives = new HashSet<>();
393            if (isCanceled()) return null;
394            fetchPrimitives(ways,OsmPrimitiveType.WAY, progressMonitor);
395            if (isCanceled()) return null;
396            fetchPrimitives(nodes,OsmPrimitiveType.NODE, progressMonitor);
397            if (isCanceled()) return null;
398            fetchPrimitives(relations,OsmPrimitiveType.RELATION, progressMonitor);
399            if (outputDataSet != null) {
400                outputDataSet.deleteInvisible();
401            }
402            return outputDataSet;
403        } finally {
404            progressMonitor.finishTask();
405        }
406    }
407
408    /**
409     * replies the set of ids of all primitives for which a fetch request to the
410     * server was submitted but which are not available from the server (the server
411     * replied a return code of 404)
412     *
413     * @return the set of ids of missing primitives
414     */
415    public Set<PrimitiveId> getMissingPrimitives() {
416        return missingPrimitives;
417    }
418
419    /**
420     * The class holding the results given by {@link Fetcher}.
421     * It is only a wrapper of the resulting {@link DataSet} and the collection of {@link PrimitiveId} that could not have been loaded.
422     */
423    protected static class FetchResult {
424
425        /**
426         * The resulting data set
427         */
428        public final DataSet dataSet;
429
430        /**
431         * The collection of primitive ids that could not have been loaded
432         */
433        public final Set<PrimitiveId> missingPrimitives;
434
435        /**
436         * Constructs a {@code FetchResult}
437         * @param dataSet The resulting data set
438         * @param missingPrimitives The collection of primitive ids that could not have been loaded
439         */
440        public FetchResult(DataSet dataSet, Set<PrimitiveId> missingPrimitives) {
441            this.dataSet = dataSet;
442            this.missingPrimitives = missingPrimitives;
443        }
444    }
445
446    /**
447     * The class that actually download data from OSM API. Several instances of this class are used by {@link MultiFetchServerObjectReader} (one per set of primitives to fetch).
448     * The inheritance of {@link OsmServerReader} is only explained by the need to have a distinct OSM connection by {@code Fetcher} instance.
449     * @see FetchResult
450     */
451    protected static class Fetcher extends OsmServerReader implements Callable<FetchResult> {
452
453        private final Set<Long> pkg;
454        private final OsmPrimitiveType type;
455        private final ProgressMonitor progressMonitor;
456
457        /**
458         * Constructs a {@code Fetcher}
459         * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
460         * @param idsPackage The set of primitives ids to fetch
461         * @param progressMonitor The progress monitor
462         */
463        public Fetcher(OsmPrimitiveType type, Set<Long> idsPackage, ProgressMonitor progressMonitor) {
464            this.pkg = idsPackage;
465            this.type = type;
466            this.progressMonitor = progressMonitor;
467        }
468
469        @Override
470        public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
471            // This method is implemented because of the OsmServerReader inheritance, but not used, as the main target of this class is the call() method.
472            return fetch(progressMonitor).dataSet;
473        }
474
475        @Override
476        public FetchResult call() throws Exception {
477            return fetch(progressMonitor);
478        }
479
480        /**
481         * fetches the requested primitives and updates the specified progress monitor.
482         * @param progressMonitor the progress monitor
483         * @return the {@link FetchResult} of this operation
484         * @throws OsmTransferException if an error occurs while communicating with the API server
485         */
486        protected FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException {
487            try {
488                return multiGetIdPackage(type, pkg, progressMonitor);
489            } catch (OsmApiException e) {
490                if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
491                    Main.info(tr("Server replied with response code 404, retrying with an individual request for each object."));
492                    return singleGetIdPackage(type, pkg, progressMonitor);
493                } else {
494                    throw e;
495                }
496            }
497        }
498
499        /**
500         * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}.
501         * The retrieved primitives are merged to {@link #outputDataSet}.
502         *
503         * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
504         * @param pkg the package of ids
505         * @return the {@link FetchResult} of this operation
506         * @throws OsmTransferException if an error occurs while communicating with the API server
507         */
508        protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException {
509            String request = buildRequestString(type, pkg);
510            FetchResult result = null;
511            try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) {
512                if (in == null) return null;
513                progressMonitor.subTask(tr("Downloading OSM data..."));
514                try {
515                    result = new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null);
516                } catch (Exception e) {
517                    throw new OsmTransferException(e);
518                }
519            } catch (IOException ex) {
520                Main.warn(ex);
521            }
522            return result;
523        }
524
525        /**
526         * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}.
527         * The retrieved primitive is merged to {@link #outputDataSet}.
528         *
529         * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
530         * @param id the id
531         * @return the {@link DataSet} resulting of this operation
532         * @throws OsmTransferException if an error occurs while communicating with the API server
533         */
534        protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException {
535            String request = buildRequestString(type, id);
536            DataSet result = null;
537            try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) {
538                if (in == null) return null;
539                progressMonitor.subTask(tr("Downloading OSM data..."));
540                try {
541                    result = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false));
542                } catch (Exception e) {
543                    throw new OsmTransferException(e);
544                }
545            } catch (IOException ex) {
546                Main.warn(ex);
547            }
548            return result;
549        }
550
551        /**
552         * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}.
553         * The retrieved primitives are merged to {@link #outputDataSet}.
554         *
555         * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404).
556         * If the set is fetched with this method it is possible to find out which of the ids doesn't exist.
557         * Unfortunately, the server does not provide an error header or an error body for a 404 reply.
558         *
559         * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
560         * @param pkg the set of ids
561         * @return the {@link FetchResult} of this operation
562         * @throws OsmTransferException if an error occurs while communicating with the API server
563         */
564        protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException {
565            FetchResult result = new FetchResult(new DataSet(), new HashSet<PrimitiveId>());
566            String baseUrl = OsmApi.getOsmApi().getBaseUrl();
567            for (long id : pkg) {
568                try {
569                    String msg = "";
570                    switch (type) {
571                        case NODE:     msg = tr("Fetching node with id {0} from ''{1}''",     id, baseUrl); break;
572                        case WAY:      msg = tr("Fetching way with id {0} from ''{1}''",      id, baseUrl); break;
573                        case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break;
574                    }
575                    progressMonitor.setCustomText(msg);
576                    result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor));
577                } catch (OsmApiException e) {
578                    if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
579                        Main.info(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id)));
580                        result.missingPrimitives.add(new SimplePrimitiveId(id, type));
581                    } else {
582                        throw e;
583                    }
584                }
585            }
586            return result;
587        }
588    }
589}