001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.tools;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.BufferedOutputStream;
007import java.io.BufferedReader;
008import java.io.IOException;
009import java.io.InputStream;
010import java.io.OutputStream;
011import java.net.CookieHandler;
012import java.net.CookieManager;
013import java.net.HttpURLConnection;
014import java.net.URL;
015import java.util.Collections;
016import java.util.List;
017import java.util.Locale;
018import java.util.Map;
019import java.util.Map.Entry;
020import java.util.Scanner;
021import java.util.TreeMap;
022import java.util.regex.Matcher;
023import java.util.regex.Pattern;
024import java.util.zip.GZIPInputStream;
025
026import org.openstreetmap.josm.Main;
027import org.openstreetmap.josm.data.Version;
028import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
029import org.openstreetmap.josm.gui.progress.ProgressMonitor;
030import org.openstreetmap.josm.io.Compression;
031import org.openstreetmap.josm.io.ProgressInputStream;
032import org.openstreetmap.josm.io.ProgressOutputStream;
033import org.openstreetmap.josm.io.UTFInputStreamReader;
034import org.openstreetmap.josm.io.auth.DefaultAuthenticator;
035
036/**
037 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}.
038 * @since 9168
039 */
040public final class HttpClient {
041
042    private URL url;
043    private final String requestMethod;
044    private int connectTimeout = Main.pref.getInteger("socket.timeout.connect", 15) * 1000;
045    private int readTimeout = Main.pref.getInteger("socket.timeout.read", 30) * 1000;
046    private byte[] requestBody;
047    private long ifModifiedSince;
048    private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
049    private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5);
050    private boolean useCache;
051    private String reasonForRequest;
052    private HttpURLConnection connection; // to allow disconnecting before `response` is set
053    private Response response;
054    private boolean finishOnCloseOutput = true;
055
056    static {
057        CookieHandler.setDefault(new CookieManager());
058    }
059
060    private HttpClient(URL url, String requestMethod) {
061        this.url = url;
062        this.requestMethod = requestMethod;
063        this.headers.put("Accept-Encoding", "gzip");
064    }
065
066    /**
067     * Opens the HTTP connection.
068     * @return HTTP response
069     * @throws IOException if any I/O error occurs
070     */
071    public Response connect() throws IOException {
072        return connect(null);
073    }
074
075    /**
076     * Opens the HTTP connection.
077     * @param progressMonitor progress monitor
078     * @return HTTP response
079     * @throws IOException if any I/O error occurs
080     * @since 9179
081     */
082    public Response connect(ProgressMonitor progressMonitor) throws IOException {
083        if (progressMonitor == null) {
084            progressMonitor = NullProgressMonitor.INSTANCE;
085        }
086        final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
087        this.connection = connection;
088        connection.setRequestMethod(requestMethod);
089        connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString());
090        connection.setConnectTimeout(connectTimeout);
091        connection.setReadTimeout(readTimeout);
092        connection.setInstanceFollowRedirects(false); // we do that ourselves
093        if (ifModifiedSince > 0) {
094            connection.setIfModifiedSince(ifModifiedSince);
095        }
096        connection.setUseCaches(useCache);
097        if (!useCache) {
098            connection.setRequestProperty("Cache-Control", "no-cache");
099        }
100        for (Map.Entry<String, String> header : headers.entrySet()) {
101            if (header.getValue() != null) {
102                connection.setRequestProperty(header.getKey(), header.getValue());
103            }
104        }
105
106        progressMonitor.beginTask(tr("Contacting Server..."), 1);
107        progressMonitor.indeterminateSubTask(null);
108
109        if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) {
110            Main.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault()));
111            connection.setFixedLengthStreamingMode(requestBody.length);
112            connection.setDoOutput(true);
113            try (OutputStream out = new BufferedOutputStream(
114                    new ProgressOutputStream(connection.getOutputStream(), requestBody.length, progressMonitor, finishOnCloseOutput))) {
115                out.write(requestBody);
116            }
117        }
118
119        boolean successfulConnection = false;
120        try {
121            try {
122                connection.connect();
123                final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty();
124                Main.info("{0} {1}{2} -> {3}{4}",
125                        requestMethod, url, hasReason ? (" (" + reasonForRequest + ')') : "",
126                        connection.getResponseCode(),
127                        connection.getContentLengthLong() > 0
128                                ? (" (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ')')
129                                : ""
130                );
131                if (Main.isDebugEnabled()) {
132                    Main.debug("RESPONSE: " + connection.getHeaderFields());
133                }
134                if (DefaultAuthenticator.getInstance().isEnabled() && connection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
135                    DefaultAuthenticator.getInstance().addFailedCredentialHost(url.getHost());
136                }
137            } catch (IOException e) {
138                Main.info("{0} {1} -> !!!", requestMethod, url);
139                Main.warn(e);
140                //noinspection ThrowableResultOfMethodCallIgnored
141                Main.addNetworkError(url, Utils.getRootCause(e));
142                throw e;
143            }
144            if (isRedirect(connection.getResponseCode())) {
145                final String redirectLocation = connection.getHeaderField("Location");
146                if (redirectLocation == null) {
147                    /* I18n: argument is HTTP response code */
148                    String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." +
149                            " Can''t redirect. Aborting.", connection.getResponseCode());
150                    throw new IOException(msg);
151                } else if (maxRedirects > 0) {
152                    url = new URL(url, redirectLocation);
153                    maxRedirects--;
154                    Main.info(tr("Download redirected to ''{0}''", redirectLocation));
155                    return connect();
156                } else if (maxRedirects == 0) {
157                    String msg = tr("Too many redirects to the download URL detected. Aborting.");
158                    throw new IOException(msg);
159                }
160            }
161            response = new Response(connection, progressMonitor);
162            successfulConnection = true;
163            return response;
164        } finally {
165            if (!successfulConnection) {
166                connection.disconnect();
167            }
168        }
169    }
170
171    /**
172     * Returns the HTTP response which is set only after calling {@link #connect()}.
173     * Calling this method again, returns the identical object (unless another {@link #connect()} is performed).
174     *
175     * @return the HTTP response
176     * @since 9309
177     */
178    public Response getResponse() {
179        return response;
180    }
181
182    /**
183     * A wrapper for the HTTP response.
184     */
185    public static final class Response {
186        private final HttpURLConnection connection;
187        private final ProgressMonitor monitor;
188        private final int responseCode;
189        private final String responseMessage;
190        private boolean uncompress;
191        private boolean uncompressAccordingToContentDisposition;
192
193        private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException {
194            CheckParameterUtil.ensureParameterNotNull(connection, "connection");
195            CheckParameterUtil.ensureParameterNotNull(monitor, "monitor");
196            this.connection = connection;
197            this.monitor = monitor;
198            this.responseCode = connection.getResponseCode();
199            this.responseMessage = connection.getResponseMessage();
200        }
201
202        /**
203         * Sets whether {@link #getContent()} should uncompress the input stream if necessary.
204         *
205         * @param uncompress whether the input stream should be uncompressed if necessary
206         * @return {@code this}
207         */
208        public Response uncompress(boolean uncompress) {
209            this.uncompress = uncompress;
210            return this;
211        }
212
213        /**
214         * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition}
215         * HTTP header.
216         * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to
217         * {@code Content-Disposition}
218         * @return {@code this}
219         * @since 9172
220         */
221        public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) {
222            this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition;
223            return this;
224        }
225
226        /**
227         * Returns the URL.
228         * @return the URL
229         * @see HttpURLConnection#getURL()
230         * @since 9172
231         */
232        public URL getURL() {
233            return connection.getURL();
234        }
235
236        /**
237         * Returns the request method.
238         * @return the HTTP request method
239         * @see HttpURLConnection#getRequestMethod()
240         * @since 9172
241         */
242        public String getRequestMethod() {
243            return connection.getRequestMethod();
244        }
245
246        /**
247         * Returns an input stream that reads from this HTTP connection, or,
248         * error stream if the connection failed but the server sent useful data.
249         * <p>
250         * Note: the return value can be null, if both the input and the error stream are null.
251         * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887
252         * @return input or error stream
253         * @throws IOException if any I/O error occurs
254         *
255         * @see HttpURLConnection#getInputStream()
256         * @see HttpURLConnection#getErrorStream()
257         */
258        @SuppressWarnings("resource")
259        public InputStream getContent() throws IOException {
260            InputStream in;
261            try {
262                in = connection.getInputStream();
263            } catch (IOException ioe) {
264                Main.debug(ioe);
265                in = connection.getErrorStream();
266            }
267            if (in != null) {
268                in = new ProgressInputStream(in, getContentLength(), monitor);
269                in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in;
270                Compression compression = Compression.NONE;
271                if (uncompress) {
272                    final String contentType = getContentType();
273                    Main.debug("Uncompressing input stream according to Content-Type header: {0}", contentType);
274                    compression = Compression.forContentType(contentType);
275                }
276                if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) {
277                    final String contentDisposition = getHeaderField("Content-Disposition");
278                    final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher(
279                            contentDisposition != null ? contentDisposition : "");
280                    if (matcher.find()) {
281                        Main.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition);
282                        compression = Compression.byExtension(matcher.group(1));
283                    }
284                }
285                in = compression.getUncompressedInputStream(in);
286            }
287            return in;
288        }
289
290        /**
291         * Returns {@link #getContent()} wrapped in a buffered reader.
292         *
293         * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
294         * @return buffered reader
295         * @throws IOException if any I/O error occurs
296         */
297        public BufferedReader getContentReader() throws IOException {
298            return new BufferedReader(
299                    UTFInputStreamReader.create(getContent())
300            );
301        }
302
303        /**
304         * Fetches the HTTP response as String.
305         * @return the response
306         * @throws IOException if any I/O error occurs
307         */
308        @SuppressWarnings("resource")
309        public String fetchContent() throws IOException {
310            try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) {
311                return scanner.hasNext() ? scanner.next() : "";
312            }
313        }
314
315        /**
316         * Gets the response code from this HTTP connection.
317         * @return HTTP response code
318         *
319         * @see HttpURLConnection#getResponseCode()
320         */
321        public int getResponseCode() {
322            return responseCode;
323        }
324
325        /**
326         * Gets the response message from this HTTP connection.
327         * @return HTTP response message
328         *
329         * @see HttpURLConnection#getResponseMessage()
330         * @since 9172
331         */
332        public String getResponseMessage() {
333            return responseMessage;
334        }
335
336        /**
337         * Returns the {@code Content-Encoding} header.
338         * @return {@code Content-Encoding} HTTP header
339         * @see HttpURLConnection#getContentEncoding()
340         */
341        public String getContentEncoding() {
342            return connection.getContentEncoding();
343        }
344
345        /**
346         * Returns the {@code Content-Type} header.
347         * @return {@code Content-Type} HTTP header
348         */
349        public String getContentType() {
350            return connection.getHeaderField("Content-Type");
351        }
352
353        /**
354         * Returns the {@code Expire} header.
355         * @return {@code Expire} HTTP header
356         * @see HttpURLConnection#getExpiration()
357         * @since 9232
358         */
359        public long getExpiration() {
360            return connection.getExpiration();
361        }
362
363        /**
364         * Returns the {@code Last-Modified} header.
365         * @return {@code Last-Modified} HTTP header
366         * @see HttpURLConnection#getLastModified()
367         * @since 9232
368         */
369        public long getLastModified() {
370            return connection.getLastModified();
371        }
372
373        /**
374         * Returns the {@code Content-Length} header.
375         * @return {@code Content-Length} HTTP header
376         * @see HttpURLConnection#getContentLengthLong()
377         */
378        public long getContentLength() {
379            return connection.getContentLengthLong();
380        }
381
382        /**
383         * Returns the value of the named header field.
384         * @param name the name of a header field
385         * @return the value of the named header field, or {@code null} if there is no such field in the header
386         * @see HttpURLConnection#getHeaderField(String)
387         * @since 9172
388         */
389        public String getHeaderField(String name) {
390            return connection.getHeaderField(name);
391        }
392
393        /**
394         * Returns an unmodifiable Map mapping header keys to a List of header values.
395         * As per RFC 2616, section 4.2 header names are case insensitive, so returned map is also case insensitive
396         * @return unmodifiable Map mapping header keys to a List of header values
397         * @see HttpURLConnection#getHeaderFields()
398         * @since 9232
399         */
400        public Map<String, List<String>> getHeaderFields() {
401            // returned map from HttpUrlConnection is case sensitive, use case insensitive TreeMap to conform to RFC 2616
402            Map<String, List<String>> ret = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
403            for (Entry<String, List<String>> e: connection.getHeaderFields().entrySet()) {
404                if (e.getKey() != null) {
405                    ret.put(e.getKey(), e.getValue());
406                }
407            }
408            return Collections.unmodifiableMap(ret);
409        }
410
411        /**
412         * @see HttpURLConnection#disconnect()
413         */
414        public void disconnect() {
415            HttpClient.disconnect(connection);
416        }
417    }
418
419    /**
420     * Creates a new instance for the given URL and a {@code GET} request
421     *
422     * @param url the URL
423     * @return a new instance
424     */
425    public static HttpClient create(URL url) {
426        return create(url, "GET");
427    }
428
429    /**
430     * Creates a new instance for the given URL and a {@code GET} request
431     *
432     * @param url the URL
433     * @param requestMethod the HTTP request method to perform when calling
434     * @return a new instance
435     */
436    public static HttpClient create(URL url, String requestMethod) {
437        return new HttpClient(url, requestMethod);
438    }
439
440    /**
441     * Returns the URL set for this connection.
442     * @return the URL
443     * @see #create(URL)
444     * @see #create(URL, String)
445     * @since 9172
446     */
447    public URL getURL() {
448        return url;
449    }
450
451    /**
452     * Returns the request method set for this connection.
453     * @return the HTTP request method
454     * @see #create(URL, String)
455     * @since 9172
456     */
457    public String getRequestMethod() {
458        return requestMethod;
459    }
460
461    /**
462     * Returns the set value for the given {@code header}.
463     * @param header HTTP header name
464     * @return HTTP header value
465     * @since 9172
466     */
467    public String getRequestHeader(String header) {
468        return headers.get(header);
469    }
470
471    /**
472     * Sets whether not to set header {@code Cache-Control=no-cache}
473     *
474     * @param useCache whether not to set header {@code Cache-Control=no-cache}
475     * @return {@code this}
476     * @see HttpURLConnection#setUseCaches(boolean)
477     */
478    public HttpClient useCache(boolean useCache) {
479        this.useCache = useCache;
480        return this;
481    }
482
483    /**
484     * Sets whether not to set header {@code Connection=close}
485     * <p>
486     * This might fix #7640, see
487     * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>.
488     *
489     * @param keepAlive whether not to set header {@code Connection=close}
490     * @return {@code this}
491     */
492    public HttpClient keepAlive(boolean keepAlive) {
493        return setHeader("Connection", keepAlive ? null : "close");
494    }
495
496    /**
497     * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced
498     * by this URLConnection. If the timeout expires before the connection can be established, a
499     * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
500     * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds
501     * @return {@code this}
502     * @see HttpURLConnection#setConnectTimeout(int)
503     */
504    public HttpClient setConnectTimeout(int connectTimeout) {
505        this.connectTimeout = connectTimeout;
506        return this;
507    }
508
509    /**
510     * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from
511     * input stream when a connection is established to a resource. If the timeout expires before there is data available for
512     * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
513     * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds
514     * @return {@code this}
515     * @see HttpURLConnection#setReadTimeout(int)
516     */
517    public HttpClient setReadTimeout(int readTimeout) {
518        this.readTimeout = readTimeout;
519        return this;
520    }
521
522    /**
523     * Sets the {@code Accept} header.
524     * @param accept header value
525     *
526     * @return {@code this}
527     */
528    public HttpClient setAccept(String accept) {
529        return setHeader("Accept", accept);
530    }
531
532    /**
533     * Sets the request body for {@code PUT}/{@code POST} requests.
534     * @param requestBody request body
535     *
536     * @return {@code this}
537     */
538    public HttpClient setRequestBody(byte[] requestBody) {
539        this.requestBody = requestBody;
540        return this;
541    }
542
543    /**
544     * Sets the {@code If-Modified-Since} header.
545     * @param ifModifiedSince header value
546     *
547     * @return {@code this}
548     */
549    public HttpClient setIfModifiedSince(long ifModifiedSince) {
550        this.ifModifiedSince = ifModifiedSince;
551        return this;
552    }
553
554    /**
555     * Sets the maximum number of redirections to follow.
556     *
557     * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e.,
558     * to not throw an {@link IOException} in {@link #connect()}.
559     * @param maxRedirects header value
560     *
561     * @return {@code this}
562     */
563    public HttpClient setMaxRedirects(int maxRedirects) {
564        this.maxRedirects = maxRedirects;
565        return this;
566    }
567
568    /**
569     * Sets an arbitrary HTTP header.
570     * @param key header name
571     * @param value header value
572     *
573     * @return {@code this}
574     */
575    public HttpClient setHeader(String key, String value) {
576        this.headers.put(key, value);
577        return this;
578    }
579
580    /**
581     * Sets arbitrary HTTP headers.
582     * @param headers HTTP headers
583     *
584     * @return {@code this}
585     */
586    public HttpClient setHeaders(Map<String, String> headers) {
587        this.headers.putAll(headers);
588        return this;
589    }
590
591    /**
592     * Sets a reason to show on console. Can be {@code null} if no reason is given.
593     * @param reasonForRequest Reason to show
594     * @return {@code this}
595     * @since 9172
596     */
597    public HttpClient setReasonForRequest(String reasonForRequest) {
598        this.reasonForRequest = reasonForRequest;
599        return this;
600    }
601
602    /**
603     * Sets whether the progress monitor task will be finished when the output stream is closed. This is {@code true} by default.
604     * @param finishOnCloseOutput whether the progress monitor task will be finished when the output stream is closed
605     * @return {@code this}
606     * @since 10302
607     */
608    public HttpClient setFinishOnCloseOutput(boolean finishOnCloseOutput) {
609        this.finishOnCloseOutput = finishOnCloseOutput;
610        return this;
611    }
612
613    private static boolean isRedirect(final int statusCode) {
614        switch (statusCode) {
615            case HttpURLConnection.HTTP_MOVED_PERM: // 301
616            case HttpURLConnection.HTTP_MOVED_TEMP: // 302
617            case HttpURLConnection.HTTP_SEE_OTHER: // 303
618            case 307: // TEMPORARY_REDIRECT:
619            case 308: // PERMANENT_REDIRECT:
620                return true;
621            default:
622                return false;
623        }
624    }
625
626    /**
627     * @see HttpURLConnection#disconnect()
628     * @since 9309
629     */
630    public void disconnect() {
631        HttpClient.disconnect(connection);
632    }
633
634    private static void disconnect(final HttpURLConnection connection) {
635        // Fix upload aborts - see #263
636        connection.setConnectTimeout(100);
637        connection.setReadTimeout(100);
638        try {
639            Thread.sleep(100);
640        } catch (InterruptedException ex) {
641            Main.warn("InterruptedException in " + HttpClient.class + " during cancel");
642        }
643        connection.disconnect();
644    }
645}