001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.imagery;
003
004import java.awt.Graphics2D;
005import java.awt.image.BufferedImage;
006import java.io.BufferedOutputStream;
007import java.io.File;
008import java.io.FileInputStream;
009import java.io.FileNotFoundException;
010import java.io.FileOutputStream;
011import java.io.IOException;
012import java.io.InputStream;
013import java.io.OutputStream;
014import java.lang.ref.SoftReference;
015import java.net.URLConnection;
016import java.util.ArrayList;
017import java.util.Calendar;
018import java.util.Collections;
019import java.util.Comparator;
020import java.util.HashMap;
021import java.util.HashSet;
022import java.util.Iterator;
023import java.util.List;
024import java.util.Map;
025import java.util.Properties;
026import java.util.Set;
027
028import javax.imageio.ImageIO;
029import javax.xml.bind.JAXBContext;
030import javax.xml.bind.Marshaller;
031import javax.xml.bind.Unmarshaller;
032
033import org.openstreetmap.josm.Main;
034import org.openstreetmap.josm.data.ProjectionBounds;
035import org.openstreetmap.josm.data.SystemOfMeasurement;
036import org.openstreetmap.josm.data.coor.EastNorth;
037import org.openstreetmap.josm.data.coor.LatLon;
038import org.openstreetmap.josm.data.imagery.types.EntryType;
039import org.openstreetmap.josm.data.imagery.types.ProjectionType;
040import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
041import org.openstreetmap.josm.data.preferences.StringProperty;
042import org.openstreetmap.josm.data.projection.Projection;
043import org.openstreetmap.josm.gui.layer.WMSLayer;
044import org.openstreetmap.josm.tools.ImageProvider;
045import org.openstreetmap.josm.tools.Utils;
046import org.openstreetmap.josm.tools.date.DateUtils;
047
048public class WmsCache {
049    //TODO Property for maximum cache size
050    //TODO Property for maximum age of tile, automatically remove old tiles
051    //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
052    //TODO Do loading from partial cache and downloading at the same time, don't wait for partial cache to load
053
054    private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms");
055    private static final String INDEX_FILENAME = "index.xml";
056    private static final String LAYERS_INDEX_FILENAME = "layers.properties";
057
058    private static class CacheEntry {
059        final double pixelPerDegree;
060        final double east;
061        final double north;
062        final ProjectionBounds bounds;
063        final String filename;
064
065        long lastUsed;
066        long lastModified;
067
068        CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) {
069            this.pixelPerDegree = pixelPerDegree;
070            this.east = east;
071            this.north = north;
072            this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
073            this.filename = filename;
074        }
075
076        @Override
077        public String toString() {
078            return "CacheEntry [pixelPerDegree=" + pixelPerDegree + ", east=" + east + ", north=" + north + ", bounds="
079                    + bounds + ", filename=" + filename + ", lastUsed=" + lastUsed + ", lastModified=" + lastModified
080                    + "]";
081        }
082    }
083
084    private static class ProjectionEntries {
085        final String projection;
086        final String cacheDirectory;
087        final List<CacheEntry> entries = new ArrayList<>();
088
089        ProjectionEntries(String projection, String cacheDirectory) {
090            this.projection = projection;
091            this.cacheDirectory = cacheDirectory;
092        }
093    }
094
095    private final Map<String, ProjectionEntries> entries = new HashMap<>();
096    private final File cacheDir;
097    private final int tileSize; // Should be always 500
098    private int totalFileSize;
099    private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
100    // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found
101    private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<>();
102    private Set<ProjectionBounds> areaToCache;
103
104    protected String cacheDirPath() {
105        String cPath = PROP_CACHE_PATH.get();
106        if (!(new File(cPath).isAbsolute())) {
107            cPath = Main.pref.getCacheDirectory() + File.separator + cPath;
108        }
109        return cPath;
110    }
111
112    public WmsCache(String url, int tileSize) {
113        File globalCacheDir = new File(cacheDirPath());
114        globalCacheDir.mkdirs();
115        cacheDir = new File(globalCacheDir, getCacheDirectory(url));
116        cacheDir.mkdirs();
117        this.tileSize = tileSize;
118    }
119
120    private String getCacheDirectory(String url) {
121        String cacheDirName = null;
122        Properties layersIndex = new Properties();
123        File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME);
124        try (InputStream fis = new FileInputStream(layerIndexFile)) {
125            layersIndex.load(fis);
126        } catch (FileNotFoundException e) {
127            Main.error("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
128        } catch (IOException e) {
129            Main.error("Unable to load layers index for wms cache");
130            Main.error(e);
131        }
132
133        for (Object propKey: layersIndex.keySet()) {
134            String s = (String)propKey;
135            if (url.equals(layersIndex.getProperty(s))) {
136                cacheDirName = s;
137                break;
138            }
139        }
140
141        if (cacheDirName == null) {
142            int counter = 0;
143            while (true) {
144                counter++;
145                if (!layersIndex.keySet().contains(String.valueOf(counter))) {
146                    break;
147                }
148            }
149            cacheDirName = String.valueOf(counter);
150            layersIndex.setProperty(cacheDirName, url);
151            try (OutputStream fos = new FileOutputStream(layerIndexFile)) {
152                layersIndex.store(fos, "");
153            } catch (IOException e) {
154                Main.error("Unable to save layer index for wms cache");
155                Main.error(e);
156            }
157        }
158
159        return cacheDirName;
160    }
161
162    private ProjectionEntries getProjectionEntries(Projection projection) {
163        return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
164    }
165
166    private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
167        ProjectionEntries result = entries.get(projection);
168        if (result == null) {
169            result = new ProjectionEntries(projection, cacheDirectory);
170            entries.put(projection, result);
171        }
172
173        return result;
174    }
175
176    public synchronized void loadIndex() {
177        File indexFile = new File(cacheDir, INDEX_FILENAME);
178        try {
179            JAXBContext context = JAXBContext.newInstance(
180                    WmsCacheType.class.getPackage().getName(),
181                    WmsCacheType.class.getClassLoader());
182            Unmarshaller unmarshaller = context.createUnmarshaller();
183            WmsCacheType cacheEntries;
184            try (InputStream is = new FileInputStream(indexFile)) {
185                cacheEntries = (WmsCacheType)unmarshaller.unmarshal(is);
186            }
187            totalFileSize = cacheEntries.getTotalFileSize();
188            if (cacheEntries.getTileSize() != tileSize) {
189                Main.info("Cache created with different tileSize, cache will be discarded");
190                return;
191            }
192            for (ProjectionType projectionType: cacheEntries.getProjection()) {
193                ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
194                for (EntryType entry: projectionType.getEntry()) {
195                    CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename());
196                    ce.lastUsed = entry.getLastUsed().getTimeInMillis();
197                    ce.lastModified = entry.getLastModified().getTimeInMillis();
198                    projection.entries.add(ce);
199                }
200            }
201        } catch (Exception e) {
202            if (indexFile.exists()) {
203                Main.error(e);
204                Main.info("Unable to load index for wms-cache, new file will be created");
205            } else {
206                Main.info("Index for wms-cache doesn't exist, new file will be created");
207            }
208        }
209
210        removeNonReferencedFiles();
211    }
212
213    private void removeNonReferencedFiles() {
214
215        Set<String> usedProjections = new HashSet<>();
216
217        for (ProjectionEntries projectionEntries: entries.values()) {
218
219            usedProjections.add(projectionEntries.cacheDirectory);
220
221            File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
222            if (projectionDir.exists()) {
223                Set<String> referencedFiles = new HashSet<>();
224
225                for (CacheEntry ce: projectionEntries.entries) {
226                    referencedFiles.add(ce.filename);
227                }
228
229                for (File file: projectionDir.listFiles()) {
230                    if (!referencedFiles.contains(file.getName())) {
231                        file.delete();
232                    }
233                }
234            }
235        }
236
237        for (File projectionDir: cacheDir.listFiles()) {
238            if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
239                Utils.deleteDirectory(projectionDir);
240            }
241        }
242    }
243
244    private int calculateTotalFileSize() {
245        int result = 0;
246        for (ProjectionEntries projectionEntries: entries.values()) {
247            Iterator<CacheEntry> it = projectionEntries.entries.iterator();
248            while (it.hasNext()) {
249                CacheEntry entry = it.next();
250                File imageFile = getImageFile(projectionEntries, entry);
251                if (!imageFile.exists()) {
252                    it.remove();
253                } else {
254                    result += imageFile.length();
255                }
256            }
257        }
258        return result;
259    }
260
261    public synchronized void saveIndex() {
262        WmsCacheType index = new WmsCacheType();
263
264        if (totalFileSizeDirty) {
265            totalFileSize = calculateTotalFileSize();
266        }
267
268        index.setTileSize(tileSize);
269        index.setTotalFileSize(totalFileSize);
270        for (ProjectionEntries projectionEntries: entries.values()) {
271            if (!projectionEntries.entries.isEmpty()) {
272                ProjectionType projectionType = new ProjectionType();
273                projectionType.setName(projectionEntries.projection);
274                projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
275                index.getProjection().add(projectionType);
276                for (CacheEntry ce: projectionEntries.entries) {
277                    EntryType entry = new EntryType();
278                    entry.setPixelPerDegree(ce.pixelPerDegree);
279                    entry.setEast(ce.east);
280                    entry.setNorth(ce.north);
281                    Calendar c = Calendar.getInstance();
282                    c.setTimeInMillis(ce.lastUsed);
283                    entry.setLastUsed(c);
284                    c = Calendar.getInstance();
285                    c.setTimeInMillis(ce.lastModified);
286                    entry.setLastModified(c);
287                    entry.setFilename(ce.filename);
288                    projectionType.getEntry().add(entry);
289                }
290            }
291        }
292        try {
293            JAXBContext context = JAXBContext.newInstance(
294                    WmsCacheType.class.getPackage().getName(),
295                    WmsCacheType.class.getClassLoader());
296            Marshaller marshaller = context.createMarshaller();
297            try (OutputStream fos = new FileOutputStream(new File(cacheDir, INDEX_FILENAME))) {
298                marshaller.marshal(index, fos);
299            }
300        } catch (Exception e) {
301            Main.error("Failed to save wms-cache file");
302            Main.error(e);
303        }
304    }
305
306    private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
307        return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
308    }
309
310    private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry, boolean enforceTransparency) throws IOException {
311        synchronized (this) {
312            entry.lastUsed = System.currentTimeMillis();
313
314            SoftReference<BufferedImage> memCache = memoryCache.get(entry);
315            if (memCache != null) {
316                BufferedImage result = memCache.get();
317                if (result != null) {
318                    if (enforceTransparency == ImageProvider.isTransparencyForced(result)) {
319                        return result;
320                    } else if (Main.isDebugEnabled()) {
321                        Main.debug("Skipping "+entry+" from memory cache (transparency enforcement)");
322                    }
323                }
324            }
325        }
326
327        try {
328            // Reading can't be in synchronized section, it's too slow
329            BufferedImage result = ImageProvider.read(getImageFile(projectionEntries, entry), true, enforceTransparency);
330            synchronized (this) {
331                if (result == null) {
332                    projectionEntries.entries.remove(entry);
333                    totalFileSizeDirty = true;
334                }
335                return result;
336            }
337        } catch (IOException e) {
338            synchronized (this) {
339                projectionEntries.entries.remove(entry);
340                totalFileSizeDirty = true;
341                throw e;
342            }
343        }
344    }
345
346    private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
347        for (CacheEntry entry: projectionEntries.entries) {
348            if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north)
349                return entry;
350        }
351        return null;
352    }
353
354    public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
355        ProjectionEntries projectionEntries = getProjectionEntries(projection);
356        CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
357        return (entry != null);
358    }
359
360    public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
361        CacheEntry entry = null;
362        ProjectionEntries projectionEntries = null;
363        synchronized (this) {
364            projectionEntries = getProjectionEntries(projection);
365            entry = findEntry(projectionEntries, pixelPerDegree, east, north);
366        }
367        if (entry != null) {
368            try {
369                return loadImage(projectionEntries, entry, WMSLayer.PROP_ALPHA_CHANNEL.get());
370            } catch (IOException e) {
371                Main.error("Unable to load file from wms cache");
372                Main.error(e);
373                return null;
374            }
375        }
376        return null;
377    }
378
379    public BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
380        ProjectionEntries projectionEntries;
381        List<CacheEntry> matches;
382        synchronized (this) {
383            matches = new ArrayList<>();
384
385            double minPPD = pixelPerDegree / 5;
386            double maxPPD = pixelPerDegree * 5;
387            projectionEntries = getProjectionEntries(projection);
388
389            double size2 = tileSize / pixelPerDegree;
390            double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly
391            ProjectionBounds bounds = new ProjectionBounds(east + border, north + border,
392                    east + size2 - border, north + size2 - border);
393
394            //TODO Do not load tile if it is completely overlapped by other tile with better ppd
395            for (CacheEntry entry: projectionEntries.entries) {
396                if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
397                    entry.lastUsed = System.currentTimeMillis();
398                    matches.add(entry);
399                }
400            }
401
402            if (matches.isEmpty())
403                return null;
404
405            Collections.sort(matches, new Comparator<CacheEntry>() {
406                @Override
407                public int compare(CacheEntry o1, CacheEntry o2) {
408                    return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
409                }
410            });
411        }
412
413        // Use alpha layer only when enabled on wms layer
414        boolean alpha = WMSLayer.PROP_ALPHA_CHANNEL.get();
415        BufferedImage result = new BufferedImage(tileSize, tileSize,
416                alpha ? BufferedImage.TYPE_4BYTE_ABGR : BufferedImage.TYPE_3BYTE_BGR);
417        Graphics2D g = result.createGraphics();
418
419        boolean drawAtLeastOnce = false;
420        Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<>();
421        for (CacheEntry ce: matches) {
422            BufferedImage img;
423            try {
424                // Enforce transparency only when alpha enabled on wms layer too
425                img = loadImage(projectionEntries, ce, alpha);
426                localCache.put(ce, new SoftReference<>(img));
427            } catch (IOException e) {
428                continue;
429            }
430
431            drawAtLeastOnce = true;
432
433            int xDiff = (int)((ce.east - east) * pixelPerDegree);
434            int yDiff = (int)((ce.north - north) * pixelPerDegree);
435            int size = (int)(pixelPerDegree / ce.pixelPerDegree  * tileSize);
436
437            int x = xDiff;
438            int y = -size + tileSize - yDiff;
439
440            g.drawImage(img, x, y, size, size, null);
441        }
442
443        if (drawAtLeastOnce) {
444            synchronized (this) {
445                memoryCache.putAll(localCache);
446            }
447            return result;
448        } else
449            return null;
450    }
451
452    private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) {
453        LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
454        LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
455        LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
456
457        double deltaLat = Math.abs(ll3.lat() - ll1.lat());
458        double deltaLon = Math.abs(ll3.lon() - ll1.lon());
459        int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
460        int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
461
462        String zoom = SystemOfMeasurement.METRIC.getDistText(ll1.greatCircleDistance(ll2));
463        String extension = "dat";
464        if (mimeType != null) {
465            switch(mimeType) {
466            case "image/jpeg":
467            case "image/jpg":
468                extension = "jpg";
469                break;
470            case "image/png":
471                extension = "png";
472                break;
473            case "image/gif":
474                extension = "gif";
475                break;
476            default:
477                Main.warn("Unrecognized MIME type: "+mimeType);
478            }
479        }
480
481        int counter = 0;
482        FILENAME_LOOP:
483            while (true) {
484                String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
485                for (CacheEntry entry: projectionEntries.entries) {
486                    if (entry.filename.equals(result)) {
487                        counter++;
488                        continue FILENAME_LOOP;
489                    }
490                }
491                return result;
492            }
493    }
494
495    /**
496     *
497     * @param img Used only when overlapping is used, when not used, used raw from imageData
498     * @param imageData
499     * @param projection
500     * @param pixelPerDegree
501     * @param east
502     * @param north
503     * @throws IOException
504     */
505    public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException {
506        ProjectionEntries projectionEntries = getProjectionEntries(projection);
507        CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
508        File imageFile;
509        if (entry == null) {
510
511            String mimeType;
512            if (img != null) {
513                mimeType = "image/png";
514            } else {
515                mimeType = URLConnection.guessContentTypeFromStream(imageData);
516            }
517            entry = new CacheEntry(pixelPerDegree, east, north, tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType));
518            entry.lastUsed = System.currentTimeMillis();
519            entry.lastModified = entry.lastUsed;
520            projectionEntries.entries.add(entry);
521            imageFile = getImageFile(projectionEntries, entry);
522        } else {
523            imageFile = getImageFile(projectionEntries, entry);
524            totalFileSize -= imageFile.length();
525        }
526
527        imageFile.getParentFile().mkdirs();
528
529        if (img != null) {
530            BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
531            copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
532            ImageIO.write(copy, "png", imageFile);
533            totalFileSize += imageFile.length();
534        } else {
535            try (OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile))) {
536                totalFileSize += Utils.copyStream(imageData, os);
537            }
538        }
539    }
540
541    public synchronized void cleanSmallFiles(int size) {
542        for (ProjectionEntries projectionEntries: entries.values()) {
543            Iterator<CacheEntry> it = projectionEntries.entries.iterator();
544            while (it.hasNext()) {
545                File file = getImageFile(projectionEntries, it.next());
546                long length = file.length();
547                if (length <= size) {
548                    if (length == 0) {
549                        totalFileSizeDirty = true; // File probably doesn't exist
550                    }
551                    totalFileSize -= size;
552                    file.delete();
553                    it.remove();
554                }
555            }
556        }
557    }
558
559    public static String printDate(Calendar c) {
560        return DateUtils.newIsoDateFormat().format(c.getTime());
561    }
562
563    private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
564        for (ProjectionBounds b: areaToCache) {
565            if (cacheEntry.bounds.intersects(b))
566                return true;
567        }
568        return false;
569    }
570
571    public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) {
572        this.areaToCache = areaToCache;
573        Iterator<CacheEntry> it = memoryCache.keySet().iterator();
574        while (it.hasNext()) {
575            if (!isInsideAreaToCache(it.next())) {
576                it.remove();
577            }
578        }
579    }
580}