001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006
007import java.awt.GridBagConstraints;
008import java.awt.event.ActionEvent;
009import java.awt.event.ActionListener;
010import java.io.BufferedReader;
011import java.io.IOException;
012import java.io.InputStream;
013import java.text.MessageFormat;
014import java.util.ArrayList;
015import java.util.Arrays;
016import java.util.Collection;
017import java.util.HashMap;
018import java.util.List;
019import java.util.Locale;
020import java.util.Map;
021import java.util.Map.Entry;
022import java.util.Set;
023import java.util.regex.Matcher;
024import java.util.regex.Pattern;
025import java.util.regex.PatternSyntaxException;
026
027import javax.swing.JCheckBox;
028import javax.swing.JLabel;
029import javax.swing.JPanel;
030
031import org.openstreetmap.josm.Main;
032import org.openstreetmap.josm.command.ChangePropertyCommand;
033import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
034import org.openstreetmap.josm.command.Command;
035import org.openstreetmap.josm.command.SequenceCommand;
036import org.openstreetmap.josm.data.osm.OsmPrimitive;
037import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
038import org.openstreetmap.josm.data.osm.OsmUtils;
039import org.openstreetmap.josm.data.osm.Tag;
040import org.openstreetmap.josm.data.validation.FixableTestError;
041import org.openstreetmap.josm.data.validation.Severity;
042import org.openstreetmap.josm.data.validation.Test.TagTest;
043import org.openstreetmap.josm.data.validation.TestError;
044import org.openstreetmap.josm.data.validation.util.Entities;
045import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference;
046import org.openstreetmap.josm.gui.progress.ProgressMonitor;
047import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
048import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
049import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
050import org.openstreetmap.josm.gui.tagging.presets.items.Check;
051import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
052import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
053import org.openstreetmap.josm.gui.widgets.EditableList;
054import org.openstreetmap.josm.io.CachedFile;
055import org.openstreetmap.josm.io.UTFInputStreamReader;
056import org.openstreetmap.josm.tools.GBC;
057import org.openstreetmap.josm.tools.MultiMap;
058import org.openstreetmap.josm.tools.Utils;
059
060/**
061 * Check for misspelled or wrong tags
062 *
063 * @author frsantos
064 * @since 3669
065 */
066public class TagChecker extends TagTest {
067
068    /** The config file of ignored tags */
069    public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
070    /** The config file of dictionary words */
071    public static final String SPELL_FILE = "resource://data/validator/words.cfg";
072
073    /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
074    private static final Map<String, String> harmonizedKeys = new HashMap<>();
075    /** The spell check preset values */
076    private static volatile MultiMap<String, String> presetsValueData;
077    /** The TagChecker data */
078    private static final List<CheckerData> checkerData = new ArrayList<>();
079    private static final List<String> ignoreDataStartsWith = new ArrayList<>();
080    private static final List<String> ignoreDataEquals = new ArrayList<>();
081    private static final List<String> ignoreDataEndsWith = new ArrayList<>();
082    private static final List<Tag> ignoreDataTag = new ArrayList<>();
083
084    /** The preferences prefix */
085    protected static final String PREFIX = ValidatorPreference.PREFIX + "." + TagChecker.class.getSimpleName();
086
087    public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
088    public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
089    public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
090    public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
091
092    public static final String PREF_SOURCES = PREFIX + ".source";
093
094    public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
095    public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
096    public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
097    public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
098
099    protected boolean checkKeys;
100    protected boolean checkValues;
101    protected boolean checkComplex;
102    protected boolean checkFixmes;
103
104    protected JCheckBox prefCheckKeys;
105    protected JCheckBox prefCheckValues;
106    protected JCheckBox prefCheckComplex;
107    protected JCheckBox prefCheckFixmes;
108    protected JCheckBox prefCheckPaint;
109
110    protected JCheckBox prefCheckKeysBeforeUpload;
111    protected JCheckBox prefCheckValuesBeforeUpload;
112    protected JCheckBox prefCheckComplexBeforeUpload;
113    protected JCheckBox prefCheckFixmesBeforeUpload;
114    protected JCheckBox prefCheckPaintBeforeUpload;
115
116    protected static final int EMPTY_VALUES      = 1200;
117    protected static final int INVALID_KEY       = 1201;
118    protected static final int INVALID_VALUE     = 1202;
119    protected static final int FIXME             = 1203;
120    protected static final int INVALID_SPACE     = 1204;
121    protected static final int INVALID_KEY_SPACE = 1205;
122    protected static final int INVALID_HTML      = 1206; /* 1207 was PAINT */
123    protected static final int LONG_VALUE        = 1208;
124    protected static final int LONG_KEY          = 1209;
125    protected static final int LOW_CHAR_VALUE    = 1210;
126    protected static final int LOW_CHAR_KEY      = 1211;
127    protected static final int MISSPELLED_VALUE  = 1212;
128    protected static final int MISSPELLED_KEY    = 1213;
129    protected static final int MULTIPLE_SPACES   = 1214;
130    /** 1250 and up is used by tagcheck */
131
132    protected EditableList sourcesList;
133
134    private static final List<String> DEFAULT_SOURCES = Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE);
135
136    /**
137     * Constructor
138     */
139    public TagChecker() {
140        super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
141    }
142
143    @Override
144    public void initialize() throws IOException {
145        initializeData();
146        initializePresets();
147    }
148
149    /**
150     * Reads the spellcheck file into a HashMap.
151     * The data file is a list of words, beginning with +/-. If it starts with +,
152     * the word is valid, but if it starts with -, the word should be replaced
153     * by the nearest + word before this.
154     *
155     * @throws IOException if any I/O error occurs
156     */
157    private static void initializeData() throws IOException {
158        checkerData.clear();
159        ignoreDataStartsWith.clear();
160        ignoreDataEquals.clear();
161        ignoreDataEndsWith.clear();
162        ignoreDataTag.clear();
163        harmonizedKeys.clear();
164
165        StringBuilder errorSources = new StringBuilder();
166        for (String source : Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES)) {
167            try (
168                CachedFile cf = new CachedFile(source);
169                InputStream s = cf.getInputStream();
170                BufferedReader reader = new BufferedReader(UTFInputStreamReader.create(s));
171            ) {
172                String okValue = null;
173                boolean tagcheckerfile = false;
174                boolean ignorefile = false;
175                boolean isFirstLine = true;
176                String line;
177                while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) {
178                    if (line.startsWith("#")) {
179                        if (line.startsWith("# JOSM TagChecker")) {
180                            tagcheckerfile = true;
181                            if (!DEFAULT_SOURCES.contains(source)) {
182                                Main.info(tr("Adding {0} to tag checker", source));
183                            }
184                        } else
185                        if (line.startsWith("# JOSM IgnoreTags")) {
186                            ignorefile = true;
187                            if (!DEFAULT_SOURCES.contains(source)) {
188                                Main.info(tr("Adding {0} to ignore tags", source));
189                            }
190                        }
191                    } else if (ignorefile) {
192                        line = line.trim();
193                        if (line.length() < 4) {
194                            continue;
195                        }
196
197                        String key = line.substring(0, 2);
198                        line = line.substring(2);
199
200                        switch (key) {
201                        case "S:":
202                            ignoreDataStartsWith.add(line);
203                            break;
204                        case "E:":
205                            ignoreDataEquals.add(line);
206                            break;
207                        case "F:":
208                            ignoreDataEndsWith.add(line);
209                            break;
210                        case "K:":
211                            ignoreDataTag.add(Tag.ofString(line));
212                        }
213                    } else if (tagcheckerfile) {
214                        if (!line.isEmpty()) {
215                            CheckerData d = new CheckerData();
216                            String err = d.getData(line);
217
218                            if (err == null) {
219                                checkerData.add(d);
220                            } else {
221                                Main.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
222                            }
223                        }
224                    } else if (line.charAt(0) == '+') {
225                        okValue = line.substring(1);
226                    } else if (line.charAt(0) == '-' && okValue != null) {
227                        harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue);
228                    } else {
229                        Main.error(tr("Invalid spellcheck line: {0}", line));
230                    }
231                    if (isFirstLine) {
232                        isFirstLine = false;
233                        if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
234                            Main.info(tr("Adding {0} to spellchecker", source));
235                        }
236                    }
237                }
238            } catch (IOException e) {
239                errorSources.append(source).append('\n');
240            }
241        }
242
243        if (errorSources.length() > 0)
244            throw new IOException(tr("Could not access data file(s):\n{0}", errorSources));
245    }
246
247    /**
248     * Reads the presets data.
249     *
250     */
251    public static void initializePresets() {
252
253        if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true))
254            return;
255
256        Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
257        if (!presets.isEmpty()) {
258            presetsValueData = new MultiMap<>();
259            for (String a : OsmPrimitive.getUninterestingKeys()) {
260                presetsValueData.putVoid(a);
261            }
262            // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
263            for (String a : Main.pref.getCollection(ValidatorPreference.PREFIX + ".knownkeys",
264                    Arrays.asList(new String[]{"is_in", "int_ref", "fixme", "population"}))) {
265                presetsValueData.putVoid(a);
266            }
267            for (TaggingPreset p : presets) {
268                for (TaggingPresetItem i : p.data) {
269                    if (i instanceof KeyedItem) {
270                        addPresetValue(p, (KeyedItem) i);
271                    } else if (i instanceof CheckGroup) {
272                        for (Check c : ((CheckGroup) i).checks) {
273                            addPresetValue(p, c);
274                        }
275                    }
276                }
277            }
278        }
279    }
280
281    private static void addPresetValue(TaggingPreset p, KeyedItem ky) {
282        Collection<String> values = ky.getValues();
283        if (ky.key != null && values != null) {
284            try {
285                presetsValueData.putAll(ky.key, values);
286                harmonizedKeys.put(harmonizeKey(ky.key), ky.key);
287            } catch (NullPointerException e) {
288                Main.error(p+": Unable to initialize "+ky);
289            }
290        }
291    }
292
293    /**
294     * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
295     * @param s string to check
296     * @return {@code true} if {@code s} contains characters with code below 0x20
297     */
298    private static boolean containsLow(String s) {
299        if (s == null)
300            return false;
301        for (int i = 0; i < s.length(); i++) {
302            if (s.charAt(i) < 0x20)
303                return true;
304        }
305        return false;
306    }
307
308    /**
309     * Determines if the given key is in internal presets.
310     * @param key key
311     * @return {@code true} if the given key is in internal presets
312     * @since 9023
313     */
314    public static boolean isKeyInPresets(String key) {
315        return presetsValueData.get(key) != null;
316    }
317
318    /**
319     * Determines if the given tag is in internal presets.
320     * @param key key
321     * @param value value
322     * @return {@code true} if the given tag is in internal presets
323     * @since 9023
324     */
325    public static boolean isTagInPresets(String key, String value) {
326        final Set<String> values = presetsValueData.get(key);
327        return values != null && (values.isEmpty() || values.contains(value));
328    }
329
330    /**
331     * Returns the list of ignored tags.
332     * @return the list of ignored tags
333     * @since 9023
334     */
335    public static List<Tag> getIgnoredTags() {
336        return new ArrayList<>(ignoreDataTag);
337    }
338
339    /**
340     * Determines if the given tag is ignored for checks "key/tag not in presets".
341     * @param key key
342     * @param value value
343     * @return {@code true} if the given tag is ignored
344     * @since 9023
345     */
346    public static boolean isTagIgnored(String key, String value) {
347        boolean tagInPresets = isTagInPresets(key, value);
348        boolean ignore = false;
349
350        for (String a : ignoreDataStartsWith) {
351            if (key.startsWith(a)) {
352                ignore = true;
353            }
354        }
355        for (String a : ignoreDataEquals) {
356            if (key.equals(a)) {
357                ignore = true;
358            }
359        }
360        for (String a : ignoreDataEndsWith) {
361            if (key.endsWith(a)) {
362                ignore = true;
363            }
364        }
365
366        if (!tagInPresets) {
367            for (Tag a : ignoreDataTag) {
368                if (key.equals(a.getKey()) && value.equals(a.getValue())) {
369                    ignore = true;
370                }
371            }
372        }
373        return ignore;
374    }
375
376    /**
377     * Checks the primitive tags
378     * @param p The primitive to check
379     */
380    @Override
381    public void check(OsmPrimitive p) {
382        // Just a collection to know if a primitive has been already marked with error
383        MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
384
385        if (checkComplex) {
386            Map<String, String> keys = p.getKeys();
387            for (CheckerData d : checkerData) {
388                if (d.match(p, keys)) {
389                    errors.add(new TestError(this, d.getSeverity(), tr("Suspicious tag/value combinations"),
390                            d.getDescription(), d.getDescriptionOrig(), d.getCode(), p));
391                    withErrors.put(p, "TC");
392                }
393            }
394        }
395
396        for (Entry<String, String> prop : p.getKeys().entrySet()) {
397            String s = marktr("Key ''{0}'' invalid.");
398            String key = prop.getKey();
399            String value = prop.getValue();
400            if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
401                errors.add(new TestError(this, Severity.WARNING, tr("Tag value contains character with code less than 0x20"),
402                        tr(s, key), MessageFormat.format(s, key), LOW_CHAR_VALUE, p));
403                withErrors.put(p, "ICV");
404            }
405            if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
406                errors.add(new TestError(this, Severity.WARNING, tr("Tag key contains character with code less than 0x20"),
407                        tr(s, key), MessageFormat.format(s, key), LOW_CHAR_KEY, p));
408                withErrors.put(p, "ICK");
409            }
410            if (checkValues && (value != null && value.length() > 255) && !withErrors.contains(p, "LV")) {
411                errors.add(new TestError(this, Severity.ERROR, tr("Tag value longer than allowed"),
412                        tr(s, key), MessageFormat.format(s, key), LONG_VALUE, p));
413                withErrors.put(p, "LV");
414            }
415            if (checkKeys && (key != null && key.length() > 255) && !withErrors.contains(p, "LK")) {
416                errors.add(new TestError(this, Severity.ERROR, tr("Tag key longer than allowed"),
417                        tr(s, key), MessageFormat.format(s, key), LONG_KEY, p));
418                withErrors.put(p, "LK");
419            }
420            if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
421                errors.add(new TestError(this, Severity.WARNING, tr("Tags with empty values"),
422                        tr(s, key), MessageFormat.format(s, key), EMPTY_VALUES, p));
423                withErrors.put(p, "EV");
424            }
425            if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
426                errors.add(new TestError(this, Severity.WARNING, tr("Invalid white space in property key"),
427                        tr(s, key), MessageFormat.format(s, key), INVALID_KEY_SPACE, p));
428                withErrors.put(p, "IPK");
429            }
430            if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
431                errors.add(new TestError(this, Severity.WARNING, tr("Property values start or end with white space"),
432                        tr(s, key), MessageFormat.format(s, key), INVALID_SPACE, p));
433                withErrors.put(p, "SPACE");
434            }
435            if (checkValues && value != null && value.contains("  ") && !withErrors.contains(p, "SPACE")) {
436                errors.add(new TestError(this, Severity.WARNING, tr("Property values contain multiple white spaces"),
437                        tr(s, key), MessageFormat.format(s, key), MULTIPLE_SPACES, p));
438                withErrors.put(p, "SPACE");
439            }
440            if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
441                errors.add(new TestError(this, Severity.OTHER, tr("Property values contain HTML entity"),
442                        tr(s, key), MessageFormat.format(s, key), INVALID_HTML, p));
443                withErrors.put(p, "HTML");
444            }
445            if (checkValues && key != null && value != null && !value.isEmpty() && presetsValueData != null) {
446                if (!isTagIgnored(key, value)) {
447                    if (!isKeyInPresets(key)) {
448                        String prettifiedKey = harmonizeKey(key);
449                        String fixedKey = harmonizedKeys.get(prettifiedKey);
450                        if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
451                            // misspelled preset key
452                            String i = marktr("Key ''{0}'' looks like ''{1}''.");
453                            final TestError error;
454                            if (p.hasKey(fixedKey)) {
455                                error = new TestError(this, Severity.WARNING, tr("Misspelled property key"),
456                                        tr(i, key, fixedKey),
457                                        MessageFormat.format(i, key, fixedKey), MISSPELLED_KEY, p);
458                            } else {
459                                error = new FixableTestError(this, Severity.WARNING, tr("Misspelled property key"),
460                                        tr(i, key, fixedKey),
461                                        MessageFormat.format(i, key, fixedKey), MISSPELLED_KEY, p,
462                                        new ChangePropertyKeyCommand(p, key, fixedKey));
463                            }
464                            errors.add(error);
465                            withErrors.put(p, "WPK");
466                        } else {
467                            String i = marktr("Key ''{0}'' not in presets.");
468                            errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property key"),
469                                    tr(i, key), MessageFormat.format(i, key), INVALID_VALUE, p));
470                            withErrors.put(p, "UPK");
471                        }
472                    } else if (!isTagInPresets(key, value)) {
473                        // try to fix common typos and check again if value is still unknown
474                        String fixedValue = harmonizeValue(prop.getValue());
475                        Map<String, String> possibleValues = getPossibleValues(presetsValueData.get(key));
476                        if (possibleValues.containsKey(fixedValue)) {
477                            fixedValue = possibleValues.get(fixedValue);
478                            // misspelled preset value
479                            String i = marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''.");
480                            errors.add(new FixableTestError(this, Severity.WARNING, tr("Misspelled property value"),
481                                    tr(i, prop.getValue(), key, fixedValue), MessageFormat.format(i, prop.getValue(), fixedValue),
482                                    MISSPELLED_VALUE, p, new ChangePropertyCommand(p, key, fixedValue)));
483                            withErrors.put(p, "WPV");
484                        } else {
485                            // unknown preset value
486                            String i = marktr("Value ''{0}'' for key ''{1}'' not in presets.");
487                            errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property value"),
488                                    tr(i, prop.getValue(), key), MessageFormat.format(i, prop.getValue(), key), INVALID_VALUE, p));
489                            withErrors.put(p, "UPV");
490                        }
491                    }
492                }
493            }
494            if (checkFixmes && key != null && value != null && !value.isEmpty()) {
495                if ((value.toLowerCase(Locale.ENGLISH).contains("fixme")
496                        || value.contains("check and delete")
497                        || key.contains("todo") || key.toLowerCase(Locale.ENGLISH).contains("fixme"))
498                        && !withErrors.contains(p, "FIXME")) {
499                    errors.add(new TestError(this, Severity.OTHER,
500                            tr("FIXMES"), FIXME, p));
501                    withErrors.put(p, "FIXME");
502                }
503            }
504        }
505    }
506
507    private static Map<String, String> getPossibleValues(Set<String> values) {
508        // generate a map with common typos
509        Map<String, String> map = new HashMap<>();
510        if (values != null) {
511            for (String value : values) {
512                map.put(value, value);
513                if (value.contains("_")) {
514                    map.put(value.replace("_", ""), value);
515                }
516            }
517        }
518        return map;
519    }
520
521    private static String harmonizeKey(String key) {
522        key = key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_');
523        return Utils.strip(key, "-_;:,");
524    }
525
526    private static String harmonizeValue(String value) {
527        value = value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_');
528        return Utils.strip(value, "-_;:,");
529    }
530
531    @Override
532    public void startTest(ProgressMonitor monitor) {
533        super.startTest(monitor);
534        checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true);
535        if (isBeforeUpload) {
536            checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
537        }
538
539        checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true);
540        if (isBeforeUpload) {
541            checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
542        }
543
544        checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true);
545        if (isBeforeUpload) {
546            checkComplex = checkComplex && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
547        }
548
549        checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true);
550        if (isBeforeUpload) {
551            checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
552        }
553    }
554
555    @Override
556    public void visit(Collection<OsmPrimitive> selection) {
557        if (checkKeys || checkValues || checkComplex || checkFixmes) {
558            super.visit(selection);
559        }
560    }
561
562    @Override
563    public void addGui(JPanel testPanel) {
564        GBC a = GBC.eol();
565        a.anchor = GridBagConstraints.EAST;
566
567        testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
568
569        prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true));
570        prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
571        testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
572
573        prefCheckKeysBeforeUpload = new JCheckBox();
574        prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
575        testPanel.add(prefCheckKeysBeforeUpload, a);
576
577        prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true));
578        prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
579        testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
580
581        prefCheckComplexBeforeUpload = new JCheckBox();
582        prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
583        testPanel.add(prefCheckComplexBeforeUpload, a);
584
585        final Collection<String> sources = Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES);
586        sourcesList = new EditableList(tr("TagChecker source"));
587        sourcesList.setItems(sources);
588        testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
589        testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
590
591        ActionListener disableCheckActionListener = new ActionListener() {
592            @Override
593            public void actionPerformed(ActionEvent e) {
594                handlePrefEnable();
595            }
596        };
597        prefCheckKeys.addActionListener(disableCheckActionListener);
598        prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
599        prefCheckComplex.addActionListener(disableCheckActionListener);
600        prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
601
602        handlePrefEnable();
603
604        prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true));
605        prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
606        testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
607
608        prefCheckValuesBeforeUpload = new JCheckBox();
609        prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
610        testPanel.add(prefCheckValuesBeforeUpload, a);
611
612        prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true));
613        prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
614        testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
615
616        prefCheckFixmesBeforeUpload = new JCheckBox();
617        prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
618        testPanel.add(prefCheckFixmesBeforeUpload, a);
619    }
620
621    public void handlePrefEnable() {
622        boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
623                || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
624        sourcesList.setEnabled(selected);
625    }
626
627    @Override
628    public boolean ok() {
629        enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
630        testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
631                || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
632
633        Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected());
634        Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
635        Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
636        Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
637        Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
638        Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
639        Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
640        Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
641        return Main.pref.putCollection(PREF_SOURCES, sourcesList.getItems());
642    }
643
644    @Override
645    public Command fixError(TestError testError) {
646        List<Command> commands = new ArrayList<>(50);
647
648        if (testError instanceof FixableTestError) {
649            commands.add(testError.getFix());
650        } else {
651            Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
652            for (OsmPrimitive p : primitives) {
653                Map<String, String> tags = p.getKeys();
654                if (tags == null || tags.isEmpty()) {
655                    continue;
656                }
657
658                for (Entry<String, String> prop: tags.entrySet()) {
659                    String key = prop.getKey();
660                    String value = prop.getValue();
661                    if (value == null || value.trim().isEmpty()) {
662                        commands.add(new ChangePropertyCommand(p, key, null));
663                    } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains("  ")) {
664                        commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value)));
665                    } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains("  ")) {
666                        commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key)));
667                    } else {
668                        String evalue = Entities.unescape(value);
669                        if (!evalue.equals(value)) {
670                            commands.add(new ChangePropertyCommand(p, key, evalue));
671                        }
672                    }
673                }
674            }
675        }
676
677        if (commands.isEmpty())
678            return null;
679        if (commands.size() == 1)
680            return commands.get(0);
681
682        return new SequenceCommand(tr("Fix tags"), commands);
683    }
684
685    @Override
686    public boolean isFixable(TestError testError) {
687        if (testError.getTester() instanceof TagChecker) {
688            int code = testError.getCode();
689            return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE ||
690                   code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE ||
691                   code == MULTIPLE_SPACES;
692        }
693
694        return false;
695    }
696
697    protected static class CheckerData {
698        private String description;
699        protected List<CheckerElement> data = new ArrayList<>();
700        private OsmPrimitiveType type;
701        private int code;
702        protected Severity severity;
703        protected static final int TAG_CHECK_ERROR  = 1250;
704        protected static final int TAG_CHECK_WARN   = 1260;
705        protected static final int TAG_CHECK_INFO   = 1270;
706
707        protected static class CheckerElement {
708            public Object tag;
709            public Object value;
710            public boolean noMatch;
711            public boolean tagAll;
712            public boolean valueAll;
713            public boolean valueBool;
714
715            private static Pattern getPattern(String str) throws PatternSyntaxException {
716                if (str.endsWith("/i"))
717                    return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE);
718                if (str.endsWith("/"))
719                    return Pattern.compile(str.substring(1, str.length()-1));
720
721                throw new IllegalStateException();
722            }
723
724            public CheckerElement(String exp) throws PatternSyntaxException {
725                Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
726                m.matches();
727
728                String n = m.group(1).trim();
729
730                if ("*".equals(n)) {
731                    tagAll = true;
732                } else {
733                    tag = n.startsWith("/") ? getPattern(n) : n;
734                    noMatch = "!=".equals(m.group(2));
735                    n = m.group(3).trim();
736                    if ("*".equals(n)) {
737                        valueAll = true;
738                    } else if ("BOOLEAN_TRUE".equals(n)) {
739                        valueBool = true;
740                        value = OsmUtils.trueval;
741                    } else if ("BOOLEAN_FALSE".equals(n)) {
742                        valueBool = true;
743                        value = OsmUtils.falseval;
744                    } else {
745                        value = n.startsWith("/") ? getPattern(n) : n;
746                    }
747                }
748            }
749
750            public boolean match(Map<String, String> keys) {
751                for (Entry<String, String> prop: keys.entrySet()) {
752                    String key = prop.getKey();
753                    String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
754                    if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
755                            && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
756                        return !noMatch;
757                }
758                return noMatch;
759            }
760        }
761
762        private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$");
763        private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *");
764        private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *");
765
766        public String getData(final String str) {
767            Matcher m = CLEAN_STR_PATTERN.matcher(str);
768            String trimmed = m.replaceFirst("").trim();
769            try {
770                description = m.group(1);
771                if (description != null && description.isEmpty()) {
772                    description = null;
773                }
774            } catch (IllegalStateException e) {
775                description = null;
776            }
777            String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3);
778            switch (n[0]) {
779            case "way":
780                type = OsmPrimitiveType.WAY;
781                break;
782            case "node":
783                type = OsmPrimitiveType.NODE;
784                break;
785            case "relation":
786                type = OsmPrimitiveType.RELATION;
787                break;
788            case "*":
789                type = null;
790                break;
791            default:
792                return tr("Could not find element type");
793            }
794            if (n.length != 3)
795                return tr("Incorrect number of parameters");
796
797            switch (n[1]) {
798            case "W":
799                severity = Severity.WARNING;
800                code = TAG_CHECK_WARN;
801                break;
802            case "E":
803                severity = Severity.ERROR;
804                code = TAG_CHECK_ERROR;
805                break;
806            case "I":
807                severity = Severity.OTHER;
808                code = TAG_CHECK_INFO;
809                break;
810            default:
811                return tr("Could not find warning level");
812            }
813            for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) {
814                try {
815                    data.add(new CheckerElement(exp));
816                } catch (IllegalStateException e) {
817                    return tr("Illegal expression ''{0}''", exp);
818                } catch (PatternSyntaxException e) {
819                    return tr("Illegal regular expression ''{0}''", exp);
820                }
821            }
822            return null;
823        }
824
825        public boolean match(OsmPrimitive osm, Map<String, String> keys) {
826            if (type != null && OsmPrimitiveType.from(osm) != type)
827                return false;
828
829            for (CheckerElement ce : data) {
830                if (!ce.match(keys))
831                    return false;
832            }
833            return true;
834        }
835
836        public String getDescription() {
837            return tr(description);
838        }
839
840        public String getDescriptionOrig() {
841            return description;
842        }
843
844        public Severity getSeverity() {
845            return severity;
846        }
847
848        public int getCode() {
849            if (type == null)
850                return code;
851
852            return code + type.ordinal() + 1;
853        }
854    }
855}