001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.marktr; 005import static org.openstreetmap.josm.tools.I18n.tr; 006 007import java.awt.GridBagConstraints; 008import java.awt.event.ActionEvent; 009import java.awt.event.ActionListener; 010import java.io.BufferedReader; 011import java.io.IOException; 012import java.io.InputStream; 013import java.text.MessageFormat; 014import java.util.ArrayList; 015import java.util.Arrays; 016import java.util.Collection; 017import java.util.HashMap; 018import java.util.List; 019import java.util.Locale; 020import java.util.Map; 021import java.util.Map.Entry; 022import java.util.Set; 023import java.util.regex.Matcher; 024import java.util.regex.Pattern; 025import java.util.regex.PatternSyntaxException; 026 027import javax.swing.JCheckBox; 028import javax.swing.JLabel; 029import javax.swing.JPanel; 030 031import org.openstreetmap.josm.Main; 032import org.openstreetmap.josm.command.ChangePropertyCommand; 033import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 034import org.openstreetmap.josm.command.Command; 035import org.openstreetmap.josm.command.SequenceCommand; 036import org.openstreetmap.josm.data.osm.OsmPrimitive; 037import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 038import org.openstreetmap.josm.data.osm.OsmUtils; 039import org.openstreetmap.josm.data.osm.Tag; 040import org.openstreetmap.josm.data.validation.FixableTestError; 041import org.openstreetmap.josm.data.validation.Severity; 042import org.openstreetmap.josm.data.validation.Test.TagTest; 043import org.openstreetmap.josm.data.validation.TestError; 044import org.openstreetmap.josm.data.validation.util.Entities; 045import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference; 046import org.openstreetmap.josm.gui.progress.ProgressMonitor; 047import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset; 048import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem; 049import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets; 050import org.openstreetmap.josm.gui.tagging.presets.items.Check; 051import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup; 052import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem; 053import org.openstreetmap.josm.gui.widgets.EditableList; 054import org.openstreetmap.josm.io.CachedFile; 055import org.openstreetmap.josm.io.UTFInputStreamReader; 056import org.openstreetmap.josm.tools.GBC; 057import org.openstreetmap.josm.tools.MultiMap; 058import org.openstreetmap.josm.tools.Utils; 059 060/** 061 * Check for misspelled or wrong tags 062 * 063 * @author frsantos 064 * @since 3669 065 */ 066public class TagChecker extends TagTest { 067 068 /** The config file of ignored tags */ 069 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg"; 070 /** The config file of dictionary words */ 071 public static final String SPELL_FILE = "resource://data/validator/words.cfg"; 072 073 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */ 074 private static final Map<String, String> harmonizedKeys = new HashMap<>(); 075 /** The spell check preset values */ 076 private static volatile MultiMap<String, String> presetsValueData; 077 /** The TagChecker data */ 078 private static final List<CheckerData> checkerData = new ArrayList<>(); 079 private static final List<String> ignoreDataStartsWith = new ArrayList<>(); 080 private static final List<String> ignoreDataEquals = new ArrayList<>(); 081 private static final List<String> ignoreDataEndsWith = new ArrayList<>(); 082 private static final List<Tag> ignoreDataTag = new ArrayList<>(); 083 084 /** The preferences prefix */ 085 protected static final String PREFIX = ValidatorPreference.PREFIX + "." + TagChecker.class.getSimpleName(); 086 087 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues"; 088 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys"; 089 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex"; 090 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes"; 091 092 public static final String PREF_SOURCES = PREFIX + ".source"; 093 094 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload"; 095 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload"; 096 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload"; 097 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload"; 098 099 protected boolean checkKeys; 100 protected boolean checkValues; 101 protected boolean checkComplex; 102 protected boolean checkFixmes; 103 104 protected JCheckBox prefCheckKeys; 105 protected JCheckBox prefCheckValues; 106 protected JCheckBox prefCheckComplex; 107 protected JCheckBox prefCheckFixmes; 108 protected JCheckBox prefCheckPaint; 109 110 protected JCheckBox prefCheckKeysBeforeUpload; 111 protected JCheckBox prefCheckValuesBeforeUpload; 112 protected JCheckBox prefCheckComplexBeforeUpload; 113 protected JCheckBox prefCheckFixmesBeforeUpload; 114 protected JCheckBox prefCheckPaintBeforeUpload; 115 116 protected static final int EMPTY_VALUES = 1200; 117 protected static final int INVALID_KEY = 1201; 118 protected static final int INVALID_VALUE = 1202; 119 protected static final int FIXME = 1203; 120 protected static final int INVALID_SPACE = 1204; 121 protected static final int INVALID_KEY_SPACE = 1205; 122 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */ 123 protected static final int LONG_VALUE = 1208; 124 protected static final int LONG_KEY = 1209; 125 protected static final int LOW_CHAR_VALUE = 1210; 126 protected static final int LOW_CHAR_KEY = 1211; 127 protected static final int MISSPELLED_VALUE = 1212; 128 protected static final int MISSPELLED_KEY = 1213; 129 protected static final int MULTIPLE_SPACES = 1214; 130 /** 1250 and up is used by tagcheck */ 131 132 protected EditableList sourcesList; 133 134 private static final List<String> DEFAULT_SOURCES = Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE); 135 136 /** 137 * Constructor 138 */ 139 public TagChecker() { 140 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values.")); 141 } 142 143 @Override 144 public void initialize() throws IOException { 145 initializeData(); 146 initializePresets(); 147 } 148 149 /** 150 * Reads the spellcheck file into a HashMap. 151 * The data file is a list of words, beginning with +/-. If it starts with +, 152 * the word is valid, but if it starts with -, the word should be replaced 153 * by the nearest + word before this. 154 * 155 * @throws IOException if any I/O error occurs 156 */ 157 private static void initializeData() throws IOException { 158 checkerData.clear(); 159 ignoreDataStartsWith.clear(); 160 ignoreDataEquals.clear(); 161 ignoreDataEndsWith.clear(); 162 ignoreDataTag.clear(); 163 harmonizedKeys.clear(); 164 165 StringBuilder errorSources = new StringBuilder(); 166 for (String source : Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES)) { 167 try ( 168 CachedFile cf = new CachedFile(source); 169 InputStream s = cf.getInputStream(); 170 BufferedReader reader = new BufferedReader(UTFInputStreamReader.create(s)); 171 ) { 172 String okValue = null; 173 boolean tagcheckerfile = false; 174 boolean ignorefile = false; 175 boolean isFirstLine = true; 176 String line; 177 while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) { 178 if (line.startsWith("#")) { 179 if (line.startsWith("# JOSM TagChecker")) { 180 tagcheckerfile = true; 181 if (!DEFAULT_SOURCES.contains(source)) { 182 Main.info(tr("Adding {0} to tag checker", source)); 183 } 184 } else 185 if (line.startsWith("# JOSM IgnoreTags")) { 186 ignorefile = true; 187 if (!DEFAULT_SOURCES.contains(source)) { 188 Main.info(tr("Adding {0} to ignore tags", source)); 189 } 190 } 191 } else if (ignorefile) { 192 line = line.trim(); 193 if (line.length() < 4) { 194 continue; 195 } 196 197 String key = line.substring(0, 2); 198 line = line.substring(2); 199 200 switch (key) { 201 case "S:": 202 ignoreDataStartsWith.add(line); 203 break; 204 case "E:": 205 ignoreDataEquals.add(line); 206 break; 207 case "F:": 208 ignoreDataEndsWith.add(line); 209 break; 210 case "K:": 211 ignoreDataTag.add(Tag.ofString(line)); 212 } 213 } else if (tagcheckerfile) { 214 if (!line.isEmpty()) { 215 CheckerData d = new CheckerData(); 216 String err = d.getData(line); 217 218 if (err == null) { 219 checkerData.add(d); 220 } else { 221 Main.error(tr("Invalid tagchecker line - {0}: {1}", err, line)); 222 } 223 } 224 } else if (line.charAt(0) == '+') { 225 okValue = line.substring(1); 226 } else if (line.charAt(0) == '-' && okValue != null) { 227 harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue); 228 } else { 229 Main.error(tr("Invalid spellcheck line: {0}", line)); 230 } 231 if (isFirstLine) { 232 isFirstLine = false; 233 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) { 234 Main.info(tr("Adding {0} to spellchecker", source)); 235 } 236 } 237 } 238 } catch (IOException e) { 239 errorSources.append(source).append('\n'); 240 } 241 } 242 243 if (errorSources.length() > 0) 244 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources)); 245 } 246 247 /** 248 * Reads the presets data. 249 * 250 */ 251 public static void initializePresets() { 252 253 if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true)) 254 return; 255 256 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets(); 257 if (!presets.isEmpty()) { 258 presetsValueData = new MultiMap<>(); 259 for (String a : OsmPrimitive.getUninterestingKeys()) { 260 presetsValueData.putVoid(a); 261 } 262 // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead) 263 for (String a : Main.pref.getCollection(ValidatorPreference.PREFIX + ".knownkeys", 264 Arrays.asList(new String[]{"is_in", "int_ref", "fixme", "population"}))) { 265 presetsValueData.putVoid(a); 266 } 267 for (TaggingPreset p : presets) { 268 for (TaggingPresetItem i : p.data) { 269 if (i instanceof KeyedItem) { 270 addPresetValue(p, (KeyedItem) i); 271 } else if (i instanceof CheckGroup) { 272 for (Check c : ((CheckGroup) i).checks) { 273 addPresetValue(p, c); 274 } 275 } 276 } 277 } 278 } 279 } 280 281 private static void addPresetValue(TaggingPreset p, KeyedItem ky) { 282 Collection<String> values = ky.getValues(); 283 if (ky.key != null && values != null) { 284 try { 285 presetsValueData.putAll(ky.key, values); 286 harmonizedKeys.put(harmonizeKey(ky.key), ky.key); 287 } catch (NullPointerException e) { 288 Main.error(p+": Unable to initialize "+ky); 289 } 290 } 291 } 292 293 /** 294 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters) 295 * @param s string to check 296 * @return {@code true} if {@code s} contains characters with code below 0x20 297 */ 298 private static boolean containsLow(String s) { 299 if (s == null) 300 return false; 301 for (int i = 0; i < s.length(); i++) { 302 if (s.charAt(i) < 0x20) 303 return true; 304 } 305 return false; 306 } 307 308 /** 309 * Determines if the given key is in internal presets. 310 * @param key key 311 * @return {@code true} if the given key is in internal presets 312 * @since 9023 313 */ 314 public static boolean isKeyInPresets(String key) { 315 return presetsValueData.get(key) != null; 316 } 317 318 /** 319 * Determines if the given tag is in internal presets. 320 * @param key key 321 * @param value value 322 * @return {@code true} if the given tag is in internal presets 323 * @since 9023 324 */ 325 public static boolean isTagInPresets(String key, String value) { 326 final Set<String> values = presetsValueData.get(key); 327 return values != null && (values.isEmpty() || values.contains(value)); 328 } 329 330 /** 331 * Returns the list of ignored tags. 332 * @return the list of ignored tags 333 * @since 9023 334 */ 335 public static List<Tag> getIgnoredTags() { 336 return new ArrayList<>(ignoreDataTag); 337 } 338 339 /** 340 * Determines if the given tag is ignored for checks "key/tag not in presets". 341 * @param key key 342 * @param value value 343 * @return {@code true} if the given tag is ignored 344 * @since 9023 345 */ 346 public static boolean isTagIgnored(String key, String value) { 347 boolean tagInPresets = isTagInPresets(key, value); 348 boolean ignore = false; 349 350 for (String a : ignoreDataStartsWith) { 351 if (key.startsWith(a)) { 352 ignore = true; 353 } 354 } 355 for (String a : ignoreDataEquals) { 356 if (key.equals(a)) { 357 ignore = true; 358 } 359 } 360 for (String a : ignoreDataEndsWith) { 361 if (key.endsWith(a)) { 362 ignore = true; 363 } 364 } 365 366 if (!tagInPresets) { 367 for (Tag a : ignoreDataTag) { 368 if (key.equals(a.getKey()) && value.equals(a.getValue())) { 369 ignore = true; 370 } 371 } 372 } 373 return ignore; 374 } 375 376 /** 377 * Checks the primitive tags 378 * @param p The primitive to check 379 */ 380 @Override 381 public void check(OsmPrimitive p) { 382 // Just a collection to know if a primitive has been already marked with error 383 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>(); 384 385 if (checkComplex) { 386 Map<String, String> keys = p.getKeys(); 387 for (CheckerData d : checkerData) { 388 if (d.match(p, keys)) { 389 errors.add(new TestError(this, d.getSeverity(), tr("Suspicious tag/value combinations"), 390 d.getDescription(), d.getDescriptionOrig(), d.getCode(), p)); 391 withErrors.put(p, "TC"); 392 } 393 } 394 } 395 396 for (Entry<String, String> prop : p.getKeys().entrySet()) { 397 String s = marktr("Key ''{0}'' invalid."); 398 String key = prop.getKey(); 399 String value = prop.getValue(); 400 if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) { 401 errors.add(new TestError(this, Severity.WARNING, tr("Tag value contains character with code less than 0x20"), 402 tr(s, key), MessageFormat.format(s, key), LOW_CHAR_VALUE, p)); 403 withErrors.put(p, "ICV"); 404 } 405 if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) { 406 errors.add(new TestError(this, Severity.WARNING, tr("Tag key contains character with code less than 0x20"), 407 tr(s, key), MessageFormat.format(s, key), LOW_CHAR_KEY, p)); 408 withErrors.put(p, "ICK"); 409 } 410 if (checkValues && (value != null && value.length() > 255) && !withErrors.contains(p, "LV")) { 411 errors.add(new TestError(this, Severity.ERROR, tr("Tag value longer than allowed"), 412 tr(s, key), MessageFormat.format(s, key), LONG_VALUE, p)); 413 withErrors.put(p, "LV"); 414 } 415 if (checkKeys && (key != null && key.length() > 255) && !withErrors.contains(p, "LK")) { 416 errors.add(new TestError(this, Severity.ERROR, tr("Tag key longer than allowed"), 417 tr(s, key), MessageFormat.format(s, key), LONG_KEY, p)); 418 withErrors.put(p, "LK"); 419 } 420 if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) { 421 errors.add(new TestError(this, Severity.WARNING, tr("Tags with empty values"), 422 tr(s, key), MessageFormat.format(s, key), EMPTY_VALUES, p)); 423 withErrors.put(p, "EV"); 424 } 425 if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) { 426 errors.add(new TestError(this, Severity.WARNING, tr("Invalid white space in property key"), 427 tr(s, key), MessageFormat.format(s, key), INVALID_KEY_SPACE, p)); 428 withErrors.put(p, "IPK"); 429 } 430 if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) { 431 errors.add(new TestError(this, Severity.WARNING, tr("Property values start or end with white space"), 432 tr(s, key), MessageFormat.format(s, key), INVALID_SPACE, p)); 433 withErrors.put(p, "SPACE"); 434 } 435 if (checkValues && value != null && value.contains(" ") && !withErrors.contains(p, "SPACE")) { 436 errors.add(new TestError(this, Severity.WARNING, tr("Property values contain multiple white spaces"), 437 tr(s, key), MessageFormat.format(s, key), MULTIPLE_SPACES, p)); 438 withErrors.put(p, "SPACE"); 439 } 440 if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) { 441 errors.add(new TestError(this, Severity.OTHER, tr("Property values contain HTML entity"), 442 tr(s, key), MessageFormat.format(s, key), INVALID_HTML, p)); 443 withErrors.put(p, "HTML"); 444 } 445 if (checkValues && key != null && value != null && !value.isEmpty() && presetsValueData != null) { 446 if (!isTagIgnored(key, value)) { 447 if (!isKeyInPresets(key)) { 448 String prettifiedKey = harmonizeKey(key); 449 String fixedKey = harmonizedKeys.get(prettifiedKey); 450 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) { 451 // misspelled preset key 452 String i = marktr("Key ''{0}'' looks like ''{1}''."); 453 final TestError error; 454 if (p.hasKey(fixedKey)) { 455 error = new TestError(this, Severity.WARNING, tr("Misspelled property key"), 456 tr(i, key, fixedKey), 457 MessageFormat.format(i, key, fixedKey), MISSPELLED_KEY, p); 458 } else { 459 error = new FixableTestError(this, Severity.WARNING, tr("Misspelled property key"), 460 tr(i, key, fixedKey), 461 MessageFormat.format(i, key, fixedKey), MISSPELLED_KEY, p, 462 new ChangePropertyKeyCommand(p, key, fixedKey)); 463 } 464 errors.add(error); 465 withErrors.put(p, "WPK"); 466 } else { 467 String i = marktr("Key ''{0}'' not in presets."); 468 errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property key"), 469 tr(i, key), MessageFormat.format(i, key), INVALID_VALUE, p)); 470 withErrors.put(p, "UPK"); 471 } 472 } else if (!isTagInPresets(key, value)) { 473 // try to fix common typos and check again if value is still unknown 474 String fixedValue = harmonizeValue(prop.getValue()); 475 Map<String, String> possibleValues = getPossibleValues(presetsValueData.get(key)); 476 if (possibleValues.containsKey(fixedValue)) { 477 fixedValue = possibleValues.get(fixedValue); 478 // misspelled preset value 479 String i = marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''."); 480 errors.add(new FixableTestError(this, Severity.WARNING, tr("Misspelled property value"), 481 tr(i, prop.getValue(), key, fixedValue), MessageFormat.format(i, prop.getValue(), fixedValue), 482 MISSPELLED_VALUE, p, new ChangePropertyCommand(p, key, fixedValue))); 483 withErrors.put(p, "WPV"); 484 } else { 485 // unknown preset value 486 String i = marktr("Value ''{0}'' for key ''{1}'' not in presets."); 487 errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property value"), 488 tr(i, prop.getValue(), key), MessageFormat.format(i, prop.getValue(), key), INVALID_VALUE, p)); 489 withErrors.put(p, "UPV"); 490 } 491 } 492 } 493 } 494 if (checkFixmes && key != null && value != null && !value.isEmpty()) { 495 if ((value.toLowerCase(Locale.ENGLISH).contains("fixme") 496 || value.contains("check and delete") 497 || key.contains("todo") || key.toLowerCase(Locale.ENGLISH).contains("fixme")) 498 && !withErrors.contains(p, "FIXME")) { 499 errors.add(new TestError(this, Severity.OTHER, 500 tr("FIXMES"), FIXME, p)); 501 withErrors.put(p, "FIXME"); 502 } 503 } 504 } 505 } 506 507 private static Map<String, String> getPossibleValues(Set<String> values) { 508 // generate a map with common typos 509 Map<String, String> map = new HashMap<>(); 510 if (values != null) { 511 for (String value : values) { 512 map.put(value, value); 513 if (value.contains("_")) { 514 map.put(value.replace("_", ""), value); 515 } 516 } 517 } 518 return map; 519 } 520 521 private static String harmonizeKey(String key) { 522 key = key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'); 523 return Utils.strip(key, "-_;:,"); 524 } 525 526 private static String harmonizeValue(String value) { 527 value = value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'); 528 return Utils.strip(value, "-_;:,"); 529 } 530 531 @Override 532 public void startTest(ProgressMonitor monitor) { 533 super.startTest(monitor); 534 checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true); 535 if (isBeforeUpload) { 536 checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true); 537 } 538 539 checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true); 540 if (isBeforeUpload) { 541 checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true); 542 } 543 544 checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true); 545 if (isBeforeUpload) { 546 checkComplex = checkComplex && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true); 547 } 548 549 checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true); 550 if (isBeforeUpload) { 551 checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true); 552 } 553 } 554 555 @Override 556 public void visit(Collection<OsmPrimitive> selection) { 557 if (checkKeys || checkValues || checkComplex || checkFixmes) { 558 super.visit(selection); 559 } 560 } 561 562 @Override 563 public void addGui(JPanel testPanel) { 564 GBC a = GBC.eol(); 565 a.anchor = GridBagConstraints.EAST; 566 567 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0)); 568 569 prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true)); 570 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words.")); 571 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0)); 572 573 prefCheckKeysBeforeUpload = new JCheckBox(); 574 prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true)); 575 testPanel.add(prefCheckKeysBeforeUpload, a); 576 577 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true)); 578 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules.")); 579 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0)); 580 581 prefCheckComplexBeforeUpload = new JCheckBox(); 582 prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true)); 583 testPanel.add(prefCheckComplexBeforeUpload, a); 584 585 final Collection<String> sources = Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES); 586 sourcesList = new EditableList(tr("TagChecker source")); 587 sourcesList.setItems(sources); 588 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0)); 589 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0)); 590 591 ActionListener disableCheckActionListener = new ActionListener() { 592 @Override 593 public void actionPerformed(ActionEvent e) { 594 handlePrefEnable(); 595 } 596 }; 597 prefCheckKeys.addActionListener(disableCheckActionListener); 598 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener); 599 prefCheckComplex.addActionListener(disableCheckActionListener); 600 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener); 601 602 handlePrefEnable(); 603 604 prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true)); 605 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets.")); 606 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0)); 607 608 prefCheckValuesBeforeUpload = new JCheckBox(); 609 prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true)); 610 testPanel.add(prefCheckValuesBeforeUpload, a); 611 612 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true)); 613 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value.")); 614 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0)); 615 616 prefCheckFixmesBeforeUpload = new JCheckBox(); 617 prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true)); 618 testPanel.add(prefCheckFixmesBeforeUpload, a); 619 } 620 621 public void handlePrefEnable() { 622 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected() 623 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 624 sourcesList.setEnabled(selected); 625 } 626 627 @Override 628 public boolean ok() { 629 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected(); 630 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected() 631 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 632 633 Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected()); 634 Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected()); 635 Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected()); 636 Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected()); 637 Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected()); 638 Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected()); 639 Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected()); 640 Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected()); 641 return Main.pref.putCollection(PREF_SOURCES, sourcesList.getItems()); 642 } 643 644 @Override 645 public Command fixError(TestError testError) { 646 List<Command> commands = new ArrayList<>(50); 647 648 if (testError instanceof FixableTestError) { 649 commands.add(testError.getFix()); 650 } else { 651 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives(); 652 for (OsmPrimitive p : primitives) { 653 Map<String, String> tags = p.getKeys(); 654 if (tags == null || tags.isEmpty()) { 655 continue; 656 } 657 658 for (Entry<String, String> prop: tags.entrySet()) { 659 String key = prop.getKey(); 660 String value = prop.getValue(); 661 if (value == null || value.trim().isEmpty()) { 662 commands.add(new ChangePropertyCommand(p, key, null)); 663 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) { 664 commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value))); 665 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) { 666 commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key))); 667 } else { 668 String evalue = Entities.unescape(value); 669 if (!evalue.equals(value)) { 670 commands.add(new ChangePropertyCommand(p, key, evalue)); 671 } 672 } 673 } 674 } 675 } 676 677 if (commands.isEmpty()) 678 return null; 679 if (commands.size() == 1) 680 return commands.get(0); 681 682 return new SequenceCommand(tr("Fix tags"), commands); 683 } 684 685 @Override 686 public boolean isFixable(TestError testError) { 687 if (testError.getTester() instanceof TagChecker) { 688 int code = testError.getCode(); 689 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE || 690 code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE || 691 code == MULTIPLE_SPACES; 692 } 693 694 return false; 695 } 696 697 protected static class CheckerData { 698 private String description; 699 protected List<CheckerElement> data = new ArrayList<>(); 700 private OsmPrimitiveType type; 701 private int code; 702 protected Severity severity; 703 protected static final int TAG_CHECK_ERROR = 1250; 704 protected static final int TAG_CHECK_WARN = 1260; 705 protected static final int TAG_CHECK_INFO = 1270; 706 707 protected static class CheckerElement { 708 public Object tag; 709 public Object value; 710 public boolean noMatch; 711 public boolean tagAll; 712 public boolean valueAll; 713 public boolean valueBool; 714 715 private static Pattern getPattern(String str) throws PatternSyntaxException { 716 if (str.endsWith("/i")) 717 return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE); 718 if (str.endsWith("/")) 719 return Pattern.compile(str.substring(1, str.length()-1)); 720 721 throw new IllegalStateException(); 722 } 723 724 public CheckerElement(String exp) throws PatternSyntaxException { 725 Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp); 726 m.matches(); 727 728 String n = m.group(1).trim(); 729 730 if ("*".equals(n)) { 731 tagAll = true; 732 } else { 733 tag = n.startsWith("/") ? getPattern(n) : n; 734 noMatch = "!=".equals(m.group(2)); 735 n = m.group(3).trim(); 736 if ("*".equals(n)) { 737 valueAll = true; 738 } else if ("BOOLEAN_TRUE".equals(n)) { 739 valueBool = true; 740 value = OsmUtils.trueval; 741 } else if ("BOOLEAN_FALSE".equals(n)) { 742 valueBool = true; 743 value = OsmUtils.falseval; 744 } else { 745 value = n.startsWith("/") ? getPattern(n) : n; 746 } 747 } 748 } 749 750 public boolean match(Map<String, String> keys) { 751 for (Entry<String, String> prop: keys.entrySet()) { 752 String key = prop.getKey(); 753 String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue(); 754 if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag))) 755 && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value)))) 756 return !noMatch; 757 } 758 return noMatch; 759 } 760 } 761 762 private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$"); 763 private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *"); 764 private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *"); 765 766 public String getData(final String str) { 767 Matcher m = CLEAN_STR_PATTERN.matcher(str); 768 String trimmed = m.replaceFirst("").trim(); 769 try { 770 description = m.group(1); 771 if (description != null && description.isEmpty()) { 772 description = null; 773 } 774 } catch (IllegalStateException e) { 775 description = null; 776 } 777 String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3); 778 switch (n[0]) { 779 case "way": 780 type = OsmPrimitiveType.WAY; 781 break; 782 case "node": 783 type = OsmPrimitiveType.NODE; 784 break; 785 case "relation": 786 type = OsmPrimitiveType.RELATION; 787 break; 788 case "*": 789 type = null; 790 break; 791 default: 792 return tr("Could not find element type"); 793 } 794 if (n.length != 3) 795 return tr("Incorrect number of parameters"); 796 797 switch (n[1]) { 798 case "W": 799 severity = Severity.WARNING; 800 code = TAG_CHECK_WARN; 801 break; 802 case "E": 803 severity = Severity.ERROR; 804 code = TAG_CHECK_ERROR; 805 break; 806 case "I": 807 severity = Severity.OTHER; 808 code = TAG_CHECK_INFO; 809 break; 810 default: 811 return tr("Could not find warning level"); 812 } 813 for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) { 814 try { 815 data.add(new CheckerElement(exp)); 816 } catch (IllegalStateException e) { 817 return tr("Illegal expression ''{0}''", exp); 818 } catch (PatternSyntaxException e) { 819 return tr("Illegal regular expression ''{0}''", exp); 820 } 821 } 822 return null; 823 } 824 825 public boolean match(OsmPrimitive osm, Map<String, String> keys) { 826 if (type != null && OsmPrimitiveType.from(osm) != type) 827 return false; 828 829 for (CheckerElement ce : data) { 830 if (!ce.match(keys)) 831 return false; 832 } 833 return true; 834 } 835 836 public String getDescription() { 837 return tr(description); 838 } 839 840 public String getDescriptionOrig() { 841 return description; 842 } 843 844 public Severity getSeverity() { 845 return severity; 846 } 847 848 public int getCode() { 849 if (type == null) 850 return code; 851 852 return code + type.ordinal() + 1; 853 } 854 } 855}