001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.awt.Rectangle; 007import java.io.BufferedReader; 008import java.io.IOException; 009import java.io.InputStream; 010import java.io.Reader; 011import java.io.StringReader; 012import java.lang.reflect.Method; 013import java.text.MessageFormat; 014import java.util.ArrayList; 015import java.util.Arrays; 016import java.util.Collection; 017import java.util.Collections; 018import java.util.HashMap; 019import java.util.HashSet; 020import java.util.Iterator; 021import java.util.LinkedHashMap; 022import java.util.LinkedHashSet; 023import java.util.LinkedList; 024import java.util.List; 025import java.util.Locale; 026import java.util.Map; 027import java.util.Objects; 028import java.util.Optional; 029import java.util.Set; 030import java.util.function.Predicate; 031import java.util.regex.Matcher; 032import java.util.regex.Pattern; 033 034import org.openstreetmap.josm.command.ChangePropertyCommand; 035import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 036import org.openstreetmap.josm.command.Command; 037import org.openstreetmap.josm.command.DeleteCommand; 038import org.openstreetmap.josm.command.SequenceCommand; 039import org.openstreetmap.josm.data.coor.LatLon; 040import org.openstreetmap.josm.data.osm.DataSet; 041import org.openstreetmap.josm.data.osm.INode; 042import org.openstreetmap.josm.data.osm.IRelation; 043import org.openstreetmap.josm.data.osm.IWay; 044import org.openstreetmap.josm.data.osm.OsmPrimitive; 045import org.openstreetmap.josm.data.osm.OsmUtils; 046import org.openstreetmap.josm.data.osm.Relation; 047import org.openstreetmap.josm.data.osm.Tag; 048import org.openstreetmap.josm.data.osm.Way; 049import org.openstreetmap.josm.data.preferences.sources.SourceEntry; 050import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper; 051import org.openstreetmap.josm.data.validation.OsmValidator; 052import org.openstreetmap.josm.data.validation.Severity; 053import org.openstreetmap.josm.data.validation.Test; 054import org.openstreetmap.josm.data.validation.TestError; 055import org.openstreetmap.josm.gui.mappaint.Environment; 056import org.openstreetmap.josm.gui.mappaint.Keyword; 057import org.openstreetmap.josm.gui.mappaint.MultiCascade; 058import org.openstreetmap.josm.gui.mappaint.mapcss.Condition; 059import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ClassCondition; 060import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ExpressionCondition; 061import org.openstreetmap.josm.gui.mappaint.mapcss.Expression; 062import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.Functions; 063import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.ParameterFunction; 064import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction; 065import org.openstreetmap.josm.gui.mappaint.mapcss.LiteralExpression; 066import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule; 067import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration; 068import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource; 069import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource.MapCSSRuleIndex; 070import org.openstreetmap.josm.gui.mappaint.mapcss.Selector; 071import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.AbstractSelector; 072import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector; 073import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.OptimizedGeneralSelector; 074import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser; 075import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException; 076import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError; 077import org.openstreetmap.josm.gui.progress.ProgressMonitor; 078import org.openstreetmap.josm.io.CachedFile; 079import org.openstreetmap.josm.io.FileWatcher; 080import org.openstreetmap.josm.io.IllegalDataException; 081import org.openstreetmap.josm.io.UTFInputStreamReader; 082import org.openstreetmap.josm.spi.preferences.Config; 083import org.openstreetmap.josm.tools.CheckParameterUtil; 084import org.openstreetmap.josm.tools.DefaultGeoProperty; 085import org.openstreetmap.josm.tools.GeoProperty; 086import org.openstreetmap.josm.tools.GeoPropertyIndex; 087import org.openstreetmap.josm.tools.I18n; 088import org.openstreetmap.josm.tools.JosmRuntimeException; 089import org.openstreetmap.josm.tools.Logging; 090import org.openstreetmap.josm.tools.MultiMap; 091import org.openstreetmap.josm.tools.Territories; 092import org.openstreetmap.josm.tools.Utils; 093 094/** 095 * MapCSS-based tag checker/fixer. 096 * @since 6506 097 */ 098public class MapCSSTagChecker extends Test.TagTest { 099 IndexData indexData; 100 101 /** 102 * Helper class to store indexes of rules. 103 * @author Gerd 104 * 105 */ 106 private static class IndexData { 107 final Map<MapCSSRule, TagCheck> ruleToCheckMap = new HashMap<>(); 108 109 /** 110 * Rules for nodes 111 */ 112 final MapCSSRuleIndex nodeRules = new MapCSSRuleIndex(); 113 /** 114 * Rules for ways without tag area=no 115 */ 116 final MapCSSRuleIndex wayRules = new MapCSSRuleIndex(); 117 /** 118 * Rules for ways with tag area=no 119 */ 120 final MapCSSRuleIndex wayNoAreaRules = new MapCSSRuleIndex(); 121 /** 122 * Rules for relations that are not multipolygon relations 123 */ 124 final MapCSSRuleIndex relationRules = new MapCSSRuleIndex(); 125 /** 126 * Rules for multipolygon relations 127 */ 128 final MapCSSRuleIndex multipolygonRules = new MapCSSRuleIndex(); 129 130 IndexData(MultiMap<String, TagCheck> checks) { 131 buildIndex(checks); 132 } 133 134 private void buildIndex(MultiMap<String, TagCheck> checks) { 135 List<TagCheck> allChecks = new ArrayList<>(); 136 for (Set<TagCheck> cs : checks.values()) { 137 allChecks.addAll(cs); 138 } 139 140 ruleToCheckMap.clear(); 141 nodeRules.clear(); 142 wayRules.clear(); 143 wayNoAreaRules.clear(); 144 relationRules.clear(); 145 multipolygonRules.clear(); 146 147 // optimization: filter rules for different primitive types 148 for (TagCheck c : allChecks) { 149 for (Selector s : c.rule.selectors) { 150 // find the rightmost selector, this must be a GeneralSelector 151 Selector selRightmost = s; 152 while (selRightmost instanceof Selector.ChildOrParentSelector) { 153 selRightmost = ((Selector.ChildOrParentSelector) selRightmost).right; 154 } 155 MapCSSRule optRule = new MapCSSRule(s.optimizedBaseCheck(), c.rule.declaration); 156 157 ruleToCheckMap.put(optRule, c); 158 final String base = ((GeneralSelector) selRightmost).getBase(); 159 switch (base) { 160 case Selector.BASE_NODE: 161 nodeRules.add(optRule); 162 break; 163 case Selector.BASE_WAY: 164 wayNoAreaRules.add(optRule); 165 wayRules.add(optRule); 166 break; 167 case Selector.BASE_AREA: 168 wayRules.add(optRule); 169 multipolygonRules.add(optRule); 170 break; 171 case Selector.BASE_RELATION: 172 relationRules.add(optRule); 173 multipolygonRules.add(optRule); 174 break; 175 case Selector.BASE_ANY: 176 nodeRules.add(optRule); 177 wayRules.add(optRule); 178 wayNoAreaRules.add(optRule); 179 relationRules.add(optRule); 180 multipolygonRules.add(optRule); 181 break; 182 case Selector.BASE_CANVAS: 183 case Selector.BASE_META: 184 case Selector.BASE_SETTING: 185 break; 186 default: 187 final RuntimeException e = new JosmRuntimeException(MessageFormat.format("Unknown MapCSS base selector {0}", base)); 188 Logging.warn(tr("Failed to index validator rules. Error was: {0}", e.getMessage())); 189 Logging.error(e); 190 } 191 } 192 } 193 nodeRules.initIndex(); 194 wayRules.initIndex(); 195 wayNoAreaRules.initIndex(); 196 relationRules.initIndex(); 197 multipolygonRules.initIndex(); 198 } 199 200 /** 201 * Get the index of rules for the given primitive. 202 * @param p the primitve 203 * @return index of rules for the given primitive 204 */ 205 public MapCSSRuleIndex get(OsmPrimitive p) { 206 if (p instanceof INode) { 207 return nodeRules; 208 } else if (p instanceof IWay) { 209 if (OsmUtils.isFalse(p.get("area"))) { 210 return wayNoAreaRules; 211 } else { 212 return wayRules; 213 } 214 } else if (p instanceof IRelation) { 215 if (((IRelation<?>) p).isMultipolygon()) { 216 return multipolygonRules; 217 } else { 218 return relationRules; 219 } 220 } else { 221 throw new IllegalArgumentException("Unsupported type: " + p); 222 } 223 } 224 225 /** 226 * return the TagCheck for which the given indexed rule was created. 227 * @param rule an indexed rule 228 * @return the original TagCheck 229 */ 230 public TagCheck getCheck(MapCSSRule rule) { 231 return ruleToCheckMap.get(rule); 232 } 233 } 234 235 /** 236 * A grouped MapCSSRule with multiple selectors for a single declaration. 237 * @see MapCSSRule 238 */ 239 public static class GroupedMapCSSRule { 240 /** MapCSS selectors **/ 241 public final List<Selector> selectors; 242 /** MapCSS declaration **/ 243 public final Declaration declaration; 244 245 /** 246 * Constructs a new {@code GroupedMapCSSRule}. 247 * @param selectors MapCSS selectors 248 * @param declaration MapCSS declaration 249 */ 250 public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) { 251 this.selectors = selectors; 252 this.declaration = declaration; 253 } 254 255 @Override 256 public int hashCode() { 257 return Objects.hash(selectors, declaration); 258 } 259 260 @Override 261 public boolean equals(Object obj) { 262 if (this == obj) return true; 263 if (obj == null || getClass() != obj.getClass()) return false; 264 GroupedMapCSSRule that = (GroupedMapCSSRule) obj; 265 return Objects.equals(selectors, that.selectors) && 266 Objects.equals(declaration, that.declaration); 267 } 268 269 @Override 270 public String toString() { 271 return "GroupedMapCSSRule [selectors=" + selectors + ", declaration=" + declaration + ']'; 272 } 273 } 274 275 /** 276 * The preference key for tag checker source entries. 277 * @since 6670 278 */ 279 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries"; 280 281 /** 282 * Constructs a new {@code MapCSSTagChecker}. 283 */ 284 public MapCSSTagChecker() { 285 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values.")); 286 } 287 288 /** 289 * Represents a fix to a validation test. The fixing {@link Command} can be obtained by {@link #createCommand(OsmPrimitive, Selector)}. 290 */ 291 @FunctionalInterface 292 interface FixCommand { 293 /** 294 * Creates the fixing {@link Command} for the given primitive. The {@code matchingSelector} is used to evaluate placeholders 295 * (cf. {@link MapCSSTagChecker.TagCheck#insertArguments(Selector, String, OsmPrimitive)}). 296 * @param p OSM primitive 297 * @param matchingSelector matching selector 298 * @return fix command 299 */ 300 Command createCommand(OsmPrimitive p, Selector matchingSelector); 301 302 /** 303 * Checks that object is either an {@link Expression} or a {@link String}. 304 * @param obj object to check 305 * @throws IllegalArgumentException if object is not an {@code Expression} or a {@code String} 306 */ 307 static void checkObject(final Object obj) { 308 CheckParameterUtil.ensureThat(obj instanceof Expression || obj instanceof String, 309 () -> "instance of Exception or String expected, but got " + obj); 310 } 311 312 /** 313 * Evaluates given object as {@link Expression} or {@link String} on the matched {@link OsmPrimitive} and {@code matchingSelector}. 314 * @param obj object to evaluate ({@link Expression} or {@link String}) 315 * @param p OSM primitive 316 * @param matchingSelector matching selector 317 * @return result string 318 */ 319 static String evaluateObject(final Object obj, final OsmPrimitive p, final Selector matchingSelector) { 320 final String s; 321 if (obj instanceof Expression) { 322 s = (String) ((Expression) obj).evaluate(new Environment(p)); 323 } else if (obj instanceof String) { 324 s = (String) obj; 325 } else { 326 return null; 327 } 328 return TagCheck.insertArguments(matchingSelector, s, p); 329 } 330 331 /** 332 * Creates a fixing command which executes a {@link ChangePropertyCommand} on the specified tag. 333 * @param obj object to evaluate ({@link Expression} or {@link String}) 334 * @return created fix command 335 */ 336 static FixCommand fixAdd(final Object obj) { 337 checkObject(obj); 338 return new FixCommand() { 339 @Override 340 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 341 final Tag tag = Tag.ofString(evaluateObject(obj, p, matchingSelector)); 342 return new ChangePropertyCommand(p, tag.getKey(), tag.getValue()); 343 } 344 345 @Override 346 public String toString() { 347 return "fixAdd: " + obj; 348 } 349 }; 350 } 351 352 /** 353 * Creates a fixing command which executes a {@link ChangePropertyCommand} to delete the specified key. 354 * @param obj object to evaluate ({@link Expression} or {@link String}) 355 * @return created fix command 356 */ 357 static FixCommand fixRemove(final Object obj) { 358 checkObject(obj); 359 return new FixCommand() { 360 @Override 361 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 362 final String key = evaluateObject(obj, p, matchingSelector); 363 return new ChangePropertyCommand(p, key, ""); 364 } 365 366 @Override 367 public String toString() { 368 return "fixRemove: " + obj; 369 } 370 }; 371 } 372 373 /** 374 * Creates a fixing command which executes a {@link ChangePropertyKeyCommand} on the specified keys. 375 * @param oldKey old key 376 * @param newKey new key 377 * @return created fix command 378 */ 379 static FixCommand fixChangeKey(final String oldKey, final String newKey) { 380 return new FixCommand() { 381 @Override 382 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 383 return new ChangePropertyKeyCommand(p, 384 TagCheck.insertArguments(matchingSelector, oldKey, p), 385 TagCheck.insertArguments(matchingSelector, newKey, p)); 386 } 387 388 @Override 389 public String toString() { 390 return "fixChangeKey: " + oldKey + " => " + newKey; 391 } 392 }; 393 } 394 } 395 396 final MultiMap<String, TagCheck> checks = new MultiMap<>(); 397 398 /** 399 * Result of {@link TagCheck#readMapCSS} 400 * @since 8936 401 */ 402 public static class ParseResult { 403 /** Checks successfully parsed */ 404 public final List<TagCheck> parseChecks; 405 /** Errors that occurred during parsing */ 406 public final Collection<Throwable> parseErrors; 407 408 /** 409 * Constructs a new {@code ParseResult}. 410 * @param parseChecks Checks successfully parsed 411 * @param parseErrors Errors that occurred during parsing 412 */ 413 public ParseResult(List<TagCheck> parseChecks, Collection<Throwable> parseErrors) { 414 this.parseChecks = parseChecks; 415 this.parseErrors = parseErrors; 416 } 417 } 418 419 /** 420 * Tag check. 421 */ 422 public static class TagCheck implements Predicate<OsmPrimitive> { 423 /** The selector of this {@code TagCheck} */ 424 protected final GroupedMapCSSRule rule; 425 /** Commands to apply in order to fix a matching primitive */ 426 protected final List<FixCommand> fixCommands = new ArrayList<>(); 427 /** Tags (or arbitraty strings) of alternatives to be presented to the user */ 428 protected final List<String> alternatives = new ArrayList<>(); 429 /** An {@link org.openstreetmap.josm.gui.mappaint.mapcss.Instruction.AssignmentInstruction}-{@link Severity} pair. 430 * Is evaluated on the matching primitive to give the error message. Map is checked to contain exactly one element. */ 431 protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>(); 432 /** Unit tests */ 433 protected final Map<String, Boolean> assertions = new HashMap<>(); 434 /** MapCSS Classes to set on matching primitives */ 435 protected final Set<String> setClassExpressions = new HashSet<>(); 436 /** Denotes whether the object should be deleted for fixing it */ 437 protected boolean deletion; 438 /** A string used to group similar tests */ 439 protected String group; 440 441 TagCheck(GroupedMapCSSRule rule) { 442 this.rule = rule; 443 } 444 445 private static final String POSSIBLE_THROWS = possibleThrows(); 446 447 static final String possibleThrows() { 448 StringBuilder sb = new StringBuilder(); 449 for (Severity s : Severity.values()) { 450 if (sb.length() > 0) { 451 sb.append('/'); 452 } 453 sb.append("throw") 454 .append(s.name().charAt(0)) 455 .append(s.name().substring(1).toLowerCase(Locale.ENGLISH)); 456 } 457 return sb.toString(); 458 } 459 460 static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) throws IllegalDataException { 461 final TagCheck check = new TagCheck(rule); 462 for (Instruction i : rule.declaration.instructions) { 463 if (i instanceof Instruction.AssignmentInstruction) { 464 final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i; 465 if (ai.isSetInstruction) { 466 check.setClassExpressions.add(ai.key); 467 continue; 468 } 469 try { 470 final String val = ai.val instanceof Expression 471 ? Optional.ofNullable(((Expression) ai.val).evaluate(new Environment())).map(Object::toString).orElse(null) 472 : ai.val instanceof String 473 ? (String) ai.val 474 : ai.val instanceof Keyword 475 ? ((Keyword) ai.val).val 476 : null; 477 if (ai.key.startsWith("throw")) { 478 try { 479 check.errors.put(ai, Severity.valueOf(ai.key.substring("throw".length()).toUpperCase(Locale.ENGLISH))); 480 } catch (IllegalArgumentException e) { 481 Logging.log(Logging.LEVEL_WARN, 482 "Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS+'.', e); 483 } 484 } else if ("fixAdd".equals(ai.key)) { 485 check.fixCommands.add(FixCommand.fixAdd(ai.val)); 486 } else if ("fixRemove".equals(ai.key)) { 487 CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")), 488 "Unexpected '='. Please only specify the key to remove in: " + ai); 489 check.fixCommands.add(FixCommand.fixRemove(ai.val)); 490 } else if (val != null && "fixChangeKey".equals(ai.key)) { 491 CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!"); 492 final String[] x = val.split("=>", 2); 493 check.fixCommands.add(FixCommand.fixChangeKey(Utils.removeWhiteSpaces(x[0]), Utils.removeWhiteSpaces(x[1]))); 494 } else if (val != null && "fixDeleteObject".equals(ai.key)) { 495 CheckParameterUtil.ensureThat("this".equals(val), "fixDeleteObject must be followed by 'this'"); 496 check.deletion = true; 497 } else if (val != null && "suggestAlternative".equals(ai.key)) { 498 check.alternatives.add(val); 499 } else if (val != null && "assertMatch".equals(ai.key)) { 500 check.assertions.put(val, Boolean.TRUE); 501 } else if (val != null && "assertNoMatch".equals(ai.key)) { 502 check.assertions.put(val, Boolean.FALSE); 503 } else if (val != null && "group".equals(ai.key)) { 504 check.group = val; 505 } else if (ai.key.startsWith("-")) { 506 Logging.debug("Ignoring extension instruction: " + ai.key + ": " + ai.val); 507 } else { 508 throw new IllegalDataException("Cannot add instruction " + ai.key + ": " + ai.val + '!'); 509 } 510 } catch (IllegalArgumentException e) { 511 throw new IllegalDataException(e); 512 } 513 } 514 } 515 if (check.errors.isEmpty() && check.setClassExpressions.isEmpty()) { 516 throw new IllegalDataException( 517 "No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors); 518 } else if (check.errors.size() > 1) { 519 throw new IllegalDataException( 520 "More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for " 521 + rule.selectors); 522 } 523 return check; 524 } 525 526 static ParseResult readMapCSS(Reader css) throws ParseException { 527 CheckParameterUtil.ensureParameterNotNull(css, "css"); 528 529 final MapCSSStyleSource source = new MapCSSStyleSource(""); 530 final MapCSSParser preprocessor = new MapCSSParser(css, MapCSSParser.LexicalState.PREPROCESSOR); 531 final StringReader mapcss = new StringReader(preprocessor.pp_root(source)); 532 final MapCSSParser parser = new MapCSSParser(mapcss, MapCSSParser.LexicalState.DEFAULT); 533 parser.sheet(source); 534 // Ignore "meta" rule(s) from external rules of JOSM wiki 535 source.removeMetaRules(); 536 // group rules with common declaration block 537 Map<Declaration, List<Selector>> g = new LinkedHashMap<>(); 538 for (MapCSSRule rule : source.rules) { 539 if (!g.containsKey(rule.declaration)) { 540 List<Selector> sels = new ArrayList<>(); 541 sels.add(rule.selector); 542 g.put(rule.declaration, sels); 543 } else { 544 g.get(rule.declaration).add(rule.selector); 545 } 546 } 547 List<TagCheck> parseChecks = new ArrayList<>(); 548 for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) { 549 try { 550 parseChecks.add(TagCheck.ofMapCSSRule( 551 new GroupedMapCSSRule(map.getValue(), map.getKey()))); 552 } catch (IllegalDataException e) { 553 Logging.error("Cannot add MapCss rule: "+e.getMessage()); 554 source.logError(e); 555 } 556 } 557 return new ParseResult(parseChecks, source.getErrors()); 558 } 559 560 @Override 561 public boolean test(OsmPrimitive primitive) { 562 // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker. 563 return whichSelectorMatchesPrimitive(primitive) != null; 564 } 565 566 Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) { 567 return whichSelectorMatchesEnvironment(new Environment(primitive)); 568 } 569 570 Selector whichSelectorMatchesEnvironment(Environment env) { 571 for (Selector i : rule.selectors) { 572 env.clearSelectorMatchingInformation(); 573 if (i.matches(env)) { 574 return i; 575 } 576 } 577 return null; 578 } 579 580 /** 581 * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the 582 * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}. 583 * @param matchingSelector matching selector 584 * @param index index 585 * @param type selector type ("key", "value" or "tag") 586 * @param p OSM primitive 587 * @return argument value, can be {@code null} 588 */ 589 static String determineArgument(OptimizedGeneralSelector matchingSelector, int index, String type, OsmPrimitive p) { 590 try { 591 final Condition c = matchingSelector.getConditions().get(index); 592 final Tag tag = c instanceof Condition.ToTagConvertable 593 ? ((Condition.ToTagConvertable) c).asTag(p) 594 : null; 595 if (tag == null) { 596 return null; 597 } else if ("key".equals(type)) { 598 return tag.getKey(); 599 } else if ("value".equals(type)) { 600 return tag.getValue(); 601 } else if ("tag".equals(type)) { 602 return tag.toString(); 603 } 604 } catch (IndexOutOfBoundsException ignore) { 605 Logging.debug(ignore); 606 } 607 return null; 608 } 609 610 /** 611 * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding 612 * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}. 613 * @param matchingSelector matching selector 614 * @param s any string 615 * @param p OSM primitive 616 * @return string with arguments inserted 617 */ 618 static String insertArguments(Selector matchingSelector, String s, OsmPrimitive p) { 619 if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) { 620 return insertArguments(((Selector.ChildOrParentSelector) matchingSelector).right, s, p); 621 } else if (s == null || !(matchingSelector instanceof Selector.OptimizedGeneralSelector)) { 622 return s; 623 } 624 final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s); 625 final StringBuffer sb = new StringBuffer(); 626 while (m.find()) { 627 final String argument = determineArgument((Selector.OptimizedGeneralSelector) matchingSelector, 628 Integer.parseInt(m.group(1)), m.group(2), p); 629 try { 630 // Perform replacement with null-safe + regex-safe handling 631 m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", "")); 632 } catch (IndexOutOfBoundsException | IllegalArgumentException e) { 633 Logging.log(Logging.LEVEL_ERROR, tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()), e); 634 } 635 } 636 m.appendTail(sb); 637 return sb.toString(); 638 } 639 640 /** 641 * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive} 642 * if the error is fixable, or {@code null} otherwise. 643 * 644 * @param p the primitive to construct the fix for 645 * @return the fix or {@code null} 646 */ 647 Command fixPrimitive(OsmPrimitive p) { 648 if (fixCommands.isEmpty() && !deletion) { 649 return null; 650 } 651 try { 652 final Selector matchingSelector = whichSelectorMatchesPrimitive(p); 653 Collection<Command> cmds = new LinkedList<>(); 654 for (FixCommand fixCommand : fixCommands) { 655 cmds.add(fixCommand.createCommand(p, matchingSelector)); 656 } 657 if (deletion && !p.isDeleted()) { 658 cmds.add(new DeleteCommand(p)); 659 } 660 return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds); 661 } catch (IllegalArgumentException e) { 662 Logging.error(e); 663 return null; 664 } 665 } 666 667 /** 668 * Constructs a (localized) message for this deprecation check. 669 * @param p OSM primitive 670 * 671 * @return a message 672 */ 673 String getMessage(OsmPrimitive p) { 674 if (errors.isEmpty()) { 675 // Return something to avoid NPEs 676 return rule.declaration.toString(); 677 } else { 678 final Object val = errors.keySet().iterator().next().val; 679 return String.valueOf( 680 val instanceof Expression 681 ? ((Expression) val).evaluate(new Environment(p)) 682 : val 683 ); 684 } 685 } 686 687 /** 688 * Constructs a (localized) description for this deprecation check. 689 * @param p OSM primitive 690 * 691 * @return a description (possibly with alternative suggestions) 692 * @see #getDescriptionForMatchingSelector 693 */ 694 String getDescription(OsmPrimitive p) { 695 if (alternatives.isEmpty()) { 696 return getMessage(p); 697 } else { 698 /* I18N: {0} is the test error message and {1} is an alternative */ 699 return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives)); 700 } 701 } 702 703 /** 704 * Constructs a (localized) description for this deprecation check 705 * where any placeholders are replaced by values of the matched selector. 706 * 707 * @param matchingSelector matching selector 708 * @param p OSM primitive 709 * @return a description (possibly with alternative suggestions) 710 */ 711 String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) { 712 return insertArguments(matchingSelector, getDescription(p), p); 713 } 714 715 Severity getSeverity() { 716 return errors.isEmpty() ? null : errors.values().iterator().next(); 717 } 718 719 @Override 720 public String toString() { 721 return getDescription(null); 722 } 723 724 /** 725 * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error. 726 * 727 * @param p the primitive to construct the error for 728 * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error. 729 */ 730 TestError getErrorForPrimitive(OsmPrimitive p) { 731 final Environment env = new Environment(p); 732 return getErrorForPrimitive(p, whichSelectorMatchesEnvironment(env), env, null); 733 } 734 735 TestError getErrorForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env, Test tester) { 736 if (matchingSelector != null && !errors.isEmpty()) { 737 final Command fix = fixPrimitive(p); 738 final String description = getDescriptionForMatchingSelector(p, matchingSelector); 739 final String description1 = group == null ? description : group; 740 final String description2 = group == null ? null : description; 741 final List<OsmPrimitive> primitives; 742 if (env.child instanceof OsmPrimitive) { 743 primitives = Arrays.asList(p, (OsmPrimitive) env.child); 744 } else { 745 primitives = Collections.singletonList(p); 746 } 747 final TestError.Builder error = TestError.builder(tester, getSeverity(), 3000) 748 .messageWithManuallyTranslatedDescription(description1, description2, matchingSelector.toString()) 749 .primitives(primitives); 750 if (fix != null) { 751 return error.fix(() -> fix).build(); 752 } else { 753 return error.build(); 754 } 755 } else { 756 return null; 757 } 758 } 759 760 /** 761 * Returns the set of tagchecks on which this check depends on. 762 * @param schecks the collection of tagcheks to search in 763 * @return the set of tagchecks on which this check depends on 764 * @since 7881 765 */ 766 public Set<TagCheck> getTagCheckDependencies(Collection<TagCheck> schecks) { 767 Set<TagCheck> result = new HashSet<>(); 768 Set<String> classes = getClassesIds(); 769 if (schecks != null && !classes.isEmpty()) { 770 for (TagCheck tc : schecks) { 771 if (this.equals(tc)) { 772 continue; 773 } 774 for (String id : tc.setClassExpressions) { 775 if (classes.contains(id)) { 776 result.add(tc); 777 break; 778 } 779 } 780 } 781 } 782 return result; 783 } 784 785 /** 786 * Returns the list of ids of all MapCSS classes referenced in the rule selectors. 787 * @return the list of ids of all MapCSS classes referenced in the rule selectors 788 * @since 7881 789 */ 790 public Set<String> getClassesIds() { 791 Set<String> result = new HashSet<>(); 792 for (Selector s : rule.selectors) { 793 if (s instanceof AbstractSelector) { 794 for (Condition c : ((AbstractSelector) s).getConditions()) { 795 if (c instanceof ClassCondition) { 796 result.add(((ClassCondition) c).id); 797 } 798 } 799 } 800 } 801 return result; 802 } 803 } 804 805 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker { 806 public final GroupedMapCSSRule rule; 807 808 MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) { 809 this.rule = rule; 810 } 811 812 @Override 813 public synchronized boolean equals(Object obj) { 814 return super.equals(obj) 815 || (obj instanceof TagCheck && rule.equals(((TagCheck) obj).rule)) 816 || (obj instanceof GroupedMapCSSRule && rule.equals(obj)); 817 } 818 819 @Override 820 public synchronized int hashCode() { 821 return Objects.hash(super.hashCode(), rule); 822 } 823 824 @Override 825 public String toString() { 826 return "MapCSSTagCheckerAndRule [rule=" + rule + ']'; 827 } 828 } 829 830 /** 831 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}. 832 * @param p The OSM primitive 833 * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned 834 * @return all errors for the given primitive, with or without those of "info" severity 835 */ 836 public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) { 837 final List<TestError> res = new ArrayList<>(); 838 if (indexData == null) 839 indexData = new IndexData(checks); 840 841 MapCSSRuleIndex matchingRuleIndex = indexData.get(p); 842 843 Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null); 844 // the declaration indices are sorted, so it suffices to save the last used index 845 Declaration lastDeclUsed = null; 846 847 Iterator<MapCSSRule> candidates = matchingRuleIndex.getRuleCandidates(p); 848 while (candidates.hasNext()) { 849 MapCSSRule r = candidates.next(); 850 env.clearSelectorMatchingInformation(); 851 if (r.selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector) 852 TagCheck check = indexData.getCheck(r); 853 if (check != null) { 854 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity; 855 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class 856 if (ignoreError && check.setClassExpressions.isEmpty()) { 857 continue; 858 } 859 if (r.declaration == lastDeclUsed) 860 continue; // don't apply one declaration more than once 861 lastDeclUsed = r.declaration; 862 863 r.declaration.execute(env); 864 if (!ignoreError && !check.errors.isEmpty()) { 865 final TestError error = check.getErrorForPrimitive(p, r.selector, env, new MapCSSTagCheckerAndRule(check.rule)); 866 if (error != null) { 867 res.add(error); 868 } 869 } 870 871 } 872 } 873 } 874 return res; 875 } 876 877 private static Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity, 878 Collection<Set<TagCheck>> checksCol) { 879 final List<TestError> r = new ArrayList<>(); 880 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null); 881 for (Set<TagCheck> schecks : checksCol) { 882 for (TagCheck check : schecks) { 883 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity; 884 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class 885 if (ignoreError && check.setClassExpressions.isEmpty()) { 886 continue; 887 } 888 final Selector selector = check.whichSelectorMatchesEnvironment(env); 889 if (selector != null) { 890 check.rule.declaration.execute(env); 891 if (!ignoreError && !check.errors.isEmpty()) { 892 final TestError error = check.getErrorForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule)); 893 if (error != null) { 894 r.add(error); 895 } 896 } 897 } 898 } 899 } 900 return r; 901 } 902 903 /** 904 * Visiting call for primitives. 905 * 906 * @param p The primitive to inspect. 907 */ 908 @Override 909 public void check(OsmPrimitive p) { 910 errors.addAll(getErrorsForPrimitive(p, ValidatorPrefHelper.PREF_OTHER.get())); 911 } 912 913 /** 914 * Adds a new MapCSS config file from the given URL. 915 * @param url The unique URL of the MapCSS config file 916 * @return List of tag checks and parsing errors, or null 917 * @throws ParseException if the config file does not match MapCSS syntax 918 * @throws IOException if any I/O error occurs 919 * @since 7275 920 */ 921 public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException { 922 CheckParameterUtil.ensureParameterNotNull(url, "url"); 923 ParseResult result; 924 try (CachedFile cache = new CachedFile(url); 925 InputStream zip = cache.findZipEntryInputStream("validator.mapcss", ""); 926 InputStream s = zip != null ? zip : cache.getInputStream(); 927 Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) { 928 if (zip != null) 929 I18n.addTexts(cache.getFile()); 930 result = TagCheck.readMapCSS(reader); 931 checks.remove(url); 932 checks.putAll(url, result.parseChecks); 933 indexData = null; 934 // Check assertions, useful for development of local files 935 if (Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url)) { 936 for (String msg : checkAsserts(result.parseChecks)) { 937 Logging.warn(msg); 938 } 939 } 940 } 941 return result; 942 } 943 944 @Override 945 public synchronized void initialize() throws Exception { 946 checks.clear(); 947 indexData = null; 948 for (SourceEntry source : new ValidatorPrefHelper().get()) { 949 if (!source.active) { 950 continue; 951 } 952 String i = source.url; 953 try { 954 if (!i.startsWith("resource:")) { 955 Logging.info(tr("Adding {0} to tag checker", i)); 956 } else if (Logging.isDebugEnabled()) { 957 Logging.debug(tr("Adding {0} to tag checker", i)); 958 } 959 addMapCSS(i); 960 if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) { 961 FileWatcher.getDefaultInstance().registerSource(source); 962 } 963 } catch (IOException | IllegalStateException | IllegalArgumentException ex) { 964 Logging.warn(tr("Failed to add {0} to tag checker", i)); 965 Logging.log(Logging.LEVEL_WARN, ex); 966 } catch (ParseException | TokenMgrError ex) { 967 Logging.warn(tr("Failed to add {0} to tag checker", i)); 968 Logging.warn(ex); 969 } 970 } 971 } 972 973 private static Method getFunctionMethod(String method) { 974 try { 975 return Functions.class.getDeclaredMethod(method, Environment.class, String.class); 976 } catch (NoSuchMethodException | SecurityException e) { 977 Logging.error(e); 978 return null; 979 } 980 } 981 982 private static Optional<String> getFirstInsideCountry(TagCheck check, Method insideMethod) { 983 return check.rule.selectors.stream() 984 .filter(s -> s instanceof GeneralSelector) 985 .flatMap(s -> ((GeneralSelector) s).getConditions().stream()) 986 .filter(c -> c instanceof ExpressionCondition) 987 .map(c -> ((ExpressionCondition) c).getExpression()) 988 .filter(c -> c instanceof ParameterFunction) 989 .map(c -> (ParameterFunction) c) 990 .filter(c -> c.getMethod().equals(insideMethod)) 991 .flatMap(c -> c.getArgs().stream()) 992 .filter(e -> e instanceof LiteralExpression) 993 .map(e -> ((LiteralExpression) e).getLiteral()) 994 .filter(l -> l instanceof String) 995 .map(l -> ((String) l).split(",")[0]) 996 .findFirst(); 997 } 998 999 private static LatLon getLocation(TagCheck check, Method insideMethod) { 1000 Optional<String> inside = getFirstInsideCountry(check, insideMethod); 1001 if (inside.isPresent()) { 1002 GeoPropertyIndex<Boolean> index = Territories.getGeoPropertyIndex(inside.get()); 1003 if (index != null) { 1004 GeoProperty<Boolean> prop = index.getGeoProperty(); 1005 if (prop instanceof DefaultGeoProperty) { 1006 Rectangle bounds = ((DefaultGeoProperty) prop).getArea().getBounds(); 1007 return new LatLon(bounds.getCenterY(), bounds.getCenterX()); 1008 } 1009 } 1010 } 1011 return LatLon.ZERO; 1012 } 1013 1014 /** 1015 * Checks that rule assertions are met for the given set of TagChecks. 1016 * @param schecks The TagChecks for which assertions have to be checked 1017 * @return A set of error messages, empty if all assertions are met 1018 * @since 7356 1019 */ 1020 public Set<String> checkAsserts(final Collection<TagCheck> schecks) { 1021 Set<String> assertionErrors = new LinkedHashSet<>(); 1022 final Method insideMethod = getFunctionMethod("inside"); 1023 final DataSet ds = new DataSet(); 1024 for (final TagCheck check : schecks) { 1025 Logging.debug("Check: {0}", check); 1026 for (final Map.Entry<String, Boolean> i : check.assertions.entrySet()) { 1027 Logging.debug("- Assertion: {0}", i); 1028 final OsmPrimitive p = OsmUtils.createPrimitive(i.getKey(), getLocation(check, insideMethod), true); 1029 // Build minimal ordered list of checks to run to test the assertion 1030 List<Set<TagCheck>> checksToRun = new ArrayList<>(); 1031 Set<TagCheck> checkDependencies = check.getTagCheckDependencies(schecks); 1032 if (!checkDependencies.isEmpty()) { 1033 checksToRun.add(checkDependencies); 1034 } 1035 checksToRun.add(Collections.singleton(check)); 1036 // Add primitive to dataset to avoid DataIntegrityProblemException when evaluating selectors 1037 addPrimitive(ds, p); 1038 final Collection<TestError> pErrors = getErrorsForPrimitive(p, true, checksToRun); 1039 Logging.debug("- Errors: {0}", pErrors); 1040 @SuppressWarnings({"EqualsBetweenInconvertibleTypes", "EqualsIncompatibleType"}) 1041 final boolean isError = pErrors.stream().anyMatch(e -> e.getTester().equals(check.rule)); 1042 if (isError != i.getValue()) { 1043 final String error = MessageFormat.format("Expecting test ''{0}'' (i.e., {1}) to {2} {3} (i.e., {4})", 1044 check.getMessage(p), check.rule.selectors, i.getValue() ? "match" : "not match", i.getKey(), p.getKeys()); 1045 assertionErrors.add(error); 1046 } 1047 ds.removePrimitive(p); 1048 } 1049 } 1050 return assertionErrors; 1051 } 1052 1053 private static void addPrimitive(DataSet ds, OsmPrimitive p) { 1054 if (p instanceof Way) { 1055 ((Way) p).getNodes().forEach(n -> addPrimitive(ds, n)); 1056 } else if (p instanceof Relation) { 1057 ((Relation) p).getMembers().forEach(m -> addPrimitive(ds, m.getMember())); 1058 } 1059 ds.addPrimitive(p); 1060 } 1061 1062 @Override 1063 public synchronized int hashCode() { 1064 return Objects.hash(super.hashCode(), checks); 1065 } 1066 1067 @Override 1068 public synchronized boolean equals(Object obj) { 1069 if (this == obj) return true; 1070 if (obj == null || getClass() != obj.getClass()) return false; 1071 if (!super.equals(obj)) return false; 1072 MapCSSTagChecker that = (MapCSSTagChecker) obj; 1073 return Objects.equals(checks, that.checks); 1074 } 1075 1076 /** 1077 * Reload tagchecker rule. 1078 * @param rule tagchecker rule to reload 1079 * @since 12825 1080 */ 1081 public static void reloadRule(SourceEntry rule) { 1082 MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class); 1083 if (tagChecker != null) { 1084 try { 1085 tagChecker.addMapCSS(rule.url); 1086 } catch (IOException | ParseException | TokenMgrError e) { 1087 Logging.warn(e); 1088 } 1089 } 1090 } 1091 1092 @Override 1093 public void startTest(ProgressMonitor progressMonitor) { 1094 super.startTest(progressMonitor); 1095 super.setShowElements(true); 1096 if (indexData == null) { 1097 indexData = new IndexData(checks); 1098 } 1099 } 1100 1101 @Override 1102 public void endTest() { 1103 super.endTest(); 1104 // no need to keep the index, it is quickly build and doubles the memory needs 1105 indexData = null; 1106 } 1107 1108}