001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.awt.Rectangle;
007import java.io.BufferedReader;
008import java.io.IOException;
009import java.io.InputStream;
010import java.io.Reader;
011import java.io.StringReader;
012import java.lang.reflect.Method;
013import java.text.MessageFormat;
014import java.util.ArrayList;
015import java.util.Arrays;
016import java.util.Collection;
017import java.util.Collections;
018import java.util.HashMap;
019import java.util.HashSet;
020import java.util.Iterator;
021import java.util.LinkedHashMap;
022import java.util.LinkedHashSet;
023import java.util.LinkedList;
024import java.util.List;
025import java.util.Locale;
026import java.util.Map;
027import java.util.Objects;
028import java.util.Optional;
029import java.util.Set;
030import java.util.function.Predicate;
031import java.util.regex.Matcher;
032import java.util.regex.Pattern;
033
034import org.openstreetmap.josm.command.ChangePropertyCommand;
035import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
036import org.openstreetmap.josm.command.Command;
037import org.openstreetmap.josm.command.DeleteCommand;
038import org.openstreetmap.josm.command.SequenceCommand;
039import org.openstreetmap.josm.data.coor.LatLon;
040import org.openstreetmap.josm.data.osm.DataSet;
041import org.openstreetmap.josm.data.osm.INode;
042import org.openstreetmap.josm.data.osm.IRelation;
043import org.openstreetmap.josm.data.osm.IWay;
044import org.openstreetmap.josm.data.osm.OsmPrimitive;
045import org.openstreetmap.josm.data.osm.OsmUtils;
046import org.openstreetmap.josm.data.osm.Relation;
047import org.openstreetmap.josm.data.osm.Tag;
048import org.openstreetmap.josm.data.osm.Way;
049import org.openstreetmap.josm.data.preferences.sources.SourceEntry;
050import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
051import org.openstreetmap.josm.data.validation.OsmValidator;
052import org.openstreetmap.josm.data.validation.Severity;
053import org.openstreetmap.josm.data.validation.Test;
054import org.openstreetmap.josm.data.validation.TestError;
055import org.openstreetmap.josm.gui.mappaint.Environment;
056import org.openstreetmap.josm.gui.mappaint.Keyword;
057import org.openstreetmap.josm.gui.mappaint.MultiCascade;
058import org.openstreetmap.josm.gui.mappaint.mapcss.Condition;
059import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ClassCondition;
060import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ExpressionCondition;
061import org.openstreetmap.josm.gui.mappaint.mapcss.Expression;
062import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.Functions;
063import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.ParameterFunction;
064import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction;
065import org.openstreetmap.josm.gui.mappaint.mapcss.LiteralExpression;
066import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
067import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration;
068import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
069import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource.MapCSSRuleIndex;
070import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
071import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.AbstractSelector;
072import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector;
073import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.OptimizedGeneralSelector;
074import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser;
075import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
076import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError;
077import org.openstreetmap.josm.gui.progress.ProgressMonitor;
078import org.openstreetmap.josm.io.CachedFile;
079import org.openstreetmap.josm.io.FileWatcher;
080import org.openstreetmap.josm.io.IllegalDataException;
081import org.openstreetmap.josm.io.UTFInputStreamReader;
082import org.openstreetmap.josm.spi.preferences.Config;
083import org.openstreetmap.josm.tools.CheckParameterUtil;
084import org.openstreetmap.josm.tools.DefaultGeoProperty;
085import org.openstreetmap.josm.tools.GeoProperty;
086import org.openstreetmap.josm.tools.GeoPropertyIndex;
087import org.openstreetmap.josm.tools.I18n;
088import org.openstreetmap.josm.tools.JosmRuntimeException;
089import org.openstreetmap.josm.tools.Logging;
090import org.openstreetmap.josm.tools.MultiMap;
091import org.openstreetmap.josm.tools.Territories;
092import org.openstreetmap.josm.tools.Utils;
093
094/**
095 * MapCSS-based tag checker/fixer.
096 * @since 6506
097 */
098public class MapCSSTagChecker extends Test.TagTest {
099    IndexData indexData;
100
101    /**
102     * Helper class to store indexes of rules.
103     * @author Gerd
104     *
105     */
106    private static class IndexData {
107        final Map<MapCSSRule, TagCheck> ruleToCheckMap = new HashMap<>();
108
109        /**
110         * Rules for nodes
111         */
112        final MapCSSRuleIndex nodeRules = new MapCSSRuleIndex();
113        /**
114         * Rules for ways without tag area=no
115         */
116        final MapCSSRuleIndex wayRules = new MapCSSRuleIndex();
117        /**
118         * Rules for ways with tag area=no
119         */
120        final MapCSSRuleIndex wayNoAreaRules = new MapCSSRuleIndex();
121        /**
122         * Rules for relations that are not multipolygon relations
123         */
124        final MapCSSRuleIndex relationRules = new MapCSSRuleIndex();
125        /**
126         * Rules for multipolygon relations
127         */
128        final MapCSSRuleIndex multipolygonRules = new MapCSSRuleIndex();
129
130        IndexData(MultiMap<String, TagCheck> checks) {
131            buildIndex(checks);
132        }
133
134        private void buildIndex(MultiMap<String, TagCheck> checks) {
135            List<TagCheck> allChecks = new ArrayList<>();
136            for (Set<TagCheck> cs : checks.values()) {
137                allChecks.addAll(cs);
138            }
139
140            ruleToCheckMap.clear();
141            nodeRules.clear();
142            wayRules.clear();
143            wayNoAreaRules.clear();
144            relationRules.clear();
145            multipolygonRules.clear();
146
147            // optimization: filter rules for different primitive types
148            for (TagCheck c : allChecks) {
149                for (Selector s : c.rule.selectors) {
150                    // find the rightmost selector, this must be a GeneralSelector
151                    Selector selRightmost = s;
152                    while (selRightmost instanceof Selector.ChildOrParentSelector) {
153                        selRightmost = ((Selector.ChildOrParentSelector) selRightmost).right;
154                    }
155                    MapCSSRule optRule = new MapCSSRule(s.optimizedBaseCheck(), c.rule.declaration);
156
157                    ruleToCheckMap.put(optRule, c);
158                    final String base = ((GeneralSelector) selRightmost).getBase();
159                    switch (base) {
160                    case Selector.BASE_NODE:
161                        nodeRules.add(optRule);
162                        break;
163                    case Selector.BASE_WAY:
164                        wayNoAreaRules.add(optRule);
165                        wayRules.add(optRule);
166                        break;
167                    case Selector.BASE_AREA:
168                        wayRules.add(optRule);
169                        multipolygonRules.add(optRule);
170                        break;
171                    case Selector.BASE_RELATION:
172                        relationRules.add(optRule);
173                        multipolygonRules.add(optRule);
174                        break;
175                    case Selector.BASE_ANY:
176                        nodeRules.add(optRule);
177                        wayRules.add(optRule);
178                        wayNoAreaRules.add(optRule);
179                        relationRules.add(optRule);
180                        multipolygonRules.add(optRule);
181                        break;
182                    case Selector.BASE_CANVAS:
183                    case Selector.BASE_META:
184                    case Selector.BASE_SETTING:
185                        break;
186                    default:
187                        final RuntimeException e = new JosmRuntimeException(MessageFormat.format("Unknown MapCSS base selector {0}", base));
188                        Logging.warn(tr("Failed to index validator rules. Error was: {0}", e.getMessage()));
189                        Logging.error(e);
190                    }
191                }
192            }
193            nodeRules.initIndex();
194            wayRules.initIndex();
195            wayNoAreaRules.initIndex();
196            relationRules.initIndex();
197            multipolygonRules.initIndex();
198        }
199
200        /**
201         * Get the index of rules for the given primitive.
202         * @param p the primitve
203         * @return index of rules for the given primitive
204         */
205        public MapCSSRuleIndex get(OsmPrimitive p) {
206            if (p instanceof INode) {
207                return nodeRules;
208            } else if (p instanceof IWay) {
209                if (OsmUtils.isFalse(p.get("area"))) {
210                    return wayNoAreaRules;
211                } else {
212                    return wayRules;
213                }
214            } else if (p instanceof IRelation) {
215                if (((IRelation<?>) p).isMultipolygon()) {
216                    return multipolygonRules;
217                } else {
218                    return relationRules;
219                }
220            } else {
221                throw new IllegalArgumentException("Unsupported type: " + p);
222            }
223        }
224
225        /**
226         * return the TagCheck for which the given indexed rule was created.
227         * @param rule an indexed rule
228         * @return the original TagCheck
229         */
230        public TagCheck getCheck(MapCSSRule rule) {
231            return ruleToCheckMap.get(rule);
232        }
233    }
234
235    /**
236    * A grouped MapCSSRule with multiple selectors for a single declaration.
237    * @see MapCSSRule
238    */
239    public static class GroupedMapCSSRule {
240        /** MapCSS selectors **/
241        public final List<Selector> selectors;
242        /** MapCSS declaration **/
243        public final Declaration declaration;
244
245        /**
246         * Constructs a new {@code GroupedMapCSSRule}.
247         * @param selectors MapCSS selectors
248         * @param declaration MapCSS declaration
249         */
250        public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) {
251            this.selectors = selectors;
252            this.declaration = declaration;
253        }
254
255        @Override
256        public int hashCode() {
257            return Objects.hash(selectors, declaration);
258        }
259
260        @Override
261        public boolean equals(Object obj) {
262            if (this == obj) return true;
263            if (obj == null || getClass() != obj.getClass()) return false;
264            GroupedMapCSSRule that = (GroupedMapCSSRule) obj;
265            return Objects.equals(selectors, that.selectors) &&
266                    Objects.equals(declaration, that.declaration);
267        }
268
269        @Override
270        public String toString() {
271            return "GroupedMapCSSRule [selectors=" + selectors + ", declaration=" + declaration + ']';
272        }
273    }
274
275    /**
276     * The preference key for tag checker source entries.
277     * @since 6670
278     */
279    public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
280
281    /**
282     * Constructs a new {@code MapCSSTagChecker}.
283     */
284    public MapCSSTagChecker() {
285        super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
286    }
287
288    /**
289     * Represents a fix to a validation test. The fixing {@link Command} can be obtained by {@link #createCommand(OsmPrimitive, Selector)}.
290     */
291    @FunctionalInterface
292    interface FixCommand {
293        /**
294         * Creates the fixing {@link Command} for the given primitive. The {@code matchingSelector} is used to evaluate placeholders
295         * (cf. {@link MapCSSTagChecker.TagCheck#insertArguments(Selector, String, OsmPrimitive)}).
296         * @param p OSM primitive
297         * @param matchingSelector  matching selector
298         * @return fix command
299         */
300        Command createCommand(OsmPrimitive p, Selector matchingSelector);
301
302        /**
303         * Checks that object is either an {@link Expression} or a {@link String}.
304         * @param obj object to check
305         * @throws IllegalArgumentException if object is not an {@code Expression} or a {@code String}
306         */
307        static void checkObject(final Object obj) {
308            CheckParameterUtil.ensureThat(obj instanceof Expression || obj instanceof String,
309                    () -> "instance of Exception or String expected, but got " + obj);
310        }
311
312        /**
313         * Evaluates given object as {@link Expression} or {@link String} on the matched {@link OsmPrimitive} and {@code matchingSelector}.
314         * @param obj object to evaluate ({@link Expression} or {@link String})
315         * @param p OSM primitive
316         * @param matchingSelector matching selector
317         * @return result string
318         */
319        static String evaluateObject(final Object obj, final OsmPrimitive p, final Selector matchingSelector) {
320            final String s;
321            if (obj instanceof Expression) {
322                s = (String) ((Expression) obj).evaluate(new Environment(p));
323            } else if (obj instanceof String) {
324                s = (String) obj;
325            } else {
326                return null;
327            }
328            return TagCheck.insertArguments(matchingSelector, s, p);
329        }
330
331        /**
332         * Creates a fixing command which executes a {@link ChangePropertyCommand} on the specified tag.
333         * @param obj object to evaluate ({@link Expression} or {@link String})
334         * @return created fix command
335         */
336        static FixCommand fixAdd(final Object obj) {
337            checkObject(obj);
338            return new FixCommand() {
339                @Override
340                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
341                    final Tag tag = Tag.ofString(evaluateObject(obj, p, matchingSelector));
342                    return new ChangePropertyCommand(p, tag.getKey(), tag.getValue());
343                }
344
345                @Override
346                public String toString() {
347                    return "fixAdd: " + obj;
348                }
349            };
350        }
351
352        /**
353         * Creates a fixing command which executes a {@link ChangePropertyCommand} to delete the specified key.
354         * @param obj object to evaluate ({@link Expression} or {@link String})
355         * @return created fix command
356         */
357        static FixCommand fixRemove(final Object obj) {
358            checkObject(obj);
359            return new FixCommand() {
360                @Override
361                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
362                    final String key = evaluateObject(obj, p, matchingSelector);
363                    return new ChangePropertyCommand(p, key, "");
364                }
365
366                @Override
367                public String toString() {
368                    return "fixRemove: " + obj;
369                }
370            };
371        }
372
373        /**
374         * Creates a fixing command which executes a {@link ChangePropertyKeyCommand} on the specified keys.
375         * @param oldKey old key
376         * @param newKey new key
377         * @return created fix command
378         */
379        static FixCommand fixChangeKey(final String oldKey, final String newKey) {
380            return new FixCommand() {
381                @Override
382                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
383                    return new ChangePropertyKeyCommand(p,
384                            TagCheck.insertArguments(matchingSelector, oldKey, p),
385                            TagCheck.insertArguments(matchingSelector, newKey, p));
386                }
387
388                @Override
389                public String toString() {
390                    return "fixChangeKey: " + oldKey + " => " + newKey;
391                }
392            };
393        }
394    }
395
396    final MultiMap<String, TagCheck> checks = new MultiMap<>();
397
398    /**
399     * Result of {@link TagCheck#readMapCSS}
400     * @since 8936
401     */
402    public static class ParseResult {
403        /** Checks successfully parsed */
404        public final List<TagCheck> parseChecks;
405        /** Errors that occurred during parsing */
406        public final Collection<Throwable> parseErrors;
407
408        /**
409         * Constructs a new {@code ParseResult}.
410         * @param parseChecks Checks successfully parsed
411         * @param parseErrors Errors that occurred during parsing
412         */
413        public ParseResult(List<TagCheck> parseChecks, Collection<Throwable> parseErrors) {
414            this.parseChecks = parseChecks;
415            this.parseErrors = parseErrors;
416        }
417    }
418
419    /**
420     * Tag check.
421     */
422    public static class TagCheck implements Predicate<OsmPrimitive> {
423        /** The selector of this {@code TagCheck} */
424        protected final GroupedMapCSSRule rule;
425        /** Commands to apply in order to fix a matching primitive */
426        protected final List<FixCommand> fixCommands = new ArrayList<>();
427        /** Tags (or arbitraty strings) of alternatives to be presented to the user */
428        protected final List<String> alternatives = new ArrayList<>();
429        /** An {@link org.openstreetmap.josm.gui.mappaint.mapcss.Instruction.AssignmentInstruction}-{@link Severity} pair.
430         * Is evaluated on the matching primitive to give the error message. Map is checked to contain exactly one element. */
431        protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>();
432        /** Unit tests */
433        protected final Map<String, Boolean> assertions = new HashMap<>();
434        /** MapCSS Classes to set on matching primitives */
435        protected final Set<String> setClassExpressions = new HashSet<>();
436        /** Denotes whether the object should be deleted for fixing it */
437        protected boolean deletion;
438        /** A string used to group similar tests */
439        protected String group;
440
441        TagCheck(GroupedMapCSSRule rule) {
442            this.rule = rule;
443        }
444
445        private static final String POSSIBLE_THROWS = possibleThrows();
446
447        static final String possibleThrows() {
448            StringBuilder sb = new StringBuilder();
449            for (Severity s : Severity.values()) {
450                if (sb.length() > 0) {
451                    sb.append('/');
452                }
453                sb.append("throw")
454                .append(s.name().charAt(0))
455                .append(s.name().substring(1).toLowerCase(Locale.ENGLISH));
456            }
457            return sb.toString();
458        }
459
460        static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) throws IllegalDataException {
461            final TagCheck check = new TagCheck(rule);
462            for (Instruction i : rule.declaration.instructions) {
463                if (i instanceof Instruction.AssignmentInstruction) {
464                    final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i;
465                    if (ai.isSetInstruction) {
466                        check.setClassExpressions.add(ai.key);
467                        continue;
468                    }
469                    try {
470                        final String val = ai.val instanceof Expression
471                                ? Optional.ofNullable(((Expression) ai.val).evaluate(new Environment())).map(Object::toString).orElse(null)
472                                : ai.val instanceof String
473                                ? (String) ai.val
474                                : ai.val instanceof Keyword
475                                ? ((Keyword) ai.val).val
476                                : null;
477                        if (ai.key.startsWith("throw")) {
478                            try {
479                                check.errors.put(ai, Severity.valueOf(ai.key.substring("throw".length()).toUpperCase(Locale.ENGLISH)));
480                            } catch (IllegalArgumentException e) {
481                                Logging.log(Logging.LEVEL_WARN,
482                                        "Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS+'.', e);
483                            }
484                        } else if ("fixAdd".equals(ai.key)) {
485                            check.fixCommands.add(FixCommand.fixAdd(ai.val));
486                        } else if ("fixRemove".equals(ai.key)) {
487                            CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")),
488                                    "Unexpected '='. Please only specify the key to remove in: " + ai);
489                            check.fixCommands.add(FixCommand.fixRemove(ai.val));
490                        } else if (val != null && "fixChangeKey".equals(ai.key)) {
491                            CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!");
492                            final String[] x = val.split("=>", 2);
493                            check.fixCommands.add(FixCommand.fixChangeKey(Utils.removeWhiteSpaces(x[0]), Utils.removeWhiteSpaces(x[1])));
494                        } else if (val != null && "fixDeleteObject".equals(ai.key)) {
495                            CheckParameterUtil.ensureThat("this".equals(val), "fixDeleteObject must be followed by 'this'");
496                            check.deletion = true;
497                        } else if (val != null && "suggestAlternative".equals(ai.key)) {
498                            check.alternatives.add(val);
499                        } else if (val != null && "assertMatch".equals(ai.key)) {
500                            check.assertions.put(val, Boolean.TRUE);
501                        } else if (val != null && "assertNoMatch".equals(ai.key)) {
502                            check.assertions.put(val, Boolean.FALSE);
503                        } else if (val != null && "group".equals(ai.key)) {
504                            check.group = val;
505                        } else if (ai.key.startsWith("-")) {
506                            Logging.debug("Ignoring extension instruction: " + ai.key + ": " + ai.val);
507                        } else {
508                            throw new IllegalDataException("Cannot add instruction " + ai.key + ": " + ai.val + '!');
509                        }
510                    } catch (IllegalArgumentException e) {
511                        throw new IllegalDataException(e);
512                    }
513                }
514            }
515            if (check.errors.isEmpty() && check.setClassExpressions.isEmpty()) {
516                throw new IllegalDataException(
517                        "No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors);
518            } else if (check.errors.size() > 1) {
519                throw new IllegalDataException(
520                        "More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for "
521                                + rule.selectors);
522            }
523            return check;
524        }
525
526        static ParseResult readMapCSS(Reader css) throws ParseException {
527            CheckParameterUtil.ensureParameterNotNull(css, "css");
528
529            final MapCSSStyleSource source = new MapCSSStyleSource("");
530            final MapCSSParser preprocessor = new MapCSSParser(css, MapCSSParser.LexicalState.PREPROCESSOR);
531            final StringReader mapcss = new StringReader(preprocessor.pp_root(source));
532            final MapCSSParser parser = new MapCSSParser(mapcss, MapCSSParser.LexicalState.DEFAULT);
533            parser.sheet(source);
534            // Ignore "meta" rule(s) from external rules of JOSM wiki
535            source.removeMetaRules();
536            // group rules with common declaration block
537            Map<Declaration, List<Selector>> g = new LinkedHashMap<>();
538            for (MapCSSRule rule : source.rules) {
539                if (!g.containsKey(rule.declaration)) {
540                    List<Selector> sels = new ArrayList<>();
541                    sels.add(rule.selector);
542                    g.put(rule.declaration, sels);
543                } else {
544                    g.get(rule.declaration).add(rule.selector);
545                }
546            }
547            List<TagCheck> parseChecks = new ArrayList<>();
548            for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) {
549                try {
550                    parseChecks.add(TagCheck.ofMapCSSRule(
551                            new GroupedMapCSSRule(map.getValue(), map.getKey())));
552                } catch (IllegalDataException e) {
553                    Logging.error("Cannot add MapCss rule: "+e.getMessage());
554                    source.logError(e);
555                }
556            }
557            return new ParseResult(parseChecks, source.getErrors());
558        }
559
560        @Override
561        public boolean test(OsmPrimitive primitive) {
562            // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker.
563            return whichSelectorMatchesPrimitive(primitive) != null;
564        }
565
566        Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) {
567            return whichSelectorMatchesEnvironment(new Environment(primitive));
568        }
569
570        Selector whichSelectorMatchesEnvironment(Environment env) {
571            for (Selector i : rule.selectors) {
572                env.clearSelectorMatchingInformation();
573                if (i.matches(env)) {
574                    return i;
575                }
576            }
577            return null;
578        }
579
580        /**
581         * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the
582         * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}.
583         * @param matchingSelector matching selector
584         * @param index index
585         * @param type selector type ("key", "value" or "tag")
586         * @param p OSM primitive
587         * @return argument value, can be {@code null}
588         */
589        static String determineArgument(OptimizedGeneralSelector matchingSelector, int index, String type, OsmPrimitive p) {
590            try {
591                final Condition c = matchingSelector.getConditions().get(index);
592                final Tag tag = c instanceof Condition.ToTagConvertable
593                        ? ((Condition.ToTagConvertable) c).asTag(p)
594                        : null;
595                if (tag == null) {
596                    return null;
597                } else if ("key".equals(type)) {
598                    return tag.getKey();
599                } else if ("value".equals(type)) {
600                    return tag.getValue();
601                } else if ("tag".equals(type)) {
602                    return tag.toString();
603                }
604            } catch (IndexOutOfBoundsException ignore) {
605                Logging.debug(ignore);
606            }
607            return null;
608        }
609
610        /**
611         * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding
612         * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}.
613         * @param matchingSelector matching selector
614         * @param s any string
615         * @param p OSM primitive
616         * @return string with arguments inserted
617         */
618        static String insertArguments(Selector matchingSelector, String s, OsmPrimitive p) {
619            if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) {
620                return insertArguments(((Selector.ChildOrParentSelector) matchingSelector).right, s, p);
621            } else if (s == null || !(matchingSelector instanceof Selector.OptimizedGeneralSelector)) {
622                return s;
623            }
624            final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s);
625            final StringBuffer sb = new StringBuffer();
626            while (m.find()) {
627                final String argument = determineArgument((Selector.OptimizedGeneralSelector) matchingSelector,
628                        Integer.parseInt(m.group(1)), m.group(2), p);
629                try {
630                    // Perform replacement with null-safe + regex-safe handling
631                    m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", ""));
632                } catch (IndexOutOfBoundsException | IllegalArgumentException e) {
633                    Logging.log(Logging.LEVEL_ERROR, tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()), e);
634                }
635            }
636            m.appendTail(sb);
637            return sb.toString();
638        }
639
640        /**
641         * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive}
642         * if the error is fixable, or {@code null} otherwise.
643         *
644         * @param p the primitive to construct the fix for
645         * @return the fix or {@code null}
646         */
647        Command fixPrimitive(OsmPrimitive p) {
648            if (fixCommands.isEmpty() && !deletion) {
649                return null;
650            }
651            try {
652                final Selector matchingSelector = whichSelectorMatchesPrimitive(p);
653                Collection<Command> cmds = new LinkedList<>();
654                for (FixCommand fixCommand : fixCommands) {
655                    cmds.add(fixCommand.createCommand(p, matchingSelector));
656                }
657                if (deletion && !p.isDeleted()) {
658                    cmds.add(new DeleteCommand(p));
659                }
660                return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds);
661            } catch (IllegalArgumentException e) {
662                Logging.error(e);
663                return null;
664            }
665        }
666
667        /**
668         * Constructs a (localized) message for this deprecation check.
669         * @param p OSM primitive
670         *
671         * @return a message
672         */
673        String getMessage(OsmPrimitive p) {
674            if (errors.isEmpty()) {
675                // Return something to avoid NPEs
676                return rule.declaration.toString();
677            } else {
678                final Object val = errors.keySet().iterator().next().val;
679                return String.valueOf(
680                        val instanceof Expression
681                                ? ((Expression) val).evaluate(new Environment(p))
682                                : val
683                );
684            }
685        }
686
687        /**
688         * Constructs a (localized) description for this deprecation check.
689         * @param p OSM primitive
690         *
691         * @return a description (possibly with alternative suggestions)
692         * @see #getDescriptionForMatchingSelector
693         */
694        String getDescription(OsmPrimitive p) {
695            if (alternatives.isEmpty()) {
696                return getMessage(p);
697            } else {
698                /* I18N: {0} is the test error message and {1} is an alternative */
699                return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives));
700            }
701        }
702
703        /**
704         * Constructs a (localized) description for this deprecation check
705         * where any placeholders are replaced by values of the matched selector.
706         *
707         * @param matchingSelector matching selector
708         * @param p OSM primitive
709         * @return a description (possibly with alternative suggestions)
710         */
711        String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) {
712            return insertArguments(matchingSelector, getDescription(p), p);
713        }
714
715        Severity getSeverity() {
716            return errors.isEmpty() ? null : errors.values().iterator().next();
717        }
718
719        @Override
720        public String toString() {
721            return getDescription(null);
722        }
723
724        /**
725         * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error.
726         *
727         * @param p the primitive to construct the error for
728         * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error.
729         */
730        TestError getErrorForPrimitive(OsmPrimitive p) {
731            final Environment env = new Environment(p);
732            return getErrorForPrimitive(p, whichSelectorMatchesEnvironment(env), env, null);
733        }
734
735        TestError getErrorForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env, Test tester) {
736            if (matchingSelector != null && !errors.isEmpty()) {
737                final Command fix = fixPrimitive(p);
738                final String description = getDescriptionForMatchingSelector(p, matchingSelector);
739                final String description1 = group == null ? description : group;
740                final String description2 = group == null ? null : description;
741                final List<OsmPrimitive> primitives;
742                if (env.child instanceof OsmPrimitive) {
743                    primitives = Arrays.asList(p, (OsmPrimitive) env.child);
744                } else {
745                    primitives = Collections.singletonList(p);
746                }
747                final TestError.Builder error = TestError.builder(tester, getSeverity(), 3000)
748                        .messageWithManuallyTranslatedDescription(description1, description2, matchingSelector.toString())
749                        .primitives(primitives);
750                if (fix != null) {
751                    return error.fix(() -> fix).build();
752                } else {
753                    return error.build();
754                }
755            } else {
756                return null;
757            }
758        }
759
760        /**
761         * Returns the set of tagchecks on which this check depends on.
762         * @param schecks the collection of tagcheks to search in
763         * @return the set of tagchecks on which this check depends on
764         * @since 7881
765         */
766        public Set<TagCheck> getTagCheckDependencies(Collection<TagCheck> schecks) {
767            Set<TagCheck> result = new HashSet<>();
768            Set<String> classes = getClassesIds();
769            if (schecks != null && !classes.isEmpty()) {
770                for (TagCheck tc : schecks) {
771                    if (this.equals(tc)) {
772                        continue;
773                    }
774                    for (String id : tc.setClassExpressions) {
775                        if (classes.contains(id)) {
776                            result.add(tc);
777                            break;
778                        }
779                    }
780                }
781            }
782            return result;
783        }
784
785        /**
786         * Returns the list of ids of all MapCSS classes referenced in the rule selectors.
787         * @return the list of ids of all MapCSS classes referenced in the rule selectors
788         * @since 7881
789         */
790        public Set<String> getClassesIds() {
791            Set<String> result = new HashSet<>();
792            for (Selector s : rule.selectors) {
793                if (s instanceof AbstractSelector) {
794                    for (Condition c : ((AbstractSelector) s).getConditions()) {
795                        if (c instanceof ClassCondition) {
796                            result.add(((ClassCondition) c).id);
797                        }
798                    }
799                }
800            }
801            return result;
802        }
803    }
804
805    static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
806        public final GroupedMapCSSRule rule;
807
808        MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) {
809            this.rule = rule;
810        }
811
812        @Override
813        public synchronized boolean equals(Object obj) {
814            return super.equals(obj)
815                    || (obj instanceof TagCheck && rule.equals(((TagCheck) obj).rule))
816                    || (obj instanceof GroupedMapCSSRule && rule.equals(obj));
817        }
818
819        @Override
820        public synchronized int hashCode() {
821            return Objects.hash(super.hashCode(), rule);
822        }
823
824        @Override
825        public String toString() {
826            return "MapCSSTagCheckerAndRule [rule=" + rule + ']';
827        }
828    }
829
830    /**
831     * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
832     * @param p The OSM primitive
833     * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned
834     * @return all errors for the given primitive, with or without those of "info" severity
835     */
836    public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
837        final List<TestError> res = new ArrayList<>();
838        if (indexData == null)
839            indexData = new IndexData(checks);
840
841        MapCSSRuleIndex matchingRuleIndex = indexData.get(p);
842
843        Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
844        // the declaration indices are sorted, so it suffices to save the last used index
845        Declaration lastDeclUsed = null;
846
847        Iterator<MapCSSRule> candidates = matchingRuleIndex.getRuleCandidates(p);
848        while (candidates.hasNext()) {
849            MapCSSRule r = candidates.next();
850            env.clearSelectorMatchingInformation();
851            if (r.selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector)
852                TagCheck check = indexData.getCheck(r);
853                if (check != null) {
854                    boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
855                    // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
856                    if (ignoreError && check.setClassExpressions.isEmpty()) {
857                        continue;
858                    }
859                    if (r.declaration == lastDeclUsed)
860                        continue; // don't apply one declaration more than once
861                    lastDeclUsed = r.declaration;
862
863                    r.declaration.execute(env);
864                    if (!ignoreError && !check.errors.isEmpty()) {
865                        final TestError error = check.getErrorForPrimitive(p, r.selector, env, new MapCSSTagCheckerAndRule(check.rule));
866                        if (error != null) {
867                            res.add(error);
868                        }
869                    }
870
871                }
872            }
873        }
874        return res;
875    }
876
877    private static Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity,
878            Collection<Set<TagCheck>> checksCol) {
879        final List<TestError> r = new ArrayList<>();
880        final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
881        for (Set<TagCheck> schecks : checksCol) {
882            for (TagCheck check : schecks) {
883                boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
884                // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
885                if (ignoreError && check.setClassExpressions.isEmpty()) {
886                    continue;
887                }
888                final Selector selector = check.whichSelectorMatchesEnvironment(env);
889                if (selector != null) {
890                    check.rule.declaration.execute(env);
891                    if (!ignoreError && !check.errors.isEmpty()) {
892                        final TestError error = check.getErrorForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule));
893                        if (error != null) {
894                            r.add(error);
895                        }
896                    }
897                }
898            }
899        }
900        return r;
901    }
902
903    /**
904     * Visiting call for primitives.
905     *
906     * @param p The primitive to inspect.
907     */
908    @Override
909    public void check(OsmPrimitive p) {
910        errors.addAll(getErrorsForPrimitive(p, ValidatorPrefHelper.PREF_OTHER.get()));
911    }
912
913    /**
914     * Adds a new MapCSS config file from the given URL.
915     * @param url The unique URL of the MapCSS config file
916     * @return List of tag checks and parsing errors, or null
917     * @throws ParseException if the config file does not match MapCSS syntax
918     * @throws IOException if any I/O error occurs
919     * @since 7275
920     */
921    public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException {
922        CheckParameterUtil.ensureParameterNotNull(url, "url");
923        ParseResult result;
924        try (CachedFile cache = new CachedFile(url);
925             InputStream zip = cache.findZipEntryInputStream("validator.mapcss", "");
926             InputStream s = zip != null ? zip : cache.getInputStream();
927             Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) {
928            if (zip != null)
929                I18n.addTexts(cache.getFile());
930            result = TagCheck.readMapCSS(reader);
931            checks.remove(url);
932            checks.putAll(url, result.parseChecks);
933            indexData = null;
934            // Check assertions, useful for development of local files
935            if (Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url)) {
936                for (String msg : checkAsserts(result.parseChecks)) {
937                    Logging.warn(msg);
938                }
939            }
940        }
941        return result;
942    }
943
944    @Override
945    public synchronized void initialize() throws Exception {
946        checks.clear();
947        indexData = null;
948        for (SourceEntry source : new ValidatorPrefHelper().get()) {
949            if (!source.active) {
950                continue;
951            }
952            String i = source.url;
953            try {
954                if (!i.startsWith("resource:")) {
955                    Logging.info(tr("Adding {0} to tag checker", i));
956                } else if (Logging.isDebugEnabled()) {
957                    Logging.debug(tr("Adding {0} to tag checker", i));
958                }
959                addMapCSS(i);
960                if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) {
961                    FileWatcher.getDefaultInstance().registerSource(source);
962                }
963            } catch (IOException | IllegalStateException | IllegalArgumentException ex) {
964                Logging.warn(tr("Failed to add {0} to tag checker", i));
965                Logging.log(Logging.LEVEL_WARN, ex);
966            } catch (ParseException | TokenMgrError ex) {
967                Logging.warn(tr("Failed to add {0} to tag checker", i));
968                Logging.warn(ex);
969            }
970        }
971    }
972
973    private static Method getFunctionMethod(String method) {
974        try {
975            return Functions.class.getDeclaredMethod(method, Environment.class, String.class);
976        } catch (NoSuchMethodException | SecurityException e) {
977            Logging.error(e);
978            return null;
979        }
980    }
981
982    private static Optional<String> getFirstInsideCountry(TagCheck check, Method insideMethod) {
983        return check.rule.selectors.stream()
984                .filter(s -> s instanceof GeneralSelector)
985                .flatMap(s -> ((GeneralSelector) s).getConditions().stream())
986                .filter(c -> c instanceof ExpressionCondition)
987                .map(c -> ((ExpressionCondition) c).getExpression())
988                .filter(c -> c instanceof ParameterFunction)
989                .map(c -> (ParameterFunction) c)
990                .filter(c -> c.getMethod().equals(insideMethod))
991                .flatMap(c -> c.getArgs().stream())
992                .filter(e -> e instanceof LiteralExpression)
993                .map(e -> ((LiteralExpression) e).getLiteral())
994                .filter(l -> l instanceof String)
995                .map(l -> ((String) l).split(",")[0])
996                .findFirst();
997    }
998
999    private static LatLon getLocation(TagCheck check, Method insideMethod) {
1000        Optional<String> inside = getFirstInsideCountry(check, insideMethod);
1001        if (inside.isPresent()) {
1002            GeoPropertyIndex<Boolean> index = Territories.getGeoPropertyIndex(inside.get());
1003            if (index != null) {
1004                GeoProperty<Boolean> prop = index.getGeoProperty();
1005                if (prop instanceof DefaultGeoProperty) {
1006                    Rectangle bounds = ((DefaultGeoProperty) prop).getArea().getBounds();
1007                    return new LatLon(bounds.getCenterY(), bounds.getCenterX());
1008                }
1009            }
1010        }
1011        return LatLon.ZERO;
1012    }
1013
1014    /**
1015     * Checks that rule assertions are met for the given set of TagChecks.
1016     * @param schecks The TagChecks for which assertions have to be checked
1017     * @return A set of error messages, empty if all assertions are met
1018     * @since 7356
1019     */
1020    public Set<String> checkAsserts(final Collection<TagCheck> schecks) {
1021        Set<String> assertionErrors = new LinkedHashSet<>();
1022        final Method insideMethod = getFunctionMethod("inside");
1023        final DataSet ds = new DataSet();
1024        for (final TagCheck check : schecks) {
1025            Logging.debug("Check: {0}", check);
1026            for (final Map.Entry<String, Boolean> i : check.assertions.entrySet()) {
1027                Logging.debug("- Assertion: {0}", i);
1028                final OsmPrimitive p = OsmUtils.createPrimitive(i.getKey(), getLocation(check, insideMethod), true);
1029                // Build minimal ordered list of checks to run to test the assertion
1030                List<Set<TagCheck>> checksToRun = new ArrayList<>();
1031                Set<TagCheck> checkDependencies = check.getTagCheckDependencies(schecks);
1032                if (!checkDependencies.isEmpty()) {
1033                    checksToRun.add(checkDependencies);
1034                }
1035                checksToRun.add(Collections.singleton(check));
1036                // Add primitive to dataset to avoid DataIntegrityProblemException when evaluating selectors
1037                addPrimitive(ds, p);
1038                final Collection<TestError> pErrors = getErrorsForPrimitive(p, true, checksToRun);
1039                Logging.debug("- Errors: {0}", pErrors);
1040                @SuppressWarnings({"EqualsBetweenInconvertibleTypes", "EqualsIncompatibleType"})
1041                final boolean isError = pErrors.stream().anyMatch(e -> e.getTester().equals(check.rule));
1042                if (isError != i.getValue()) {
1043                    final String error = MessageFormat.format("Expecting test ''{0}'' (i.e., {1}) to {2} {3} (i.e., {4})",
1044                            check.getMessage(p), check.rule.selectors, i.getValue() ? "match" : "not match", i.getKey(), p.getKeys());
1045                    assertionErrors.add(error);
1046                }
1047                ds.removePrimitive(p);
1048            }
1049        }
1050        return assertionErrors;
1051    }
1052
1053    private static void addPrimitive(DataSet ds, OsmPrimitive p) {
1054        if (p instanceof Way) {
1055            ((Way) p).getNodes().forEach(n -> addPrimitive(ds, n));
1056        } else if (p instanceof Relation) {
1057            ((Relation) p).getMembers().forEach(m -> addPrimitive(ds, m.getMember()));
1058        }
1059        ds.addPrimitive(p);
1060    }
1061
1062    @Override
1063    public synchronized int hashCode() {
1064        return Objects.hash(super.hashCode(), checks);
1065    }
1066
1067    @Override
1068    public synchronized boolean equals(Object obj) {
1069        if (this == obj) return true;
1070        if (obj == null || getClass() != obj.getClass()) return false;
1071        if (!super.equals(obj)) return false;
1072        MapCSSTagChecker that = (MapCSSTagChecker) obj;
1073        return Objects.equals(checks, that.checks);
1074    }
1075
1076    /**
1077     * Reload tagchecker rule.
1078     * @param rule tagchecker rule to reload
1079     * @since 12825
1080     */
1081    public static void reloadRule(SourceEntry rule) {
1082        MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class);
1083        if (tagChecker != null) {
1084            try {
1085                tagChecker.addMapCSS(rule.url);
1086            } catch (IOException | ParseException | TokenMgrError e) {
1087                Logging.warn(e);
1088            }
1089        }
1090    }
1091
1092    @Override
1093    public void startTest(ProgressMonitor progressMonitor) {
1094        super.startTest(progressMonitor);
1095        super.setShowElements(true);
1096        if (indexData == null) {
1097            indexData = new IndexData(checks);
1098        }
1099    }
1100
1101    @Override
1102    public void endTest() {
1103        super.endTest();
1104        // no need to keep the index, it is quickly build and doubles the memory needs
1105        indexData = null;
1106    }
1107
1108}