001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.marktr; 005import static org.openstreetmap.josm.tools.I18n.tr; 006 007import java.util.ArrayList; 008import java.util.Arrays; 009import java.util.Collection; 010import java.util.HashMap; 011import java.util.HashSet; 012import java.util.List; 013import java.util.Locale; 014import java.util.Map; 015import java.util.Map.Entry; 016import java.util.Objects; 017import java.util.Set; 018import java.util.stream.Collectors; 019import java.util.stream.Stream; 020 021import org.openstreetmap.josm.command.Command; 022import org.openstreetmap.josm.command.DeleteCommand; 023import org.openstreetmap.josm.data.coor.EastNorth; 024import org.openstreetmap.josm.data.coor.LatLon; 025import org.openstreetmap.josm.data.osm.Node; 026import org.openstreetmap.josm.data.osm.OsmPrimitive; 027import org.openstreetmap.josm.data.osm.Relation; 028import org.openstreetmap.josm.data.osm.RelationMember; 029import org.openstreetmap.josm.data.osm.TagMap; 030import org.openstreetmap.josm.data.osm.Way; 031import org.openstreetmap.josm.data.preferences.DoubleProperty; 032import org.openstreetmap.josm.data.validation.Severity; 033import org.openstreetmap.josm.data.validation.Test; 034import org.openstreetmap.josm.data.validation.TestError; 035import org.openstreetmap.josm.tools.Geometry; 036import org.openstreetmap.josm.tools.Logging; 037import org.openstreetmap.josm.tools.Pair; 038import org.openstreetmap.josm.tools.SubclassFilteredCollection; 039import org.openstreetmap.josm.tools.Territories; 040import org.openstreetmap.josm.tools.Utils; 041 042/** 043 * Performs validation tests on addresses (addr:housenumber) and associatedStreet relations. 044 * @since 5644 045 */ 046public class Addresses extends Test { 047 048 protected static final int HOUSE_NUMBER_WITHOUT_STREET = 2601; 049 protected static final int DUPLICATE_HOUSE_NUMBER = 2602; 050 protected static final int MULTIPLE_STREET_NAMES = 2603; 051 protected static final int MULTIPLE_STREET_RELATIONS = 2604; 052 protected static final int HOUSE_NUMBER_TOO_FAR = 2605; 053 protected static final int OBSOLETE_RELATION = 2606; 054 055 protected static final DoubleProperty MAX_DUPLICATE_DISTANCE = new DoubleProperty("validator.addresses.max_duplicate_distance", 200.0); 056 protected static final DoubleProperty MAX_STREET_DISTANCE = new DoubleProperty("validator.addresses.max_street_distance", 200.0); 057 058 // CHECKSTYLE.OFF: SingleSpaceSeparator 059 protected static final String ADDR_HOUSE_NUMBER = "addr:housenumber"; 060 protected static final String ADDR_INTERPOLATION = "addr:interpolation"; 061 protected static final String ADDR_NEIGHBOURHOOD = "addr:neighbourhood"; 062 protected static final String ADDR_PLACE = "addr:place"; 063 protected static final String ADDR_STREET = "addr:street"; 064 protected static final String ADDR_CITY = "addr:city"; 065 protected static final String ADDR_UNIT = "addr:unit"; 066 protected static final String ADDR_FLATS = "addr:flats"; 067 protected static final String ADDR_HOUSE_NAME = "addr:housename"; 068 protected static final String ADDR_POSTCODE = "addr:postcode"; 069 protected static final String ASSOCIATED_STREET = "associatedStreet"; 070 // CHECKSTYLE.ON: SingleSpaceSeparator 071 072 private Map<String, Collection<OsmPrimitive>> knownAddresses; 073 private Set<String> ignoredAddresses; 074 075 076 /** 077 * Constructor 078 */ 079 public Addresses() { 080 super(tr("Addresses"), tr("Checks for errors in addresses and associatedStreet relations.")); 081 } 082 083 protected List<Relation> getAndCheckAssociatedStreets(OsmPrimitive p) { 084 final List<Relation> list = p.referrers(Relation.class) 085 .filter(r -> r.hasTag("type", ASSOCIATED_STREET)) 086 .collect(Collectors.toList()); 087 if (list.size() > 1) { 088 Severity level; 089 // warning level only if several relations have different names, see #10945 090 final String name = list.get(0).get("name"); 091 if (name == null || SubclassFilteredCollection.filter(list, r -> r.hasTag("name", name)).size() < list.size()) { 092 level = Severity.WARNING; 093 } else { 094 level = Severity.OTHER; 095 } 096 List<OsmPrimitive> errorList = new ArrayList<>(list); 097 errorList.add(0, p); 098 errors.add(TestError.builder(this, level, MULTIPLE_STREET_RELATIONS) 099 .message(tr("Multiple associatedStreet relations")) 100 .primitives(errorList) 101 .build()); 102 } 103 return list; 104 } 105 106 protected void checkHouseNumbersWithoutStreet(OsmPrimitive p) { 107 List<Relation> associatedStreets = getAndCheckAssociatedStreets(p); 108 // Find house number without proper location 109 // (neither addr:street, associatedStreet, addr:place, addr:neighbourhood or addr:interpolation) 110 if (p.hasKey(ADDR_HOUSE_NUMBER) && !p.hasKey(ADDR_STREET, ADDR_PLACE, ADDR_NEIGHBOURHOOD)) { 111 for (Relation r : associatedStreets) { 112 if (r.hasTag("type", ASSOCIATED_STREET)) { 113 return; 114 } 115 } 116 if (p.referrers(Way.class).anyMatch(w -> w.hasKey(ADDR_INTERPOLATION) && w.hasKey(ADDR_STREET))) { 117 return; 118 } 119 // No street found 120 errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_WITHOUT_STREET) 121 .message(tr("House number without street")) 122 .primitives(p) 123 .build()); 124 } 125 } 126 127 static boolean isPOI(OsmPrimitive p) { 128 return p.hasKey("shop", "amenity", "tourism", "leisure", "emergency", "craft", "office", "name"); 129 } 130 131 static boolean hasAddress(OsmPrimitive p) { 132 return p.hasKey(ADDR_HOUSE_NUMBER) && p.hasKey(ADDR_STREET, ADDR_PLACE); 133 } 134 135 /** 136 * adds the OsmPrimitive to the address map if it complies to the restrictions 137 * @param p OsmPrimitive that has an address 138 */ 139 private void collectAddress(OsmPrimitive p) { 140 if (!isPOI(p)) { 141 String simplifiedAddress = getSimplifiedAddress(p); 142 if (!ignoredAddresses.contains(simplifiedAddress)) { 143 knownAddresses.computeIfAbsent(simplifiedAddress, x -> new ArrayList<>()).add(p); 144 } 145 } 146 } 147 148 protected void initAddressMap(OsmPrimitive primitive) { 149 knownAddresses = new HashMap<>(); 150 ignoredAddresses = new HashSet<>(); 151 for (OsmPrimitive p : primitive.getDataSet().allNonDeletedPrimitives()) { 152 if (p instanceof Node && p.hasKey(ADDR_UNIT, ADDR_FLATS)) { 153 for (OsmPrimitive r : p.getReferrers()) { 154 if (hasAddress(r)) { 155 // ignore addresses of buildings that are connected to addr:unit nodes 156 // it's quite reasonable that there are more buildings with this address 157 String simplifiedAddress = getSimplifiedAddress(r); 158 if (!ignoredAddresses.contains(simplifiedAddress)) { 159 ignoredAddresses.add(simplifiedAddress); 160 } else if (knownAddresses.containsKey(simplifiedAddress)) { 161 knownAddresses.remove(simplifiedAddress); 162 } 163 } 164 } 165 } 166 if (hasAddress(p)) { 167 collectAddress(p); 168 } 169 } 170 } 171 172 @Override 173 public void endTest() { 174 knownAddresses = null; 175 ignoredAddresses = null; 176 super.endTest(); 177 } 178 179 protected void checkForDuplicate(OsmPrimitive p) { 180 if (knownAddresses == null) { 181 initAddressMap(p); 182 } 183 if (!isPOI(p) && hasAddress(p)) { 184 String simplifiedAddress = getSimplifiedAddress(p); 185 if (ignoredAddresses.contains(simplifiedAddress)) { 186 return; 187 } 188 if (knownAddresses.containsKey(simplifiedAddress)) { 189 double maxDistance = MAX_DUPLICATE_DISTANCE.get(); 190 for (OsmPrimitive p2 : knownAddresses.get(simplifiedAddress)) { 191 if (p == p2) { 192 continue; 193 } 194 Severity severityLevel; 195 String city1 = p.get(ADDR_CITY); 196 String city2 = p2.get(ADDR_CITY); 197 double distance = getDistance(p, p2); 198 if (city1 != null && city2 != null) { 199 if (city1.equals(city2)) { 200 if (!p.hasKey(ADDR_POSTCODE) || !p2.hasKey(ADDR_POSTCODE) || p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) { 201 severityLevel = Severity.WARNING; 202 } else { 203 // address including city identical but postcode differs 204 // most likely perfectly fine 205 severityLevel = Severity.OTHER; 206 } 207 } else { 208 // address differs only by city - notify if very close, otherwise ignore 209 if (distance < maxDistance) { 210 severityLevel = Severity.OTHER; 211 } else { 212 continue; 213 } 214 } 215 } else { 216 // at least one address has no city specified 217 if (p.hasKey(ADDR_POSTCODE) && p2.hasKey(ADDR_POSTCODE) && p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) { 218 // address including postcode identical 219 severityLevel = Severity.WARNING; 220 } else { 221 // city/postcode unclear - warn if very close, otherwise only notify 222 // TODO: get city from surrounding boundaries? 223 if (distance < maxDistance) { 224 severityLevel = Severity.WARNING; 225 } else { 226 severityLevel = Severity.OTHER; 227 } 228 } 229 } 230 errors.add(TestError.builder(this, severityLevel, DUPLICATE_HOUSE_NUMBER) 231 .message(tr("Duplicate house numbers"), marktr("''{0}'' ({1}m)"), simplifiedAddress, (int) distance) 232 .primitives(Arrays.asList(p, p2)).build()); 233 } 234 knownAddresses.get(simplifiedAddress).remove(p); // otherwise we would get every warning two times 235 } 236 } 237 } 238 239 static String getSimplifiedAddress(OsmPrimitive p) { 240 String simplifiedStreetName = p.hasKey(ADDR_STREET) ? p.get(ADDR_STREET) : p.get(ADDR_PLACE); 241 // ignore whitespaces and dashes in street name, so that "Mozart-Gasse", "Mozart Gasse" and "Mozartgasse" are all seen as equal 242 return Utils.strip(Stream.of( 243 simplifiedStreetName.replaceAll("[ -]", ""), 244 p.get(ADDR_HOUSE_NUMBER), 245 p.get(ADDR_HOUSE_NAME), 246 p.get(ADDR_UNIT), 247 p.get(ADDR_FLATS)) 248 .filter(Objects::nonNull) 249 .collect(Collectors.joining(" "))) 250 .toUpperCase(Locale.ENGLISH); 251 } 252 253 @Override 254 public void visit(Node n) { 255 checkHouseNumbersWithoutStreet(n); 256 checkForDuplicate(n); 257 } 258 259 @Override 260 public void visit(Way w) { 261 checkHouseNumbersWithoutStreet(w); 262 checkForDuplicate(w); 263 } 264 265 @Override 266 public void visit(Relation r) { 267 checkHouseNumbersWithoutStreet(r); 268 checkForDuplicate(r); 269 if (r.hasTag("type", ASSOCIATED_STREET)) { 270 checkIfObsolete(r); 271 // Used to count occurrences of each house number in order to find duplicates 272 Map<String, List<OsmPrimitive>> map = new HashMap<>(); 273 // Used to detect different street names 274 String relationName = r.get("name"); 275 Set<OsmPrimitive> wrongStreetNames = new HashSet<>(); 276 // Used to check distance 277 Set<OsmPrimitive> houses = new HashSet<>(); 278 Set<Way> street = new HashSet<>(); 279 for (RelationMember m : r.getMembers()) { 280 String role = m.getRole(); 281 OsmPrimitive p = m.getMember(); 282 if ("house".equals(role)) { 283 houses.add(p); 284 String number = p.get(ADDR_HOUSE_NUMBER); 285 if (number != null) { 286 number = number.trim().toUpperCase(Locale.ENGLISH); 287 List<OsmPrimitive> list = map.get(number); 288 if (list == null) { 289 list = new ArrayList<>(); 290 map.put(number, list); 291 } 292 list.add(p); 293 } 294 if (relationName != null && p.hasKey(ADDR_STREET) && !relationName.equals(p.get(ADDR_STREET))) { 295 if (wrongStreetNames.isEmpty()) { 296 wrongStreetNames.add(r); 297 } 298 wrongStreetNames.add(p); 299 } 300 } else if ("street".equals(role)) { 301 if (p instanceof Way) { 302 street.add((Way) p); 303 } 304 if (relationName != null && p.hasTagDifferent("name", relationName)) { 305 if (wrongStreetNames.isEmpty()) { 306 wrongStreetNames.add(r); 307 } 308 wrongStreetNames.add(p); 309 } 310 } 311 } 312 // Report duplicate house numbers 313 for (Entry<String, List<OsmPrimitive>> entry : map.entrySet()) { 314 List<OsmPrimitive> list = entry.getValue(); 315 if (list.size() > 1) { 316 errors.add(TestError.builder(this, Severity.WARNING, DUPLICATE_HOUSE_NUMBER) 317 .message(tr("Duplicate house numbers"), marktr("House number ''{0}'' duplicated"), entry.getKey()) 318 .primitives(list) 319 .build()); 320 } 321 } 322 // Report wrong street names 323 if (!wrongStreetNames.isEmpty()) { 324 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_STREET_NAMES) 325 .message(tr("Multiple street names in relation")) 326 .primitives(wrongStreetNames) 327 .build()); 328 } 329 // Report addresses too far away 330 if (!street.isEmpty()) { 331 for (OsmPrimitive house : houses) { 332 if (house.isUsable()) { 333 checkDistance(house, street); 334 } 335 } 336 } 337 } 338 } 339 340 /** 341 * returns rough distance between two OsmPrimitives 342 * @param a primitive a 343 * @param b primitive b 344 * @return distance of center of bounding boxes in meters 345 */ 346 static double getDistance(OsmPrimitive a, OsmPrimitive b) { 347 LatLon centerA = a.getBBox().getCenter(); 348 LatLon centerB = b.getBBox().getCenter(); 349 return (centerA.greatCircleDistance(centerB)); 350 } 351 352 protected void checkDistance(OsmPrimitive house, Collection<Way> street) { 353 EastNorth centroid; 354 if (house instanceof Node) { 355 centroid = ((Node) house).getEastNorth(); 356 } else if (house instanceof Way) { 357 List<Node> nodes = ((Way) house).getNodes(); 358 if (house.hasKey(ADDR_INTERPOLATION)) { 359 for (Node n : nodes) { 360 if (n.hasKey(ADDR_HOUSE_NUMBER)) { 361 checkDistance(n, street); 362 } 363 } 364 return; 365 } 366 centroid = Geometry.getCentroid(nodes); 367 } else { 368 return; // TODO handle multipolygon houses ? 369 } 370 if (centroid == null) return; // fix #8305 371 double maxDistance = MAX_STREET_DISTANCE.get(); 372 boolean hasIncompleteWays = false; 373 for (Way streetPart : street) { 374 for (Pair<Node, Node> chunk : streetPart.getNodePairs(false)) { 375 EastNorth p1 = chunk.a.getEastNorth(); 376 EastNorth p2 = chunk.b.getEastNorth(); 377 if (p1 != null && p2 != null) { 378 EastNorth closest = Geometry.closestPointToSegment(p1, p2, centroid); 379 if (closest.distance(centroid) <= maxDistance) { 380 return; 381 } 382 } else { 383 Logging.warn("Addresses test skipped chunck "+chunk+" for street part "+streetPart+" because p1 or p2 is null"); 384 } 385 } 386 if (!hasIncompleteWays && streetPart.isIncomplete()) { 387 hasIncompleteWays = true; 388 } 389 } 390 // No street segment found near this house, report error on if the relation does not contain incomplete street ways (fix #8314) 391 if (hasIncompleteWays) return; 392 List<OsmPrimitive> errorList = new ArrayList<>(street); 393 errorList.add(0, house); 394 errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_TOO_FAR) 395 .message(tr("House number too far from street")) 396 .primitives(errorList) 397 .build()); 398 } 399 400 /** 401 * Check if an associatedStreet Relation is obsolete. This test marks only those relations which 402 * are complete and don't contain any information which isn't also tagged on the members. 403 * The strategy is to avoid any false positive. 404 * @param r the relation 405 */ 406 private void checkIfObsolete(Relation r) { 407 if (r.isIncomplete()) 408 return; 409 /** array of country codes for which the test should be performed. For now, only Germany */ 410 String[] countryCodes = {"DE"}; 411 TagMap neededtagsForHouse = new TagMap(); 412 for (Entry<String, String> tag : r.getKeys().entrySet()) { 413 String key = tag.getKey(); 414 if (key.startsWith("name:")) { 415 return; // maybe check if all members have corresponding tags? 416 } else if (key.startsWith("addr:")) { 417 neededtagsForHouse.put(key, tag.getValue()); 418 } else { 419 switch (key) { 420 case "name": 421 case "type": 422 case "source": 423 break; 424 default: 425 // unexpected tag in relation 426 return; 427 } 428 } 429 } 430 431 for (RelationMember m : r.getMembers()) { 432 if (m.getMember().isIncomplete() || !isInWarnCountry(m, countryCodes)) 433 return; 434 435 String role = m.getRole(); 436 if ("".equals(role)) { 437 if (m.isWay() && m.getMember().hasKey("highway")) { 438 role = "street"; 439 } else if (m.getMember().hasTag("building")) 440 role = "house"; 441 } 442 switch (role) { 443 case "house": 444 case "addr:houselink": 445 case "address": 446 if (!m.getMember().hasTag(ADDR_STREET) || !m.getMember().hasTag(ADDR_HOUSE_NUMBER)) 447 return; 448 for (Entry<String, String> tag : neededtagsForHouse.entrySet()) { 449 if (!m.getMember().hasTag(tag.getKey(), tag.getValue())) 450 return; 451 } 452 break; 453 case "street": 454 if (!m.getMember().hasTag("name") && r.hasTag("name")) 455 return; 456 break; 457 default: 458 // unknown role: don't create auto-fix 459 return; 460 } 461 } 462 errors.add(TestError.builder(this, Severity.WARNING, OBSOLETE_RELATION) 463 .message(tr("Relation is obsolete")) 464 .primitives(r) 465 .build()); 466 } 467 468 private static boolean isInWarnCountry(RelationMember m, String[] countryCodes) { 469 if (countryCodes.length == 0) 470 return true; 471 LatLon center = null; 472 473 if (m.isNode()) { 474 center = m.getNode().getCoor(); 475 } else if (m.isWay()) { 476 center = m.getWay().getBBox().getCenter(); 477 } else if (m.isRelation() && m.getRelation().isMultipolygon()) { 478 center = m.getRelation().getBBox().getCenter(); 479 } 480 if (center == null) 481 return false; 482 for (String country : countryCodes) { 483 if (Territories.isIso3166Code(country, center)) 484 return true; 485 } 486 return false; 487 } 488 489 /** 490 * remove obsolete relation. 491 */ 492 @Override 493 public Command fixError(TestError testError) { 494 return new DeleteCommand(testError.getPrimitives()); 495 } 496 497 @Override 498 public boolean isFixable(TestError testError) { 499 if (!(testError.getTester() instanceof Addresses)) 500 return false; 501 return testError.getCode() == OBSOLETE_RELATION; 502 } 503 504}