001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.actions.downloadtasks; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.net.MalformedURLException; 008import java.net.URL; 009import java.util.ArrayList; 010import java.util.Arrays; 011import java.util.Collection; 012import java.util.Collections; 013import java.util.HashSet; 014import java.util.Objects; 015import java.util.Optional; 016import java.util.Set; 017import java.util.concurrent.Future; 018import java.util.regex.Matcher; 019import java.util.regex.Pattern; 020import java.util.stream.Stream; 021 022import org.openstreetmap.josm.data.Bounds; 023import org.openstreetmap.josm.data.DataSource; 024import org.openstreetmap.josm.data.ProjectionBounds; 025import org.openstreetmap.josm.data.ViewportData; 026import org.openstreetmap.josm.data.coor.LatLon; 027import org.openstreetmap.josm.data.osm.DataSet; 028import org.openstreetmap.josm.data.osm.OsmPrimitive; 029import org.openstreetmap.josm.data.osm.Relation; 030import org.openstreetmap.josm.data.osm.Way; 031import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor; 032import org.openstreetmap.josm.gui.MainApplication; 033import org.openstreetmap.josm.gui.MapFrame; 034import org.openstreetmap.josm.gui.PleaseWaitRunnable; 035import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask; 036import org.openstreetmap.josm.gui.layer.OsmDataLayer; 037import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 038import org.openstreetmap.josm.gui.progress.ProgressMonitor; 039import org.openstreetmap.josm.io.BoundingBoxDownloader; 040import org.openstreetmap.josm.io.OsmServerLocationReader; 041import org.openstreetmap.josm.io.OsmServerLocationReader.OsmUrlPattern; 042import org.openstreetmap.josm.io.OsmServerReader; 043import org.openstreetmap.josm.io.OsmTransferCanceledException; 044import org.openstreetmap.josm.io.OsmTransferException; 045import org.openstreetmap.josm.io.OverpassDownloadReader; 046import org.openstreetmap.josm.tools.Logging; 047import org.openstreetmap.josm.tools.Utils; 048import org.xml.sax.SAXException; 049 050/** 051 * Open the download dialog and download the data. 052 * Run in the worker thread. 053 */ 054public class DownloadOsmTask extends AbstractDownloadTask<DataSet> { 055 056 protected Bounds currentBounds; 057 protected DownloadTask downloadTask; 058 059 protected String newLayerName; 060 061 /** This allows subclasses to ignore this warning */ 062 protected boolean warnAboutEmptyArea = true; 063 064 protected static final String OVERPASS_INTERPRETER_DATA = "interpreter?data="; 065 066 @Override 067 public String[] getPatterns() { 068 if (this.getClass() == DownloadOsmTask.class) { 069 return Arrays.stream(OsmUrlPattern.values()).map(OsmUrlPattern::pattern).toArray(String[]::new); 070 } else { 071 return super.getPatterns(); 072 } 073 } 074 075 @Override 076 public String getTitle() { 077 if (this.getClass() == DownloadOsmTask.class) { 078 return tr("Download OSM"); 079 } else { 080 return super.getTitle(); 081 } 082 } 083 084 @Override 085 public Future<?> download(DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) { 086 return download(new BoundingBoxDownloader(downloadArea), settings, downloadArea, progressMonitor); 087 } 088 089 /** 090 * Asynchronously launches the download task for a given bounding box. 091 * 092 * Set <code>progressMonitor</code> to null, if the task should create, open, and close a progress monitor. 093 * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to 094 * be discarded. 095 * 096 * You can wait for the asynchronous download task to finish by synchronizing on the returned 097 * {@link Future}, but make sure not to freeze up JOSM. Example: 098 * <pre> 099 * Future<?> future = task.download(...); 100 * // DON'T run this on the Swing EDT or JOSM will freeze 101 * future.get(); // waits for the dowload task to complete 102 * </pre> 103 * 104 * The following example uses a pattern which is better suited if a task is launched from 105 * the Swing EDT: 106 * <pre> 107 * final Future<?> future = task.download(...); 108 * Runnable runAfterTask = new Runnable() { 109 * public void run() { 110 * // this is not strictly necessary because of the type of executor service 111 * // Main.worker is initialized with, but it doesn't harm either 112 * // 113 * future.get(); // wait for the download task to complete 114 * doSomethingAfterTheTaskCompleted(); 115 * } 116 * } 117 * MainApplication.worker.submit(runAfterTask); 118 * </pre> 119 * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm}) 120 * @param settings download settings 121 * @param downloadArea the area to download 122 * @param progressMonitor the progressMonitor 123 * @return the future representing the asynchronous task 124 * @since 13927 125 */ 126 public Future<?> download(OsmServerReader reader, DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) { 127 return download(new DownloadTask(settings, reader, progressMonitor, zoomAfterDownload), downloadArea); 128 } 129 130 protected Future<?> download(DownloadTask downloadTask, Bounds downloadArea) { 131 this.downloadTask = downloadTask; 132 this.currentBounds = new Bounds(downloadArea); 133 // We need submit instead of execute so we can wait for it to finish and get the error 134 // message if necessary. If no one calls getErrorMessage() it just behaves like execute. 135 return MainApplication.worker.submit(downloadTask); 136 } 137 138 /** 139 * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed. 140 * @param url the original URL 141 * @return the modified URL 142 */ 143 protected String modifyUrlBeforeLoad(String url) { 144 return url; 145 } 146 147 /** 148 * Loads a given URL from the OSM Server 149 * @param settings download settings 150 * @param url The URL as String 151 */ 152 @Override 153 public Future<?> loadUrl(DownloadParams settings, String url, ProgressMonitor progressMonitor) { 154 String newUrl = modifyUrlBeforeLoad(url); 155 downloadTask = new DownloadTask(settings, getOsmServerReader(newUrl), progressMonitor); 156 currentBounds = null; 157 // Extract .osm filename from URL to set the new layer name 158 extractOsmFilename(settings, "https?://.*/(.*\\.osm)", newUrl); 159 return MainApplication.worker.submit(downloadTask); 160 } 161 162 protected OsmServerReader getOsmServerReader(String url) { 163 try { 164 String host = new URL(url).getHost(); 165 for (String knownOverpassServer : OverpassDownloadReader.OVERPASS_SERVER_HISTORY.get()) { 166 if (host.equals(new URL(knownOverpassServer).getHost())) { 167 int index = url.indexOf(OVERPASS_INTERPRETER_DATA); 168 if (index > 0) { 169 return new OverpassDownloadReader(new Bounds(LatLon.ZERO), knownOverpassServer, 170 Utils.decodeUrl(url.substring(index + OVERPASS_INTERPRETER_DATA.length()))); 171 } 172 } 173 } 174 } catch (MalformedURLException e) { 175 Logging.error(e); 176 } 177 return new OsmServerLocationReader(url); 178 } 179 180 protected final void extractOsmFilename(DownloadParams settings, String pattern, String url) { 181 newLayerName = settings.getLayerName(); 182 if (newLayerName == null || newLayerName.isEmpty()) { 183 Matcher matcher = Pattern.compile(pattern).matcher(url); 184 newLayerName = matcher.matches() ? matcher.group(1) : null; 185 } 186 } 187 188 @Override 189 public void cancel() { 190 if (downloadTask != null) { 191 downloadTask.cancel(); 192 } 193 } 194 195 @Override 196 public boolean isSafeForRemotecontrolRequests() { 197 return true; 198 } 199 200 @Override 201 public ProjectionBounds getDownloadProjectionBounds() { 202 return downloadTask != null ? downloadTask.computeBbox(currentBounds).orElse(null) : null; 203 } 204 205 /** 206 * Superclass of internal download task. 207 * @since 7636 208 */ 209 public abstract static class AbstractInternalTask extends PleaseWaitRunnable { 210 211 protected final DownloadParams settings; 212 protected final boolean zoomAfterDownload; 213 protected DataSet dataSet; 214 215 /** 216 * Constructs a new {@code AbstractInternalTask}. 217 * @param settings download settings 218 * @param title message for the user 219 * @param ignoreException If true, exception will be propagated to calling code. If false then 220 * exception will be thrown directly in EDT. When this runnable is executed using executor framework 221 * then use false unless you read result of task (because exception will get lost if you don't) 222 * @param zoomAfterDownload If true, the map view will zoom to download area after download 223 */ 224 public AbstractInternalTask(DownloadParams settings, String title, boolean ignoreException, boolean zoomAfterDownload) { 225 super(title, ignoreException); 226 this.settings = Objects.requireNonNull(settings); 227 this.zoomAfterDownload = zoomAfterDownload; 228 } 229 230 /** 231 * Constructs a new {@code AbstractInternalTask}. 232 * @param settings download settings 233 * @param title message for the user 234 * @param progressMonitor progress monitor 235 * @param ignoreException If true, exception will be propagated to calling code. If false then 236 * exception will be thrown directly in EDT. When this runnable is executed using executor framework 237 * then use false unless you read result of task (because exception will get lost if you don't) 238 * @param zoomAfterDownload If true, the map view will zoom to download area after download 239 */ 240 public AbstractInternalTask(DownloadParams settings, String title, ProgressMonitor progressMonitor, boolean ignoreException, 241 boolean zoomAfterDownload) { 242 super(title, progressMonitor, ignoreException); 243 this.settings = Objects.requireNonNull(settings); 244 this.zoomAfterDownload = zoomAfterDownload; 245 } 246 247 protected OsmDataLayer getEditLayer() { 248 return MainApplication.getLayerManager().getEditLayer(); 249 } 250 251 private static Stream<OsmDataLayer> getModifiableDataLayers() { 252 return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class) 253 .stream().filter(OsmDataLayer::isDownloadable); 254 } 255 256 /** 257 * Returns the number of modifiable data layers 258 * @return number of modifiable data layers 259 * @since 13434 260 */ 261 protected long getNumModifiableDataLayers() { 262 return getModifiableDataLayers().count(); 263 } 264 265 /** 266 * Returns the first modifiable data layer 267 * @return the first modifiable data layer 268 * @since 13434 269 */ 270 protected OsmDataLayer getFirstModifiableDataLayer() { 271 return getModifiableDataLayers().findFirst().orElse(null); 272 } 273 274 /** 275 * Creates a name for a new layer by utilizing the settings ({@link DownloadParams#getLayerName()}) or 276 * {@link OsmDataLayer#createNewName()} if the former option is {@code null}. 277 * 278 * @return a name for a new layer 279 * @since 14347 280 */ 281 protected String generateLayerName() { 282 return Optional.ofNullable(settings.getLayerName()) 283 .filter(layerName -> !Utils.isStripEmpty(layerName)) 284 .orElse(OsmDataLayer.createNewName()); 285 } 286 287 /** 288 * Can be overridden (e.g. by plugins) if a subclass of {@link OsmDataLayer} is needed. 289 * If you want to change how the name is determined, consider overriding 290 * {@link #generateLayerName()} instead. 291 * 292 * @param ds the dataset on which the layer is based, must be non-null 293 * @param layerName the name of the new layer, must be either non-blank or non-present 294 * @return a new instance of {@link OsmDataLayer} constructed with the given arguments 295 * @since 14347 296 */ 297 protected OsmDataLayer createNewLayer(final DataSet ds, final Optional<String> layerName) { 298 if (layerName.filter(Utils::isStripEmpty).isPresent()) { 299 throw new IllegalArgumentException("Blank layer name!"); 300 } 301 return new OsmDataLayer( 302 Objects.requireNonNull(ds, "dataset parameter"), 303 layerName.orElseGet(this::generateLayerName), 304 null 305 ); 306 } 307 308 /** 309 * Convenience method for {@link #createNewLayer(DataSet, Optional)}, uses the dataset 310 * from field {@link #dataSet} and applies the settings from field {@link #settings}. 311 * 312 * @param layerName an optional layer name, must be non-blank if the [Optional] is present 313 * @return a newly constructed layer 314 * @since 14347 315 */ 316 protected final OsmDataLayer createNewLayer(final Optional<String> layerName) { 317 Optional.ofNullable(settings.getDownloadPolicy()) 318 .ifPresent(dataSet::setDownloadPolicy); 319 Optional.ofNullable(settings.getUploadPolicy()) 320 .ifPresent(dataSet::setUploadPolicy); 321 if (dataSet.isLocked() && !settings.isLocked()) { 322 dataSet.unlock(); 323 } else if (!dataSet.isLocked() && settings.isLocked()) { 324 dataSet.lock(); 325 } 326 return createNewLayer(dataSet, layerName); 327 } 328 329 protected Optional<ProjectionBounds> computeBbox(Bounds bounds) { 330 BoundingXYVisitor v = new BoundingXYVisitor(); 331 if (bounds != null) { 332 v.visit(bounds); 333 } else { 334 v.computeBoundingBox(dataSet.getNodes()); 335 } 336 return Optional.ofNullable(v.getBounds()); 337 } 338 339 protected OsmDataLayer addNewLayerIfRequired(String newLayerName) { 340 long numDataLayers = getNumModifiableDataLayers(); 341 if (settings.isNewLayer() || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) { 342 // the user explicitly wants a new layer, we don't have any layer at all 343 // or it is not clear which layer to merge to 344 final OsmDataLayer layer = createNewLayer(Optional.ofNullable(newLayerName).filter(it -> !Utils.isStripEmpty(it))); 345 MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload); 346 return layer; 347 } 348 return null; 349 } 350 351 protected void loadData(String newLayerName, Bounds bounds) { 352 OsmDataLayer layer = addNewLayerIfRequired(newLayerName); 353 if (layer == null) { 354 layer = getEditLayer(); 355 if (layer == null || !layer.isDownloadable()) { 356 layer = getFirstModifiableDataLayer(); 357 } 358 Collection<OsmPrimitive> primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.getDataSet()); 359 layer.mergeFrom(dataSet); 360 MapFrame map = MainApplication.getMap(); 361 if (map != null && zoomAfterDownload) { 362 computeBbox(bounds).map(ViewportData::new).ifPresent(map.mapView::zoomTo); 363 } 364 if (!primitivesToUpdate.isEmpty()) { 365 MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate)); 366 } 367 layer.onPostDownloadFromServer(); 368 } 369 } 370 371 /** 372 * Look for primitives deleted on server (thus absent from downloaded data) 373 * but still present in existing data layer 374 * @param bounds download bounds 375 * @param ds existing data set 376 * @return the primitives to update 377 */ 378 private Collection<OsmPrimitive> searchPrimitivesToUpdate(Bounds bounds, DataSet ds) { 379 if (bounds == null) 380 return Collections.emptySet(); 381 Collection<OsmPrimitive> col = new ArrayList<>(); 382 ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add); 383 if (!col.isEmpty()) { 384 Set<Way> ways = new HashSet<>(); 385 Set<Relation> rels = new HashSet<>(); 386 for (OsmPrimitive n : col) { 387 for (OsmPrimitive ref : n.getReferrers()) { 388 if (ref.isNew()) { 389 continue; 390 } else if (ref instanceof Way) { 391 ways.add((Way) ref); 392 } else if (ref instanceof Relation) { 393 rels.add((Relation) ref); 394 } 395 } 396 } 397 ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add); 398 rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add); 399 } 400 return col; 401 } 402 } 403 404 protected class DownloadTask extends AbstractInternalTask { 405 protected final OsmServerReader reader; 406 407 /** 408 * Constructs a new {@code DownloadTask}. 409 * @param settings download settings 410 * @param reader OSM data reader 411 * @param progressMonitor progress monitor 412 * @since 13927 413 */ 414 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor) { 415 this(settings, reader, progressMonitor, true); 416 } 417 418 /** 419 * Constructs a new {@code DownloadTask}. 420 * @param settings download settings 421 * @param reader OSM data reader 422 * @param progressMonitor progress monitor 423 * @param zoomAfterDownload If true, the map view will zoom to download area after download 424 * @since 13927 425 */ 426 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) { 427 super(settings, tr("Downloading data"), progressMonitor, false, zoomAfterDownload); 428 this.reader = reader; 429 } 430 431 protected DataSet parseDataSet() throws OsmTransferException { 432 return reader.parseOsm(progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false)); 433 } 434 435 @Override 436 public void realRun() throws IOException, SAXException, OsmTransferException { 437 try { 438 if (isCanceled()) 439 return; 440 dataSet = parseDataSet(); 441 } catch (OsmTransferException e) { 442 if (isCanceled()) { 443 Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString())); 444 return; 445 } 446 if (e instanceof OsmTransferCanceledException) { 447 setCanceled(true); 448 return; 449 } else { 450 rememberException(e); 451 } 452 DownloadOsmTask.this.setFailed(true); 453 } 454 } 455 456 @Override 457 protected void finish() { 458 if (isFailed() || isCanceled()) 459 return; 460 if (dataSet == null) 461 return; // user canceled download or error occurred 462 if (dataSet.allPrimitives().isEmpty()) { 463 if (warnAboutEmptyArea) { 464 rememberErrorMessage(tr("No data found in this area.")); 465 } 466 String remark = dataSet.getRemark(); 467 if (remark != null && !remark.isEmpty()) { 468 rememberErrorMessage(remark); 469 } 470 // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work 471 dataSet.addDataSource(new DataSource(currentBounds != null ? currentBounds : 472 new Bounds(LatLon.ZERO), "OpenStreetMap server")); 473 } 474 475 rememberDownloadedData(dataSet); 476 loadData(newLayerName, currentBounds); 477 } 478 479 @Override 480 protected void cancel() { 481 setCanceled(true); 482 if (reader != null) { 483 reader.cancel(); 484 } 485 } 486 } 487 488 @Override 489 public String getConfirmationMessage(URL url) { 490 if (url != null) { 491 String urlString = url.toExternalForm(); 492 if (urlString.matches(OsmUrlPattern.OSM_API_URL.pattern())) { 493 // TODO: proper i18n after stabilization 494 Collection<String> items = new ArrayList<>(); 495 items.add(tr("OSM Server URL:") + ' ' + url.getHost()); 496 items.add(tr("Command")+": "+url.getPath()); 497 if (url.getQuery() != null) { 498 items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", "))); 499 } 500 return Utils.joinAsHtmlUnorderedList(items); 501 } 502 // TODO: other APIs 503 } 504 return null; 505 } 506}