001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedInputStream; 007import java.io.BufferedOutputStream; 008import java.io.File; 009import java.io.FileInputStream; 010import java.io.FileOutputStream; 011import java.io.IOException; 012import java.io.InputStream; 013import java.io.OutputStream; 014import java.net.HttpURLConnection; 015import java.net.MalformedURLException; 016import java.net.URL; 017import java.nio.charset.StandardCharsets; 018import java.util.ArrayList; 019import java.util.Arrays; 020import java.util.Enumeration; 021import java.util.List; 022import java.util.zip.ZipEntry; 023import java.util.zip.ZipFile; 024 025import org.openstreetmap.josm.Main; 026import org.openstreetmap.josm.tools.CheckParameterUtil; 027import org.openstreetmap.josm.tools.Pair; 028import org.openstreetmap.josm.tools.Utils; 029 030/** 031 * Downloads a file and caches it on disk in order to reduce network load. 032 * 033 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get 034 * resources from the current *.jar file. (Local caching is only done for URLs.) 035 * <p> 036 * The mirrored file is only downloaded if it has been more than 7 days since 037 * last download. (Time can be configured.) 038 * <p> 039 * The file content is normally accessed with {@link #getInputStream()}, but 040 * you can also get the mirrored copy with {@link #getFile()}. 041 */ 042public class CachedFile { 043 044 /** 045 * Caching strategy. 046 */ 047 public enum CachingStrategy { 048 /** 049 * If cached file on disk is older than a certain time (7 days by default), 050 * consider the cache stale and try to download the file again. 051 */ 052 MaxAge, 053 /** 054 * Similar to MaxAge, considers the cache stale when a certain age is 055 * exceeded. In addition, a If-Modified-Since HTTP header is added. 056 * When the server replies "304 Not Modified", this is considered the same 057 * as a full download. 058 */ 059 IfModifiedSince 060 } 061 protected String name; 062 protected long maxAge; 063 protected String destDir; 064 protected String httpAccept; 065 protected CachingStrategy cachingStrategy; 066 067 protected File cacheFile = null; 068 boolean initialized = false; 069 070 public static final long DEFAULT_MAXTIME = -1L; 071 public static final long DAYS = 24*60*60; // factor to get caching time in days 072 073 /** 074 * Constructs a CachedFile object from a given filename, URL or internal resource. 075 * 076 * @param name can be:<ul> 077 * <li>relative or absolute file name</li> 078 * <li>{@code file:///SOME/FILE} the same as above</li> 079 * <li>{@code http://...} a URL. It will be cached on disk.</li></ul> 080 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 081 * <li>{@code josmdir://SOME/FILE} file inside josm config directory (since r7058)</li></ul> 082 */ 083 public CachedFile(String name) { 084 this.name = name; 085 } 086 087 /** 088 * Set the name of the resource. 089 * @param name can be:<ul> 090 * <li>relative or absolute file name</li> 091 * <li>{@code file:///SOME/FILE} the same as above</li> 092 * <li>{@code http://...} a URL. It will be cached on disk.</li></ul> 093 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 094 * <li>{@code josmdir://SOME/FILE} file inside josm config directory (since r7058)</li></ul> 095 * @return this object 096 */ 097 public CachedFile setName(String name) { 098 this.name = name; 099 return this; 100 } 101 102 /** 103 * Set maximum age of cache file. Only applies to URLs. 104 * When this time has passed after the last download of the file, the 105 * cache is considered stale and a new download will be attempted. 106 * @param maxAge the maximum cache age in seconds 107 * @return this object 108 */ 109 public CachedFile setMaxAge(long maxAge) { 110 this.maxAge = maxAge; 111 return this; 112 } 113 114 /** 115 * Set the destination directory for the cache file. Only applies to URLs. 116 * @param destDir the destination directory 117 * @return this object 118 */ 119 public CachedFile setDestDir(String destDir) { 120 this.destDir = destDir; 121 return this; 122 } 123 124 /** 125 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs. 126 * @param httpAccept the accepted MIME types 127 * @return this object 128 */ 129 public CachedFile setHttpAccept(String httpAccept) { 130 this.httpAccept = httpAccept; 131 return this; 132 } 133 134 /** 135 * Set the caching strategy. Only applies to URLs. 136 * @param cachingStrategy 137 * @return this object 138 */ 139 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) { 140 this.cachingStrategy = cachingStrategy; 141 return this; 142 } 143 144 public String getName() { 145 return name; 146 } 147 148 public long getMaxAge() { 149 return maxAge; 150 } 151 152 public String getDestDir() { 153 return destDir; 154 } 155 156 public String getHttpAccept() { 157 return httpAccept; 158 } 159 160 public CachingStrategy getCachingStrategy() { 161 return cachingStrategy; 162 } 163 164 /** 165 * Get InputStream to the requested resource. 166 * @return the InputStream 167 * @throws IOException when the resource with the given name could not be retrieved 168 */ 169 public InputStream getInputStream() throws IOException { 170 File file = getFile(); 171 if (file == null) { 172 if (name.startsWith("resource://")) { 173 InputStream is = getClass().getResourceAsStream( 174 name.substring("resource:/".length())); 175 if (is == null) 176 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name)); 177 return is; 178 } else throw new IOException(); 179 } 180 return new FileInputStream(file); 181 } 182 183 /** 184 * Get local file for the requested resource. 185 * @return The local cache file for URLs. If the resource is a local file, 186 * returns just that file. 187 * @throws IOException when the resource with the given name could not be retrieved 188 */ 189 public File getFile() throws IOException { 190 if (initialized) 191 return cacheFile; 192 initialized = true; 193 URL url; 194 try { 195 url = new URL(name); 196 if ("file".equals(url.getProtocol())) { 197 cacheFile = new File(name.substring("file:/".length())); 198 if (!cacheFile.exists()) { 199 cacheFile = new File(name.substring("file://".length())); 200 } 201 } else { 202 cacheFile = checkLocal(url); 203 } 204 } catch (MalformedURLException e) { 205 if (name.startsWith("resource://")) { 206 return null; 207 } else if (name.startsWith("josmdir://")) { 208 cacheFile = new File(Main.pref.getPreferencesDir(), name.substring("josmdir://".length())); 209 } else { 210 cacheFile = new File(name); 211 } 212 } 213 if (cacheFile == null) 214 throw new IOException("Unable to get cache file for "+name); 215 return cacheFile; 216 } 217 218 /** 219 * Looks for a certain entry inside a zip file and returns the entry path. 220 * 221 * Replies a file in the top level directory of the ZIP file which has an 222 * extension <code>extension</code>. If more than one files have this 223 * extension, the last file whose name includes <code>namepart</code> 224 * is opened. 225 * 226 * @param extension the extension of the file we're looking for 227 * @param namepart the name part 228 * @return The zip entry path of the matching file. Null if this cached file 229 * doesn't represent a zip file or if there was no matching 230 * file in the ZIP file. 231 */ 232 public String findZipEntryPath(String extension, String namepart) { 233 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 234 if (ze == null) return null; 235 return ze.a; 236 } 237 238 /** 239 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream. 240 * @param extension the extension of the file we're looking for 241 * @param namepart the name part 242 * @return InputStream to the matching file. Null if this cached file 243 * doesn't represent a zip file or if there was no matching 244 * file in the ZIP file. 245 * @since 6148 246 */ 247 public InputStream findZipEntryInputStream(String extension, String namepart) { 248 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 249 if (ze == null) return null; 250 return ze.b; 251 } 252 253 @SuppressWarnings("resource") 254 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) { 255 File file = null; 256 try { 257 file = getFile(); 258 } catch (IOException ex) { 259 } 260 if (file == null) 261 return null; 262 Pair<String, InputStream> res = null; 263 try { 264 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8); 265 ZipEntry resentry = null; 266 Enumeration<? extends ZipEntry> entries = zipFile.entries(); 267 while (entries.hasMoreElements()) { 268 ZipEntry entry = entries.nextElement(); 269 if (entry.getName().endsWith("." + extension)) { 270 /* choose any file with correct extension. When more than 271 one file, prefer the one which matches namepart */ 272 if (resentry == null || entry.getName().indexOf(namepart) >= 0) { 273 resentry = entry; 274 } 275 } 276 } 277 if (resentry != null) { 278 InputStream is = zipFile.getInputStream(resentry); 279 res = Pair.create(resentry.getName(), is); 280 } else { 281 Utils.close(zipFile); 282 } 283 } catch (Exception e) { 284 if (file.getName().endsWith(".zip")) { 285 Main.warn(tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}", 286 file.getName(), e.toString(), extension, namepart)); 287 } 288 } 289 return res; 290 } 291 292 /** 293 * Clear the cache for the given resource. 294 * This forces a fresh download. 295 * @param name the URL 296 */ 297 public static void cleanup(String name) { 298 cleanup(name, null); 299 } 300 301 /** 302 * Clear the cache for the given resource. 303 * This forces a fresh download. 304 * @param name the URL 305 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)}) 306 */ 307 public static void cleanup(String name, String destDir) { 308 URL url; 309 try { 310 url = new URL(name); 311 if (!"file".equals(url.getProtocol())) { 312 String prefKey = getPrefKey(url, destDir); 313 List<String> localPath = new ArrayList<>(Main.pref.getCollection(prefKey)); 314 if (localPath.size() == 2) { 315 File lfile = new File(localPath.get(1)); 316 if(lfile.exists()) { 317 lfile.delete(); 318 } 319 } 320 Main.pref.putCollection(prefKey, null); 321 } 322 } catch (MalformedURLException e) { 323 Main.warn(e); 324 } 325 } 326 327 /** 328 * Get preference key to store the location and age of the cached file. 329 * 2 resources that point to the same url, but that are to be stored in different 330 * directories will not share a cache file. 331 */ 332 private static String getPrefKey(URL url, String destDir) { 333 StringBuilder prefKey = new StringBuilder("mirror."); 334 if (destDir != null) { 335 prefKey.append(destDir); 336 prefKey.append("."); 337 } 338 prefKey.append(url.toString()); 339 return prefKey.toString().replaceAll("=","_"); 340 } 341 342 private File checkLocal(URL url) throws IOException { 343 String prefKey = getPrefKey(url, destDir); 344 String urlStr = url.toExternalForm(); 345 long age = 0L; 346 long lMaxAge = maxAge; 347 Long ifModifiedSince = null; 348 File localFile = null; 349 List<String> localPathEntry = new ArrayList<>(Main.pref.getCollection(prefKey)); 350 boolean offline = false; 351 try { 352 checkOfflineAccess(urlStr); 353 } catch (OfflineAccessException e) { 354 offline = true; 355 } 356 if (localPathEntry.size() == 2) { 357 localFile = new File(localPathEntry.get(1)); 358 if (!localFile.exists()) { 359 localFile = null; 360 } else { 361 if ( maxAge == DEFAULT_MAXTIME 362 || maxAge <= 0 // arbitrary value <= 0 is deprecated 363 ) { 364 lMaxAge = Main.pref.getInteger("mirror.maxtime", 7*24*60*60); // one week 365 } 366 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0)); 367 if (offline || age < lMaxAge*1000) { 368 return localFile; 369 } 370 if (cachingStrategy == CachingStrategy.IfModifiedSince) { 371 ifModifiedSince = Long.parseLong(localPathEntry.get(0)); 372 } 373 } 374 } 375 if (destDir == null) { 376 destDir = Main.pref.getCacheDirectory().getPath(); 377 } 378 379 File destDirFile = new File(destDir); 380 if (!destDirFile.exists()) { 381 destDirFile.mkdirs(); 382 } 383 384 // No local file + offline => nothing to do 385 if (offline) { 386 return null; 387 } 388 389 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_"); 390 String localPath = "mirror_" + a; 391 destDirFile = new File(destDir, localPath + ".tmp"); 392 try { 393 HttpURLConnection con = connectFollowingRedirect(url, httpAccept, ifModifiedSince); 394 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { 395 if (Main.isDebugEnabled()) { 396 Main.debug("304 Not Modified ("+urlStr+")"); 397 } 398 if (localFile == null) 399 throw new AssertionError(); 400 Main.pref.putCollection(prefKey, 401 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1))); 402 return localFile; 403 } 404 try ( 405 InputStream bis = new BufferedInputStream(con.getInputStream()); 406 OutputStream fos = new FileOutputStream(destDirFile); 407 OutputStream bos = new BufferedOutputStream(fos) 408 ) { 409 byte[] buffer = new byte[4096]; 410 int length; 411 while ((length = bis.read(buffer)) > -1) { 412 bos.write(buffer, 0, length); 413 } 414 } 415 localFile = new File(destDir, localPath); 416 if (Main.platform.rename(destDirFile, localFile)) { 417 Main.pref.putCollection(prefKey, 418 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString())); 419 } else { 420 Main.warn(tr("Failed to rename file {0} to {1}.", 421 destDirFile.getPath(), localFile.getPath())); 422 } 423 } catch (IOException e) { 424 if (age >= lMaxAge*1000 && age < lMaxAge*1000*2) { 425 Main.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e)); 426 return localFile; 427 } else { 428 throw e; 429 } 430 } 431 432 return localFile; 433 } 434 435 private static void checkOfflineAccess(String urlString) { 436 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Main.getJOSMWebsite()); 437 OnlineResource.OSM_API.checkOfflineAccess(urlString, Main.pref.get("osm-server.url", OsmApi.DEFAULT_API_URL)); 438 } 439 440 /** 441 * Opens a connection for downloading a resource. 442 * <p> 443 * Manually follows redirects because 444 * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect 445 * is going from a http to a https URL, see <a href="https://bugs.openjdk.java.net/browse/JDK-4620571">bug report</a>. 446 * <p> 447 * This can cause problems when downloading from certain GitHub URLs. 448 * 449 * @param downloadUrl The resource URL to download 450 * @param httpAccept The accepted MIME types sent in the HTTP Accept header. Can be {@code null} 451 * @param ifModifiedSince The download time of the cache file, optional 452 * @return The HTTP connection effectively linked to the resource, after all potential redirections 453 * @throws MalformedURLException If a redirected URL is wrong 454 * @throws IOException If any I/O operation goes wrong 455 * @throws OfflineAccessException if resource is accessed in offline mode, in any protocol 456 * @since 6867 457 */ 458 public static HttpURLConnection connectFollowingRedirect(URL downloadUrl, String httpAccept, Long ifModifiedSince) throws MalformedURLException, IOException { 459 CheckParameterUtil.ensureParameterNotNull(downloadUrl, "downloadUrl"); 460 String downloadString = downloadUrl.toExternalForm(); 461 462 checkOfflineAccess(downloadString); 463 464 HttpURLConnection con = null; 465 int numRedirects = 0; 466 while(true) { 467 con = Utils.openHttpConnection(downloadUrl); 468 if (con == null) { 469 throw new IOException("Cannot open http connection to "+downloadString); 470 } 471 if (ifModifiedSince != null) { 472 con.setIfModifiedSince(ifModifiedSince); 473 } 474 con.setInstanceFollowRedirects(false); 475 con.setConnectTimeout(Main.pref.getInteger("socket.timeout.connect",15)*1000); 476 con.setReadTimeout(Main.pref.getInteger("socket.timeout.read",30)*1000); 477 if (Main.isDebugEnabled()) { 478 Main.debug("GET "+downloadString); 479 } 480 if (httpAccept != null) { 481 if (Main.isTraceEnabled()) { 482 Main.trace("Accept: "+httpAccept); 483 } 484 con.setRequestProperty("Accept", httpAccept); 485 } 486 try { 487 con.connect(); 488 } catch (IOException e) { 489 Main.addNetworkError(downloadUrl, Utils.getRootCause(e)); 490 throw e; 491 } 492 switch(con.getResponseCode()) { 493 case HttpURLConnection.HTTP_OK: 494 return con; 495 case HttpURLConnection.HTTP_NOT_MODIFIED: 496 if (ifModifiedSince != null) 497 return con; 498 case HttpURLConnection.HTTP_MOVED_PERM: 499 case HttpURLConnection.HTTP_MOVED_TEMP: 500 case HttpURLConnection.HTTP_SEE_OTHER: 501 String redirectLocation = con.getHeaderField("Location"); 502 if (redirectLocation == null) { 503 /* I18n: argument is HTTP response code */ 504 String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header."+ 505 " Can''t redirect. Aborting.", con.getResponseCode()); 506 throw new IOException(msg); 507 } 508 downloadUrl = new URL(redirectLocation); 509 downloadString = downloadUrl.toExternalForm(); 510 // keep track of redirect attempts to break a redirect loops if it happens 511 // to occur for whatever reason 512 numRedirects++; 513 if (numRedirects >= Main.pref.getInteger("socket.maxredirects", 5)) { 514 String msg = tr("Too many redirects to the download URL detected. Aborting."); 515 throw new IOException(msg); 516 } 517 Main.info(tr("Download redirected to ''{0}''", downloadString)); 518 break; 519 default: 520 String msg = tr("Failed to read from ''{0}''. Server responded with status code {1}.", downloadString, con.getResponseCode()); 521 throw new IOException(msg); 522 } 523 } 524 } 525}