Libosmium  2.2.0
Fast and flexible C++ library for working with OpenStreetMap data
memory_mapping.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_UTIL_MEMORY_MAPPING_HPP
2 #define OSMIUM_UTIL_MEMORY_MAPPING_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <cassert>
37 #include <cerrno>
38 #include <stdexcept>
39 #include <system_error>
40 
41 #include <osmium/util/file.hpp>
42 
43 #ifndef _WIN32
44 # include <sys/mman.h>
45 #else
46 # include <io.h>
47 # include <windows.h>
48 # include <sys/types.h>
49 #endif
50 
51 namespace osmium {
52 
53  namespace util {
54 
89  class MemoryMapping {
90 
92  size_t m_size;
93 
95  off_t m_offset;
96 
98  int m_fd;
99 
102 
103 #ifdef _WIN32
104  HANDLE m_handle;
105 #endif
106 
108  void* m_addr;
109 
110  bool is_valid() const noexcept;
111 
112  void make_invalid() noexcept;
113 
114 #ifdef _WIN32
115  typedef DWORD flag_type;
116 #else
117  typedef int flag_type;
118 #endif
119 
120  flag_type get_protection() const noexcept;
121 
122  flag_type get_flags() const noexcept;
123 
124  // A zero-sized mapping is not allowed by the operating system.
125  // So if the user asks for a mapping of size 0, we map a full
126  // page instead. This way we don't have a special case in the rest
127  // of the code.
128  static size_t initial_size(size_t size) {
129  if (size == 0) {
131  }
132  return size;
133  }
134 
135 #ifdef _WIN32
136  HANDLE get_handle() const noexcept;
137  HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept;
138  void* osmium::util::MemoryMapping::map_view_of_file() const noexcept;
139 #endif
140 
141  int resize_fd(int fd) {
142  // Anonymous mapping doesn't need resizing.
143  if (fd == -1) {
144  return -1;
145  }
146 
147  // Make sure the file backing this mapping is large enough.
148  if (osmium::util::file_size(fd) < m_size + m_offset) {
149  osmium::util::resize_file(fd, m_size + m_offset);
150  }
151  return fd;
152  }
153 
154  public:
155 
171  MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0);
172 
174  MemoryMapping(const MemoryMapping&) = delete;
175 
177  MemoryMapping& operator=(const MemoryMapping&) = delete;
178 
183  MemoryMapping(MemoryMapping&& other);
184 
189 
194  ~MemoryMapping() noexcept {
195  try {
196  unmap();
197  } catch (std::system_error&) {
198  // ignore
199  }
200  }
201 
208  void unmap();
209 
220  void resize(size_t new_size);
221 
226  operator bool() const noexcept {
227  return is_valid();
228  }
229 
235  size_t size() const noexcept {
236  return m_size;
237  }
238 
244  int fd() const noexcept {
245  return m_fd;
246  }
247 
251  bool writable() const noexcept {
252  return m_writable;
253  }
254 
260  template <typename T = void>
261  T* get_addr() const {
262  if (is_valid()) {
263  return reinterpret_cast<T*>(m_addr);
264  }
265  throw std::runtime_error("invalid memory mapping");
266  }
267 
268  }; // class MemoryMapping
269 
281 
282  public:
283 
285  MemoryMapping(size) {
286  }
287 
288 #ifndef __linux__
289 
293  void resize(size_t) = delete;
294 #endif
295 
296  }; // class AnonymousMemoryMapping
297 
307  template <typename T>
309 
311 
312  public:
313 
321  m_mapping(sizeof(T) * size) {
322  }
323 
334  TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) :
335  m_mapping(sizeof(T) * size, writable, fd, sizeof(T) * offset) {
336  }
337 
339  TypedMemoryMapping(const TypedMemoryMapping&) = delete;
340 
343 
348  TypedMemoryMapping(TypedMemoryMapping&& other) = default;
349 
353  TypedMemoryMapping& operator=(TypedMemoryMapping&& other) = default;
354 
359  ~TypedMemoryMapping() = default;
360 
367  void unmap() {
368  m_mapping.unmap();
369  }
370 
381  void resize(size_t new_size) {
382  m_mapping.resize(sizeof(T) * new_size);
383  }
384 
389  operator bool() const noexcept {
390  return !!m_mapping;
391  }
392 
398  size_t size() const noexcept {
399  assert(m_mapping.size() % sizeof(T) == 0);
400  return m_mapping.size() / sizeof(T);
401  }
402 
408  int fd() const noexcept {
409  return m_mapping.fd();
410  }
411 
415  bool writable() const noexcept {
416  return m_mapping.writable();
417  }
418 
424  T* begin() {
425  return m_mapping.get_addr<T>();
426  }
427 
433  T* end() {
434  return m_mapping.get_addr<T>() + size();
435  }
436 
437  const T* cbegin() const {
438  return m_mapping.get_addr<T>();
439  }
440 
441  const T* cend() const {
442  return m_mapping.get_addr<T>() + size();
443  }
444 
445  const T* begin() const {
446  return m_mapping.get_addr<T>();
447  }
448 
449  const T* end() const {
450  return m_mapping.get_addr<T>() + size();
451  }
452 
453  }; // class TypedMemoryMapping
454 
455  template <typename T>
457 
458  public:
459 
461  TypedMemoryMapping<T>(size) {
462  }
463 
464 #ifndef __linux__
465 
469  void resize(size_t) = delete;
470 #endif
471 
472  }; // class AnonymousTypedMemoryMapping
473 
474  } // namespace util
475 
476 } // namespace osmium
477 
478 #ifndef _WIN32
479 
480 // =========== Unix implementation =============
481 
482 // MAP_FAILED is often a macro containing an old style cast
483 #pragma GCC diagnostic push
484 #pragma GCC diagnostic ignored "-Wold-style-cast"
485 
486 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
487  return m_addr != MAP_FAILED;
488 }
489 
491  m_addr = MAP_FAILED;
492 }
493 
494 #pragma GCC diagnostic pop
495 
496 // for BSD systems
497 #ifndef MAP_ANONYMOUS
498 # define MAP_ANONYMOUS MAP_ANON
499 #endif
500 
501 inline int osmium::util::MemoryMapping::get_protection() const noexcept {
502  if (m_writable) {
503  return PROT_READ | PROT_WRITE;
504  }
505  return PROT_READ;
506 }
507 
508 inline int osmium::util::MemoryMapping::get_flags() const noexcept {
509  if (m_fd == -1) {
510  return MAP_PRIVATE | MAP_ANONYMOUS;
511  }
512  return MAP_SHARED;
513 }
514 
515 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, bool writable, int fd, off_t offset) :
516  m_size(initial_size(size)),
517  m_offset(offset),
518  m_fd(resize_fd(fd)),
519  m_writable(writable),
520  m_addr(::mmap(nullptr, m_size, get_protection(), get_flags(), m_fd, m_offset)) {
521  assert(writable || fd != -1);
522  if (!is_valid()) {
523  throw std::system_error(errno, std::system_category(), "mmap failed");
524  }
525 }
526 
528  m_size(other.m_size),
529  m_offset(other.m_offset),
530  m_fd(other.m_fd),
531  m_writable(other.m_writable),
532  m_addr(other.m_addr) {
533  other.make_invalid();
534 }
535 
537  unmap();
538  m_size = other.m_size;
539  m_offset = other.m_offset;
540  m_fd = other.m_fd;
541  m_writable = other.m_writable;
542  m_addr = other.m_addr;
543  other.make_invalid();
544  return *this;
545 }
546 
548  if (is_valid()) {
549  if (::munmap(m_addr, m_size) != 0) {
550  throw std::system_error(errno, std::system_category(), "munmap failed");
551  }
552  make_invalid();
553  }
554 }
555 
556 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
557  assert(new_size > 0 && "can not resize to zero size");
558  if (m_fd == -1) { // anonymous mapping
559 #ifdef __linux__
560  m_addr = ::mremap(m_addr, m_size, new_size, MREMAP_MAYMOVE);
561  if (!is_valid()) {
562  throw std::system_error(errno, std::system_category(), "mremap failed");
563  }
564  m_size = new_size;
565 #else
566  assert(false && "can't resize anonymous mappings on non-linux systems");
567 #endif
568  } else { // file-based mapping
569  unmap();
570  m_size = new_size;
571  resize_fd(m_fd);
572  m_addr = ::mmap(nullptr, new_size, get_protection(), get_flags(), m_fd, m_offset);
573  if (!is_valid()) {
574  throw std::system_error(errno, std::system_category(), "mmap (remap) failed");
575  }
576  }
577 }
578 
579 #else
580 
581 // =========== Windows implementation =============
582 
583 /* References:
584  * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx
585  * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx
586  * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx
587  * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx
588  */
589 
590 namespace osmium {
591 
592  namespace util {
593 
594  inline DWORD dword_hi(uint64_t x) {
595  return static_cast<DWORD>(x >> 32);
596  }
597 
598  inline DWORD dword_lo(uint64_t x) {
599  return static_cast<DWORD>(x & 0xffffffff);
600  }
601 
602  } // namespace util
603 
604 } // namespace osmium
605 
606 inline DWORD osmium::util::MemoryMapping::get_protection() const noexcept {
607  if (m_writable) {
608  return PAGE_READWRITE;
609  }
610  return PAGE_READONLY;
611 }
612 
613 inline DWORD osmium::util::MemoryMapping::get_flags() const noexcept {
614  if (m_fd == -1) {
615  return FILE_MAP_WRITE | FILE_MAP_COPY;
616  }
617  if (m_writable) {
618  return FILE_MAP_WRITE;
619  }
620  return FILE_MAP_READ;
621 }
622 
623 inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept {
624  if (m_fd == -1) {
625  return INVALID_HANDLE_VALUE;
626  }
627  return reinterpret_cast<HANDLE>(_get_osfhandle(m_fd));
628 }
629 
630 inline HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept {
631  return CreateFileMapping(get_handle(), nullptr, get_protection(), osmium::util::dword_hi(static_cast<uint64_t>(m_size) + m_offset), osmium::util::dword_lo(static_cast<uint64_t>(m_size) + m_offset), nullptr);
632 }
633 
634 inline void* osmium::util::MemoryMapping::map_view_of_file() const noexcept {
635  return MapViewOfFile(m_handle, get_flags(), osmium::util::dword_hi(m_offset), osmium::util::dword_lo(m_offset), m_size);
636 }
637 
638 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
639  return m_addr != nullptr;
640 }
641 
642 inline void osmium::util::MemoryMapping::make_invalid() noexcept {
643  m_addr = nullptr;
644 }
645 
646 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, bool writable, int fd, off_t offset) :
647  m_size(initial_size(size)),
648  m_offset(offset),
649  m_fd(resize_fd(fd)),
650  m_writable(writable),
651  m_handle(create_file_mapping()),
652  m_addr(nullptr) {
653 
654  if (!m_handle) {
655  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
656  }
657 
658  m_addr = map_view_of_file();
659  if (!is_valid()) {
660  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
661  }
662 }
663 
664 inline osmium::util::MemoryMapping::MemoryMapping(MemoryMapping&& other) :
665  m_size(other.m_size),
666  m_offset(other.m_offset),
667  m_fd(other.m_fd),
668  m_writable(other.m_writable),
669  m_handle(std::move(other.m_handle)),
670  m_addr(other.m_addr) {
671  other.make_invalid();
672  other.m_handle = nullptr;
673 }
674 
676  unmap();
677  m_size = other.m_size;
678  m_offset = other.m_offset;
679  m_fd = other.m_fd;
680  m_writable = other.m_writable;
681  m_handle = std::move(other.m_handle);
682  m_addr = other.m_addr;
683  other.make_invalid();
684  other.m_handle = nullptr;
685  return *this;
686 }
687 
689  if (is_valid()) {
690  if (! UnmapViewOfFile(m_addr)) {
691  throw std::system_error(GetLastError(), std::system_category(), "UnmapViewOfFile failed");
692  }
693  make_invalid();
694  }
695 
696  if (m_handle) {
697  if (! CloseHandle(m_handle)) {
698  throw std::system_error(GetLastError(), std::system_category(), "CloseHandle failed");
699  }
700  m_handle = nullptr;
701  }
702 }
703 
704 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
705  unmap();
706 
707  m_size = new_size;
708  resize_fd(m_fd);
709 
710  m_handle = create_file_mapping();
711  if (!m_handle) {
712  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
713  }
714 
715  m_addr = map_view_of_file();
716  if (!is_valid()) {
717  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
718  }
719 }
720 
721 #endif
722 
723 #endif // OSMIUM_UTIL_MEMORY_MAPPING_HPP
~MemoryMapping() noexcept
Definition: memory_mapping.hpp:194
const T * end() const
Definition: memory_mapping.hpp:449
bool is_valid() const noexcept
Definition: memory_mapping.hpp:486
flag_type get_protection() const noexcept
Definition: memory_mapping.hpp:501
MemoryMapping m_mapping
Definition: memory_mapping.hpp:310
size_t file_size(int fd)
Definition: file.hpp:66
flag_type get_flags() const noexcept
Definition: memory_mapping.hpp:508
int fd() const noexcept
Definition: memory_mapping.hpp:244
TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset=0)
Definition: memory_mapping.hpp:334
static size_t initial_size(size_t size)
Definition: memory_mapping.hpp:128
void unmap()
Definition: memory_mapping.hpp:547
Definition: memory_mapping.hpp:89
Definition: reader_iterator.hpp:39
void resize(size_t new_size)
Definition: memory_mapping.hpp:381
int flag_type
Definition: memory_mapping.hpp:117
int resize_fd(int fd)
Definition: memory_mapping.hpp:141
T * end()
Definition: memory_mapping.hpp:433
size_t get_pagesize()
Definition: file.hpp:102
void * m_addr
The address where the memory is mapped.
Definition: memory_mapping.hpp:108
const T * cbegin() const
Definition: memory_mapping.hpp:437
off_t m_offset
Offset into the file.
Definition: memory_mapping.hpp:95
#define MAP_ANONYMOUS
Definition: memory_mapping.hpp:498
int m_fd
File handle we got the mapping from.
Definition: memory_mapping.hpp:98
size_t size() const noexcept
Definition: memory_mapping.hpp:235
bool m_writable
Is the memory writable?
Definition: memory_mapping.hpp:101
Namespace for everything in the Osmium library.
Definition: assembler.hpp:55
AnonymousMemoryMapping(size_t size)
Definition: memory_mapping.hpp:284
Definition: memory_mapping.hpp:456
size_t size() const noexcept
Definition: memory_mapping.hpp:398
T * begin()
Definition: memory_mapping.hpp:424
Definition: memory_mapping.hpp:280
void make_invalid() noexcept
Definition: memory_mapping.hpp:490
MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:515
AnonymousTypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:460
int fd() const noexcept
Definition: memory_mapping.hpp:408
void resize(size_t new_size)
Definition: memory_mapping.hpp:556
MemoryMapping & operator=(const MemoryMapping &)=delete
You can not copy a MemoryMapping.
Definition: memory_mapping.hpp:308
TypedMemoryMapping & operator=(const TypedMemoryMapping &)=delete
You can not copy a MemoryMapping.
void unmap()
Definition: memory_mapping.hpp:367
bool writable() const noexcept
Definition: memory_mapping.hpp:251
void resize_file(int fd, size_t new_size)
Definition: file.hpp:93
const T * begin() const
Definition: memory_mapping.hpp:445
size_t m_size
The size of the mapping.
Definition: memory_mapping.hpp:92
const T * cend() const
Definition: memory_mapping.hpp:441
TypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:320
T * get_addr() const
Definition: memory_mapping.hpp:261
bool writable() const noexcept
Definition: memory_mapping.hpp:415