Libosmium  2.12.2
Fast and flexible C++ library for working with OpenStreetMap data
buffer.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_MEMORY_BUFFER_HPP
2 #define OSMIUM_MEMORY_BUFFER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2017 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstring>
40 #include <functional>
41 #include <iterator>
42 #include <memory>
43 #include <stdexcept>
44 #include <utility>
45 
46 #include <osmium/memory/item.hpp>
48 #include <osmium/osm/entity.hpp>
50 
51 namespace osmium {
52 
58  struct buffer_is_full : public std::runtime_error {
59 
61  std::runtime_error("Osmium buffer is full") {
62  }
63 
64  }; // struct buffer_is_full
65 
69  namespace memory {
70 
97  class Buffer {
98 
99  public:
100 
101  // This is needed so we can call std::back_inserter() on a Buffer.
102  using value_type = Item;
103 
104  enum class auto_grow : bool {
105  yes = true,
106  no = false
107  }; // enum class auto_grow
108 
109  private:
110 
111  std::unique_ptr<unsigned char[]> m_memory;
112  unsigned char* m_data;
113  size_t m_capacity;
114  size_t m_written;
115  size_t m_committed;
116 #ifndef NDEBUG
117  uint8_t m_builder_count{0};
118 #endif
119  auto_grow m_auto_grow{auto_grow::no};
120  std::function<void(Buffer&)> m_full;
121 
122  static size_t calculate_capacity(size_t capacity) noexcept {
123  // The majority of all Nodes will fit into this size.
124  constexpr static const size_t min_capacity = 64;
125  if (capacity < min_capacity) {
126  return min_capacity;
127  }
128  return padded_length(capacity);
129  }
130 
131  public:
132 
141  Buffer() noexcept :
142  m_memory(),
143  m_data(nullptr),
144  m_capacity(0),
145  m_written(0),
146  m_committed(0) {
147  }
148 
159  explicit Buffer(unsigned char* data, size_t size) :
160  m_memory(),
161  m_data(data),
162  m_capacity(size),
163  m_written(size),
164  m_committed(size) {
165  if (size % align_bytes != 0) {
166  throw std::invalid_argument("buffer size needs to be multiple of alignment");
167  }
168  }
169 
181  explicit Buffer(unsigned char* data, size_t capacity, size_t committed) :
182  m_memory(),
183  m_data(data),
184  m_capacity(capacity),
185  m_written(committed),
186  m_committed(committed) {
187  if (capacity % align_bytes != 0) {
188  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
189  }
190  if (committed % align_bytes != 0) {
191  throw std::invalid_argument("buffer parameter 'committed' needs to be multiple of alignment");
192  }
193  }
194 
207  explicit Buffer(size_t capacity, auto_grow auto_grow = auto_grow::yes) :
208  m_memory(new unsigned char[calculate_capacity(capacity)]),
209  m_data(m_memory.get()),
210  m_capacity(calculate_capacity(capacity)),
211  m_written(0),
212  m_committed(0),
213  m_auto_grow(auto_grow) {
214  }
215 
216  // buffers can not be copied
217  Buffer(const Buffer&) = delete;
218  Buffer& operator=(const Buffer&) = delete;
219 
220  // buffers can be moved
221  Buffer(Buffer&&) = default;
222  Buffer& operator=(Buffer&&) = default;
223 
224  ~Buffer() = default;
225 
226 #ifndef NDEBUG
227  void increment_builder_count() noexcept {
228  ++m_builder_count;
229  }
230 
231  void decrement_builder_count() noexcept {
232  assert(m_builder_count > 0);
233  --m_builder_count;
234  }
235 
236  uint8_t builder_count() const noexcept {
237  return m_builder_count;
238  }
239 #endif
240 
246  unsigned char* data() const noexcept {
247  assert(m_data && "This must be a valid buffer");
248  return m_data;
249  }
250 
255  size_t capacity() const noexcept {
256  return m_capacity;
257  }
258 
263  size_t committed() const noexcept {
264  return m_committed;
265  }
266 
272  size_t written() const noexcept {
273  return m_written;
274  }
275 
282  bool is_aligned() const noexcept {
283  assert(m_data && "This must be a valid buffer");
284  return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
285  }
286 
307  OSMIUM_DEPRECATED void set_full_callback(std::function<void(Buffer&)> full) {
308  assert(m_data && "This must be a valid buffer");
309  m_full = full;
310  }
311 
328  void grow(size_t size) {
329  assert(m_data && "This must be a valid buffer");
330  if (!m_memory) {
331  throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management.");
332  }
333  if (m_capacity < size) {
334  if (size % align_bytes != 0) {
335  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
336  }
337  std::unique_ptr<unsigned char[]> memory(new unsigned char[size]);
338  std::copy_n(m_memory.get(), m_capacity, memory.get());
339  using std::swap;
340  swap(m_memory, memory);
341  m_data = m_memory.get();
342  m_capacity = size;
343  }
344  }
345 
358  size_t commit() {
359  assert(m_data && "This must be a valid buffer");
360  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
361  assert(is_aligned());
362 
363  const size_t offset = m_committed;
364  m_committed = m_written;
365  return offset;
366  }
367 
374  void rollback() {
375  assert(m_data && "This must be a valid buffer");
376  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
377  m_written = m_committed;
378  }
379 
389  size_t clear() {
390  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
391  const size_t committed = m_committed;
392  m_written = 0;
393  m_committed = 0;
394  return committed;
395  }
396 
407  template <typename T>
408  T& get(const size_t offset) const {
409  assert(m_data && "This must be a valid buffer");
410  return *reinterpret_cast<T*>(&m_data[offset]);
411  }
412 
446  unsigned char* reserve_space(const size_t size) {
447  assert(m_data && "This must be a valid buffer");
448  // try to flush the buffer empty first.
449  if (m_written + size > m_capacity && m_full) {
450  m_full(*this);
451  }
452  // if there's still not enough space, then try growing the buffer.
453  if (m_written + size > m_capacity) {
454  if (m_memory && (m_auto_grow == auto_grow::yes)) {
455  // double buffer size until there is enough space
456  size_t new_capacity = m_capacity * 2;
457  while (m_written + size > new_capacity) {
458  new_capacity *= 2;
459  }
460  grow(new_capacity);
461  } else {
462  throw osmium::buffer_is_full();
463  }
464  }
465  unsigned char* data = &m_data[m_written];
466  m_written += size;
467  return data;
468  }
469 
485  template <typename T>
486  T& add_item(const T& item) {
487  assert(m_data && "This must be a valid buffer");
488  unsigned char* target = reserve_space(item.padded_size());
489  std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
490  return *reinterpret_cast<T*>(target);
491  }
492 
504  void add_buffer(const Buffer& buffer) {
505  assert(m_data && "This must be a valid buffer");
506  assert(buffer && "Buffer parameter must be a valid buffer");
507  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
508  unsigned char* target = reserve_space(buffer.committed());
509  std::copy_n(buffer.data(), buffer.committed(), target);
510  }
511 
521  void push_back(const osmium::memory::Item& item) {
522  assert(m_data && "This must be a valid buffer");
523  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
524  add_item(item);
525  commit();
526  }
527 
532  template <typename T>
534 
539  template <typename T>
541 
547 
553 
554  template <typename T>
556  return ItemIteratorRange<T>{m_data, m_data + m_committed};
557  }
558 
559  template <typename T>
561  return ItemIteratorRange<const T>{m_data, m_data + m_committed};
562  }
563 
572  template <typename T>
574  assert(m_data && "This must be a valid buffer");
575  return t_iterator<T>(m_data, m_data + m_committed);
576  }
577 
587  assert(m_data && "This must be a valid buffer");
588  return iterator(m_data, m_data + m_committed);
589  }
590 
600  template <typename T>
601  t_iterator<T> get_iterator(size_t offset) {
602  assert(m_data && "This must be a valid buffer");
603  return t_iterator<T>(m_data + offset, m_data + m_committed);
604  }
605 
615  iterator get_iterator(size_t offset) {
616  assert(m_data && "This must be a valid buffer");
617  return iterator(m_data + offset, m_data + m_committed);
618  }
619 
628  template <typename T>
630  assert(m_data && "This must be a valid buffer");
631  return t_iterator<T>(m_data + m_committed, m_data + m_committed);
632  }
633 
643  assert(m_data && "This must be a valid buffer");
644  return iterator(m_data + m_committed, m_data + m_committed);
645  }
646 
647  template <typename T>
649  assert(m_data && "This must be a valid buffer");
650  return t_const_iterator<T>(m_data, m_data + m_committed);
651  }
652 
654  assert(m_data && "This must be a valid buffer");
655  return const_iterator(m_data, m_data + m_committed);
656  }
657 
658  template <typename T>
659  t_const_iterator<T> get_iterator(size_t offset) const {
660  assert(m_data && "This must be a valid buffer");
661  return t_const_iterator<T>(m_data + offset, m_data + m_committed);
662  }
663 
664  const_iterator get_iterator(size_t offset) const {
665  assert(m_data && "This must be a valid buffer");
666  return const_iterator(m_data + offset, m_data + m_committed);
667  }
668 
669  template <typename T>
671  assert(m_data && "This must be a valid buffer");
672  return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);
673  }
674 
676  assert(m_data && "This must be a valid buffer");
677  return const_iterator(m_data + m_committed, m_data + m_committed);
678  }
679 
680  template <typename T>
682  return cbegin<T>();
683  }
684 
686  return cbegin();
687  }
688 
689  template <typename T>
691  return cend<T>();
692  }
693 
694  const_iterator end() const {
695  return cend();
696  }
697 
701  explicit operator bool() const noexcept {
702  return m_data != nullptr;
703  }
704 
705  void swap(Buffer& other) {
706  using std::swap;
707 
708  swap(m_memory, other.m_memory);
709  swap(m_data, other.m_data);
710  swap(m_capacity, other.m_capacity);
711  swap(m_written, other.m_written);
712  swap(m_committed, other.m_committed);
713  swap(m_auto_grow, other.m_auto_grow);
714  swap(m_full, other.m_full);
715  }
716 
733  template <typename TCallbackClass>
734  void purge_removed(TCallbackClass* callback) {
735  assert(m_data && "This must be a valid buffer");
736  if (begin() == end()) {
737  return;
738  }
739 
740  iterator it_write = begin();
741 
742  iterator next;
743  for (iterator it_read = begin(); it_read != end(); it_read = next) {
744  next = std::next(it_read);
745  if (!it_read->removed()) {
746  if (it_read != it_write) {
747  assert(it_read.data() >= data());
748  assert(it_write.data() >= data());
749  size_t old_offset = static_cast<size_t>(it_read.data() - data());
750  size_t new_offset = static_cast<size_t>(it_write.data() - data());
751  callback->moving_in_buffer(old_offset, new_offset);
752  std::memmove(it_write.data(), it_read.data(), it_read->padded_size());
753  }
754  it_write.advance_once();
755  }
756  }
757 
758  assert(it_write.data() >= data());
759  m_written = static_cast<size_t>(it_write.data() - data());
760  m_committed = m_written;
761  }
762 
763  }; // class Buffer
764 
765  inline void swap(Buffer& lhs, Buffer& rhs) {
766  lhs.swap(rhs);
767  }
768 
776  inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
777  if (!lhs || !rhs) {
778  return !lhs && !rhs;
779  }
780  return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
781  }
782 
783  inline bool operator!=(const Buffer& lhs, const Buffer& rhs) noexcept {
784  return ! (lhs == rhs);
785  }
786 
787  } // namespace memory
788 
789 } // namespace osmium
790 
791 #endif // OSMIUM_MEMORY_BUFFER_HPP
size_t m_written
Definition: buffer.hpp:114
void swap(Buffer &other)
Definition: buffer.hpp:705
t_const_iterator< T > begin() const
Definition: buffer.hpp:681
size_t clear()
Definition: buffer.hpp:389
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
bool is_aligned() const noexcept
Definition: buffer.hpp:282
t_const_iterator< T > get_iterator(size_t offset) const
Definition: buffer.hpp:659
size_t written() const noexcept
Definition: buffer.hpp:272
OSMIUM_DEPRECATED void set_full_callback(std::function< void(Buffer &)> full)
Definition: buffer.hpp:307
Definition: item_iterator.hpp:175
iterator get_iterator(size_t offset)
Definition: buffer.hpp:615
void grow(size_t size)
Definition: buffer.hpp:328
t_const_iterator< T > cend() const
Definition: buffer.hpp:670
const_iterator cend() const
Definition: buffer.hpp:675
Definition: item_iterator.hpp:59
constexpr bool operator==(const Box &lhs, const Box &rhs) noexcept
Definition: box.hpp:221
unsigned char * m_data
Definition: buffer.hpp:112
const_iterator get_iterator(size_t offset) const
Definition: buffer.hpp:664
void increment_builder_count() noexcept
Definition: buffer.hpp:227
Buffer(unsigned char *data, size_t capacity, size_t committed)
Definition: buffer.hpp:181
Definition: reader_iterator.hpp:39
ItemIteratorRange< const T > select() const
Definition: buffer.hpp:560
void swap(Buffer &lhs, Buffer &rhs)
Definition: buffer.hpp:765
Buffer(size_t capacity, auto_grow auto_grow=auto_grow::yes)
Definition: buffer.hpp:207
constexpr std::size_t padded_length(std::size_t length) noexcept
Definition: item.hpp:64
const_iterator begin() const
Definition: buffer.hpp:685
t_iterator< T > end()
Definition: buffer.hpp:629
Definition: item.hpp:105
Namespace for everything in the Osmium library.
Definition: assembler.hpp:63
T & add_item(const T &item)
Definition: buffer.hpp:486
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:734
t_iterator< T > begin()
Definition: buffer.hpp:573
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:504
uint8_t builder_count() const noexcept
Definition: buffer.hpp:236
size_t m_committed
Definition: buffer.hpp:115
ItemIterator< TMember > & advance_once() noexcept
Definition: item_iterator.hpp:114
unsigned char * data() const noexcept
Definition: buffer.hpp:246
Buffer() noexcept
Definition: buffer.hpp:141
size_t capacity() const noexcept
Definition: buffer.hpp:255
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
unsigned char * reserve_space(const size_t size)
Definition: buffer.hpp:446
auto_grow m_auto_grow
Definition: buffer.hpp:119
void push_back(const osmium::memory::Item &item)
Definition: buffer.hpp:521
constexpr const item_size_type align_bytes
Definition: item.hpp:62
iterator end()
Definition: buffer.hpp:642
iterator begin()
Definition: buffer.hpp:586
size_t m_capacity
Definition: buffer.hpp:113
size_t committed() const noexcept
Definition: buffer.hpp:263
Buffer(unsigned char *data, size_t size)
Definition: buffer.hpp:159
static size_t calculate_capacity(size_t capacity) noexcept
Definition: buffer.hpp:122
Definition: buffer.hpp:97
t_const_iterator< T > cbegin() const
Definition: buffer.hpp:648
Definition: buffer.hpp:58
void decrement_builder_count() noexcept
Definition: buffer.hpp:231
const_iterator end() const
Definition: buffer.hpp:694
auto_grow
Definition: buffer.hpp:104
t_const_iterator< T > end() const
Definition: buffer.hpp:690
data_type data() noexcept
Definition: item_iterator.hpp:135
buffer_is_full()
Definition: buffer.hpp:60
ItemIteratorRange< T > select()
Definition: buffer.hpp:555
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
void rollback()
Definition: buffer.hpp:374
std::unique_ptr< unsigned char[]> m_memory
Definition: buffer.hpp:111
bool operator!=(const Changeset &lhs, const Changeset &rhs)
Definition: changeset.hpp:440
t_iterator< T > get_iterator(size_t offset)
Definition: buffer.hpp:601
const_iterator cbegin() const
Definition: buffer.hpp:653
std::function< void(Buffer &)> m_full
Definition: buffer.hpp:120
size_t commit()
Definition: buffer.hpp:358