Libosmium  2.5.3
Fast and flexible C++ library for working with OpenStreetMap data
buffer.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_MEMORY_BUFFER_HPP
2 #define OSMIUM_MEMORY_BUFFER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstring>
40 #include <functional>
41 #include <iterator>
42 #include <stdexcept>
43 #include <utility>
44 #include <vector>
45 
46 #include <osmium/memory/item.hpp>
48 #include <osmium/osm/entity.hpp>
50 
51 namespace osmium {
52 
58  struct buffer_is_full : public std::runtime_error {
59 
61  std::runtime_error("Osmium buffer is full") {
62  }
63 
64  }; // struct buffer_is_full
65 
69  namespace memory {
70 
97  class Buffer {
98 
99  public:
100 
101  enum class auto_grow : bool {
102  yes = true,
103  no = false
104  }; // enum class auto_grow
105 
106  private:
107 
108  std::vector<unsigned char> m_memory;
109  unsigned char* m_data;
110  size_t m_capacity;
111  size_t m_written;
112  size_t m_committed;
114  std::function<void(Buffer&)> m_full;
115 
116  public:
117 
126  Buffer() noexcept :
127  m_memory(),
128  m_data(nullptr),
129  m_capacity(0),
130  m_written(0),
131  m_committed(0) {
132  }
133 
144  explicit Buffer(unsigned char* data, size_t size) :
145  m_memory(),
146  m_data(data),
147  m_capacity(size),
148  m_written(size),
149  m_committed(size) {
150  if (size % align_bytes != 0) {
151  throw std::invalid_argument("buffer size needs to be multiple of alignment");
152  }
153  }
154 
166  explicit Buffer(unsigned char* data, size_t capacity, size_t committed) :
167  m_memory(),
168  m_data(data),
169  m_capacity(capacity),
170  m_written(committed),
171  m_committed(committed) {
172  if (capacity % align_bytes != 0) {
173  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
174  }
175  if (committed % align_bytes != 0) {
176  throw std::invalid_argument("buffer parameter 'committed' needs to be multiple of alignment");
177  }
178  }
179 
195  m_memory(capacity),
196  m_data(m_memory.data()),
197  m_capacity(capacity),
198  m_written(0),
199  m_committed(0),
201  if (capacity % align_bytes != 0) {
202  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
203  }
204  }
205 
206  // buffers can not be copied
207  Buffer(const Buffer&) = delete;
208  Buffer& operator=(const Buffer&) = delete;
209 
210  // buffers can be moved
211  Buffer(Buffer&&) = default;
212  Buffer& operator=(Buffer&&) = default;
213 
214  ~Buffer() = default;
215 
221  unsigned char* data() const noexcept {
222  assert(m_data);
223  return m_data;
224  }
225 
230  size_t capacity() const noexcept {
231  return m_capacity;
232  }
233 
238  size_t committed() const noexcept {
239  return m_committed;
240  }
241 
247  size_t written() const noexcept {
248  return m_written;
249  }
250 
257  bool is_aligned() const noexcept {
258  assert(m_data);
259  return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
260  }
261 
282  OSMIUM_DEPRECATED void set_full_callback(std::function<void(Buffer&)> full) {
283  assert(m_data);
284  m_full = full;
285  }
286 
303  void grow(size_t size) {
304  assert(m_data);
305  if (m_memory.empty()) {
306  throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management.");
307  }
308  if (m_capacity < size) {
309  if (size % align_bytes != 0) {
310  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
311  }
312  m_memory.resize(size);
313  m_data = m_memory.data();
314  m_capacity = size;
315  }
316  }
317 
328  size_t commit() {
329  assert(m_data);
330  assert(is_aligned());
331 
332  const size_t offset = m_committed;
333  m_committed = m_written;
334  return offset;
335  }
336 
342  void rollback() {
343  assert(m_data);
344  m_written = m_committed;
345  }
346 
354  size_t clear() {
355  const size_t committed = m_committed;
356  m_written = 0;
357  m_committed = 0;
358  return committed;
359  }
360 
371  template <typename T>
372  T& get(const size_t offset) const {
373  assert(m_data);
374  return *reinterpret_cast<T*>(&m_data[offset]);
375  }
376 
410  unsigned char* reserve_space(const size_t size) {
411  assert(m_data);
412  // try to flush the buffer empty first.
413  if (m_written + size > m_capacity && m_full) {
414  m_full(*this);
415  }
416  // if there's still not enough space, then try growing the buffer.
417  if (m_written + size > m_capacity) {
418  if (!m_memory.empty() && (m_auto_grow == auto_grow::yes)) {
419  // double buffer size until there is enough space
420  size_t new_capacity = m_capacity * 2;
421  while (m_written + size > new_capacity) {
422  new_capacity *= 2;
423  }
424  grow(new_capacity);
425  } else {
426  throw osmium::buffer_is_full();
427  }
428  }
429  unsigned char* data = &m_data[m_written];
430  m_written += size;
431  return data;
432  }
433 
449  template <typename T>
450  T& add_item(const T& item) {
451  assert(m_data);
452  unsigned char* target = reserve_space(item.padded_size());
453  std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
454  return *reinterpret_cast<T*>(target);
455  }
456 
467  void add_buffer(const Buffer& buffer) {
468  assert(m_data && buffer);
469  unsigned char* target = reserve_space(buffer.committed());
470  std::copy_n(buffer.data(), buffer.committed(), target);
471  }
472 
481  void push_back(const osmium::memory::Item& item) {
482  assert(m_data);
483  add_item(item);
484  commit();
485  }
486 
491  template <typename T>
493 
498  template <typename T>
500 
506 
512 
521  template <typename T>
523  assert(m_data);
524  return t_iterator<T>(m_data, m_data + m_committed);
525  }
526 
536  assert(m_data);
537  return iterator(m_data, m_data + m_committed);
538  }
539 
549  template <typename T>
550  t_iterator<T> get_iterator(size_t offset) {
551  assert(m_data);
552  return t_iterator<T>(m_data + offset, m_data + m_committed);
553  }
554 
564  iterator get_iterator(size_t offset) {
565  assert(m_data);
566  return iterator(m_data + offset, m_data + m_committed);
567  }
568 
577  template <typename T>
579  assert(m_data);
580  return t_iterator<T>(m_data + m_committed, m_data + m_committed);
581  }
582 
592  assert(m_data);
593  return iterator(m_data + m_committed, m_data + m_committed);
594  }
595 
596  template <typename T>
598  assert(m_data);
599  return t_const_iterator<T>(m_data, m_data + m_committed);
600  }
601 
603  assert(m_data);
604  return const_iterator(m_data, m_data + m_committed);
605  }
606 
607  template <typename T>
608  t_const_iterator<T> get_iterator(size_t offset) const {
609  assert(m_data);
610  return t_const_iterator<T>(m_data + offset, m_data + m_committed);
611  }
612 
613  const_iterator get_iterator(size_t offset) const {
614  assert(m_data);
615  return const_iterator(m_data + offset, m_data + m_committed);
616  }
617 
618  template <typename T>
620  assert(m_data);
621  return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);
622  }
623 
625  assert(m_data);
626  return const_iterator(m_data + m_committed, m_data + m_committed);
627  }
628 
629  template <typename T>
631  return cbegin<T>();
632  }
633 
635  return cbegin();
636  }
637 
638  template <typename T>
640  return cend<T>();
641  }
642 
643  const_iterator end() const {
644  return cend();
645  }
646 
650  explicit operator bool() const {
651  return m_data != nullptr;
652  }
653 
654  friend void swap(Buffer& lhs, Buffer& rhs) {
655  using std::swap;
656 
657  swap(lhs.m_memory, rhs.m_memory);
658  swap(lhs.m_data, rhs.m_data);
659  swap(lhs.m_capacity, rhs.m_capacity);
660  swap(lhs.m_written, rhs.m_written);
661  swap(lhs.m_committed, rhs.m_committed);
662  swap(lhs.m_auto_grow, rhs.m_auto_grow);
663  swap(lhs.m_full, rhs.m_full);
664  }
665 
682  template <typename TCallbackClass>
683  void purge_removed(TCallbackClass* callback) {
684  assert(m_data);
685  if (begin() == end()) {
686  return;
687  }
688 
689  iterator it_write = begin();
690 
691  iterator next;
692  for (iterator it_read = begin(); it_read != end(); it_read = next) {
693  next = std::next(it_read);
694  if (!it_read->removed()) {
695  if (it_read != it_write) {
696  assert(it_read.data() >= data());
697  assert(it_write.data() >= data());
698  size_t old_offset = static_cast<size_t>(it_read.data() - data());
699  size_t new_offset = static_cast<size_t>(it_write.data() - data());
700  callback->moving_in_buffer(old_offset, new_offset);
701  std::memmove(it_write.data(), it_read.data(), it_read->padded_size());
702  }
703  it_write.advance_once();
704  }
705  }
706 
707  assert(it_write.data() >= data());
708  m_written = static_cast<size_t>(it_write.data() - data());
709  m_committed = m_written;
710  }
711 
712  }; // class Buffer
713 
721  inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
722  if (!lhs || !rhs) {
723  return !lhs && !rhs;
724  }
725  return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
726  }
727 
728  inline bool operator!=(const Buffer& lhs, const Buffer& rhs) noexcept {
729  return ! (lhs == rhs);
730  }
731 
732  } // namespace memory
733 
734 } // namespace osmium
735 
736 #endif // OSMIUM_MEMORY_BUFFER_HPP
size_t m_written
Definition: buffer.hpp:111
t_const_iterator< T > cend() const
Definition: buffer.hpp:619
const_iterator begin() const
Definition: buffer.hpp:634
size_t clear()
Definition: buffer.hpp:354
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:53
OSMIUM_DEPRECATED void set_full_callback(std::function< void(Buffer &)> full)
Definition: buffer.hpp:282
iterator get_iterator(size_t offset)
Definition: buffer.hpp:564
bool operator!=(const Buffer &lhs, const Buffer &rhs) noexcept
Definition: buffer.hpp:728
void grow(size_t size)
Definition: buffer.hpp:303
const_iterator end() const
Definition: buffer.hpp:643
Definition: item_iterator.hpp:119
unsigned char * m_data
Definition: buffer.hpp:109
constexpr item_size_type align_bytes
Definition: item.hpp:53
size_t capacity() const noexcept
Definition: buffer.hpp:230
Buffer(unsigned char *data, size_t capacity, size_t committed)
Definition: buffer.hpp:166
Definition: reader_iterator.hpp:39
ItemIterator< TMember > & advance_once()
Definition: item_iterator.hpp:168
Buffer(size_t capacity, auto_grow auto_grow=auto_grow::yes)
Definition: buffer.hpp:194
unsigned char * data() const
Definition: item_iterator.hpp:189
bool operator==(const Buffer &lhs, const Buffer &rhs) noexcept
Definition: buffer.hpp:721
t_iterator< T > end()
Definition: buffer.hpp:578
Definition: item.hpp:97
Namespace for everything in the Osmium library.
Definition: assembler.hpp:59
T & add_item(const T &item)
Definition: buffer.hpp:450
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:683
t_iterator< T > begin()
Definition: buffer.hpp:522
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:467
friend void swap(Buffer &lhs, Buffer &rhs)
Definition: buffer.hpp:654
const_iterator cbegin() const
Definition: buffer.hpp:602
size_t m_committed
Definition: buffer.hpp:112
bool is_aligned() const noexcept
Definition: buffer.hpp:257
t_iterator< osmium::OSMEntity > iterator
Definition: buffer.hpp:505
Buffer() noexcept
Definition: buffer.hpp:126
unsigned char * reserve_space(const size_t size)
Definition: buffer.hpp:410
auto_grow m_auto_grow
Definition: buffer.hpp:113
void push_back(const osmium::memory::Item &item)
Definition: buffer.hpp:481
iterator end()
Definition: buffer.hpp:591
size_t committed() const noexcept
Definition: buffer.hpp:238
iterator begin()
Definition: buffer.hpp:535
const_iterator cend() const
Definition: buffer.hpp:624
size_t m_capacity
Definition: buffer.hpp:110
t_const_iterator< T > get_iterator(size_t offset) const
Definition: buffer.hpp:608
size_t written() const noexcept
Definition: buffer.hpp:247
Buffer(unsigned char *data, size_t size)
Definition: buffer.hpp:144
t_const_iterator< T > end() const
Definition: buffer.hpp:639
Definition: buffer.hpp:97
unsigned char * data() const noexcept
Definition: buffer.hpp:221
Definition: buffer.hpp:58
const_iterator get_iterator(size_t offset) const
Definition: buffer.hpp:613
t_const_iterator< T > cbegin() const
Definition: buffer.hpp:597
t_const_iterator< T > begin() const
Definition: buffer.hpp:630
auto_grow
Definition: buffer.hpp:101
std::vector< unsigned char > m_memory
Definition: buffer.hpp:108
buffer_is_full()
Definition: buffer.hpp:60
void rollback()
Definition: buffer.hpp:342
t_iterator< T > get_iterator(size_t offset)
Definition: buffer.hpp:550
std::function< void(Buffer &)> m_full
Definition: buffer.hpp:114
Buffer & operator=(const Buffer &)=delete
size_t commit()
Definition: buffer.hpp:328
t_const_iterator< osmium::OSMEntity > const_iterator
Definition: buffer.hpp:511