Libosmium  2.5.4
Fast and flexible C++ library for working with OpenStreetMap data
buffer.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_MEMORY_BUFFER_HPP
2 #define OSMIUM_MEMORY_BUFFER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstring>
40 #include <functional>
41 #include <iterator>
42 #include <stdexcept>
43 #include <utility>
44 #include <vector>
45 
46 #include <osmium/memory/item.hpp>
48 #include <osmium/osm/entity.hpp>
50 
51 namespace osmium {
52 
58  struct buffer_is_full : public std::runtime_error {
59 
61  std::runtime_error("Osmium buffer is full") {
62  }
63 
64  }; // struct buffer_is_full
65 
69  namespace memory {
70 
97  class Buffer {
98 
99  public:
100 
101  // This is needed so we can call std::back_inserter() on a Buffer.
102  using value_type = Item;
103 
104  enum class auto_grow : bool {
105  yes = true,
106  no = false
107  }; // enum class auto_grow
108 
109  private:
110 
111  std::vector<unsigned char> m_memory;
112  unsigned char* m_data;
113  size_t m_capacity;
114  size_t m_written;
115  size_t m_committed;
117  std::function<void(Buffer&)> m_full;
118 
119  public:
120 
129  Buffer() noexcept :
130  m_memory(),
131  m_data(nullptr),
132  m_capacity(0),
133  m_written(0),
134  m_committed(0) {
135  }
136 
147  explicit Buffer(unsigned char* data, size_t size) :
148  m_memory(),
149  m_data(data),
150  m_capacity(size),
151  m_written(size),
152  m_committed(size) {
153  if (size % align_bytes != 0) {
154  throw std::invalid_argument("buffer size needs to be multiple of alignment");
155  }
156  }
157 
169  explicit Buffer(unsigned char* data, size_t capacity, size_t committed) :
170  m_memory(),
171  m_data(data),
172  m_capacity(capacity),
173  m_written(committed),
174  m_committed(committed) {
175  if (capacity % align_bytes != 0) {
176  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
177  }
178  if (committed % align_bytes != 0) {
179  throw std::invalid_argument("buffer parameter 'committed' needs to be multiple of alignment");
180  }
181  }
182 
198  m_memory(capacity),
199  m_data(m_memory.data()),
200  m_capacity(capacity),
201  m_written(0),
202  m_committed(0),
204  if (capacity % align_bytes != 0) {
205  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
206  }
207  }
208 
209  // buffers can not be copied
210  Buffer(const Buffer&) = delete;
211  Buffer& operator=(const Buffer&) = delete;
212 
213  // buffers can be moved
214  Buffer(Buffer&&) = default;
215  Buffer& operator=(Buffer&&) = default;
216 
217  ~Buffer() = default;
218 
224  unsigned char* data() const noexcept {
225  assert(m_data);
226  return m_data;
227  }
228 
233  size_t capacity() const noexcept {
234  return m_capacity;
235  }
236 
241  size_t committed() const noexcept {
242  return m_committed;
243  }
244 
250  size_t written() const noexcept {
251  return m_written;
252  }
253 
260  bool is_aligned() const noexcept {
261  assert(m_data);
262  return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
263  }
264 
285  OSMIUM_DEPRECATED void set_full_callback(std::function<void(Buffer&)> full) {
286  assert(m_data);
287  m_full = full;
288  }
289 
306  void grow(size_t size) {
307  assert(m_data);
308  if (m_memory.empty()) {
309  throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management.");
310  }
311  if (m_capacity < size) {
312  if (size % align_bytes != 0) {
313  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
314  }
315  m_memory.resize(size);
316  m_data = m_memory.data();
317  m_capacity = size;
318  }
319  }
320 
331  size_t commit() {
332  assert(m_data);
333  assert(is_aligned());
334 
335  const size_t offset = m_committed;
336  m_committed = m_written;
337  return offset;
338  }
339 
345  void rollback() {
346  assert(m_data);
347  m_written = m_committed;
348  }
349 
357  size_t clear() {
358  const size_t committed = m_committed;
359  m_written = 0;
360  m_committed = 0;
361  return committed;
362  }
363 
374  template <typename T>
375  T& get(const size_t offset) const {
376  assert(m_data);
377  return *reinterpret_cast<T*>(&m_data[offset]);
378  }
379 
413  unsigned char* reserve_space(const size_t size) {
414  assert(m_data);
415  // try to flush the buffer empty first.
416  if (m_written + size > m_capacity && m_full) {
417  m_full(*this);
418  }
419  // if there's still not enough space, then try growing the buffer.
420  if (m_written + size > m_capacity) {
421  if (!m_memory.empty() && (m_auto_grow == auto_grow::yes)) {
422  // double buffer size until there is enough space
423  size_t new_capacity = m_capacity * 2;
424  while (m_written + size > new_capacity) {
425  new_capacity *= 2;
426  }
427  grow(new_capacity);
428  } else {
429  throw osmium::buffer_is_full();
430  }
431  }
432  unsigned char* data = &m_data[m_written];
433  m_written += size;
434  return data;
435  }
436 
452  template <typename T>
453  T& add_item(const T& item) {
454  assert(m_data);
455  unsigned char* target = reserve_space(item.padded_size());
456  std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
457  return *reinterpret_cast<T*>(target);
458  }
459 
470  void add_buffer(const Buffer& buffer) {
471  assert(m_data && buffer);
472  unsigned char* target = reserve_space(buffer.committed());
473  std::copy_n(buffer.data(), buffer.committed(), target);
474  }
475 
484  void push_back(const osmium::memory::Item& item) {
485  assert(m_data);
486  add_item(item);
487  commit();
488  }
489 
494  template <typename T>
496 
501  template <typename T>
503 
509 
515 
524  template <typename T>
526  assert(m_data);
527  return t_iterator<T>(m_data, m_data + m_committed);
528  }
529 
539  assert(m_data);
540  return iterator(m_data, m_data + m_committed);
541  }
542 
552  template <typename T>
553  t_iterator<T> get_iterator(size_t offset) {
554  assert(m_data);
555  return t_iterator<T>(m_data + offset, m_data + m_committed);
556  }
557 
567  iterator get_iterator(size_t offset) {
568  assert(m_data);
569  return iterator(m_data + offset, m_data + m_committed);
570  }
571 
580  template <typename T>
582  assert(m_data);
583  return t_iterator<T>(m_data + m_committed, m_data + m_committed);
584  }
585 
595  assert(m_data);
596  return iterator(m_data + m_committed, m_data + m_committed);
597  }
598 
599  template <typename T>
601  assert(m_data);
602  return t_const_iterator<T>(m_data, m_data + m_committed);
603  }
604 
606  assert(m_data);
607  return const_iterator(m_data, m_data + m_committed);
608  }
609 
610  template <typename T>
611  t_const_iterator<T> get_iterator(size_t offset) const {
612  assert(m_data);
613  return t_const_iterator<T>(m_data + offset, m_data + m_committed);
614  }
615 
616  const_iterator get_iterator(size_t offset) const {
617  assert(m_data);
618  return const_iterator(m_data + offset, m_data + m_committed);
619  }
620 
621  template <typename T>
623  assert(m_data);
624  return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);
625  }
626 
628  assert(m_data);
629  return const_iterator(m_data + m_committed, m_data + m_committed);
630  }
631 
632  template <typename T>
634  return cbegin<T>();
635  }
636 
638  return cbegin();
639  }
640 
641  template <typename T>
643  return cend<T>();
644  }
645 
646  const_iterator end() const {
647  return cend();
648  }
649 
653  explicit operator bool() const noexcept {
654  return m_data != nullptr;
655  }
656 
657  friend void swap(Buffer& lhs, Buffer& rhs) {
658  using std::swap;
659 
660  swap(lhs.m_memory, rhs.m_memory);
661  swap(lhs.m_data, rhs.m_data);
662  swap(lhs.m_capacity, rhs.m_capacity);
663  swap(lhs.m_written, rhs.m_written);
664  swap(lhs.m_committed, rhs.m_committed);
665  swap(lhs.m_auto_grow, rhs.m_auto_grow);
666  swap(lhs.m_full, rhs.m_full);
667  }
668 
685  template <typename TCallbackClass>
686  void purge_removed(TCallbackClass* callback) {
687  assert(m_data);
688  if (begin() == end()) {
689  return;
690  }
691 
692  iterator it_write = begin();
693 
694  iterator next;
695  for (iterator it_read = begin(); it_read != end(); it_read = next) {
696  next = std::next(it_read);
697  if (!it_read->removed()) {
698  if (it_read != it_write) {
699  assert(it_read.data() >= data());
700  assert(it_write.data() >= data());
701  size_t old_offset = static_cast<size_t>(it_read.data() - data());
702  size_t new_offset = static_cast<size_t>(it_write.data() - data());
703  callback->moving_in_buffer(old_offset, new_offset);
704  std::memmove(it_write.data(), it_read.data(), it_read->padded_size());
705  }
706  it_write.advance_once();
707  }
708  }
709 
710  assert(it_write.data() >= data());
711  m_written = static_cast<size_t>(it_write.data() - data());
712  m_committed = m_written;
713  }
714 
715  }; // class Buffer
716 
724  inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
725  if (!lhs || !rhs) {
726  return !lhs && !rhs;
727  }
728  return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
729  }
730 
731  inline bool operator!=(const Buffer& lhs, const Buffer& rhs) noexcept {
732  return ! (lhs == rhs);
733  }
734 
735  } // namespace memory
736 
737 } // namespace osmium
738 
739 #endif // OSMIUM_MEMORY_BUFFER_HPP
size_t m_written
Definition: buffer.hpp:114
t_const_iterator< T > cend() const
Definition: buffer.hpp:622
const_iterator begin() const
Definition: buffer.hpp:637
size_t clear()
Definition: buffer.hpp:357
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
OSMIUM_DEPRECATED void set_full_callback(std::function< void(Buffer &)> full)
Definition: buffer.hpp:285
iterator get_iterator(size_t offset)
Definition: buffer.hpp:567
bool operator!=(const Buffer &lhs, const Buffer &rhs) noexcept
Definition: buffer.hpp:731
void grow(size_t size)
Definition: buffer.hpp:306
const_iterator end() const
Definition: buffer.hpp:646
Definition: item_iterator.hpp:119
unsigned char * m_data
Definition: buffer.hpp:112
constexpr item_size_type align_bytes
Definition: item.hpp:53
size_t capacity() const noexcept
Definition: buffer.hpp:233
Buffer(unsigned char *data, size_t capacity, size_t committed)
Definition: buffer.hpp:169
Definition: reader_iterator.hpp:39
ItemIterator< TMember > & advance_once()
Definition: item_iterator.hpp:168
Buffer(size_t capacity, auto_grow auto_grow=auto_grow::yes)
Definition: buffer.hpp:197
unsigned char * data() const
Definition: item_iterator.hpp:189
bool operator==(const Buffer &lhs, const Buffer &rhs) noexcept
Definition: buffer.hpp:724
t_iterator< T > end()
Definition: buffer.hpp:581
Definition: item.hpp:97
Namespace for everything in the Osmium library.
Definition: assembler.hpp:59
T & add_item(const T &item)
Definition: buffer.hpp:453
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:686
t_iterator< T > begin()
Definition: buffer.hpp:525
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:470
friend void swap(Buffer &lhs, Buffer &rhs)
Definition: buffer.hpp:657
const_iterator cbegin() const
Definition: buffer.hpp:605
size_t m_committed
Definition: buffer.hpp:115
bool is_aligned() const noexcept
Definition: buffer.hpp:260
t_iterator< osmium::OSMEntity > iterator
Definition: buffer.hpp:508
Buffer() noexcept
Definition: buffer.hpp:129
unsigned char * reserve_space(const size_t size)
Definition: buffer.hpp:413
auto_grow m_auto_grow
Definition: buffer.hpp:116
void push_back(const osmium::memory::Item &item)
Definition: buffer.hpp:484
iterator end()
Definition: buffer.hpp:594
size_t committed() const noexcept
Definition: buffer.hpp:241
iterator begin()
Definition: buffer.hpp:538
const_iterator cend() const
Definition: buffer.hpp:627
size_t m_capacity
Definition: buffer.hpp:113
t_const_iterator< T > get_iterator(size_t offset) const
Definition: buffer.hpp:611
size_t written() const noexcept
Definition: buffer.hpp:250
Buffer(unsigned char *data, size_t size)
Definition: buffer.hpp:147
t_const_iterator< T > end() const
Definition: buffer.hpp:642
Definition: buffer.hpp:97
unsigned char * data() const noexcept
Definition: buffer.hpp:224
Definition: buffer.hpp:58
const_iterator get_iterator(size_t offset) const
Definition: buffer.hpp:616
t_const_iterator< T > cbegin() const
Definition: buffer.hpp:600
t_const_iterator< T > begin() const
Definition: buffer.hpp:633
auto_grow
Definition: buffer.hpp:104
std::vector< unsigned char > m_memory
Definition: buffer.hpp:111
buffer_is_full()
Definition: buffer.hpp:60
void rollback()
Definition: buffer.hpp:345
t_iterator< T > get_iterator(size_t offset)
Definition: buffer.hpp:553
std::function< void(Buffer &)> m_full
Definition: buffer.hpp:117
Buffer & operator=(const Buffer &)=delete
size_t commit()
Definition: buffer.hpp:331
t_const_iterator< osmium::OSMEntity > const_iterator
Definition: buffer.hpp:514