Libosmium  2.13.1
Fast and flexible C++ library for working with OpenStreetMap data
bzip2_compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_BZIP2_COMPRESSION_HPP
2 #define OSMIUM_IO_BZIP2_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2017 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
45 #include <cerrno>
46 #include <cstdio>
47 #include <string>
48 #include <system_error>
49 
50 #include <bzlib.h>
51 
52 #ifndef _MSC_VER
53 # include <unistd.h>
54 #endif
55 
57 #include <osmium/io/detail/read_write.hpp>
58 #include <osmium/io/error.hpp>
61 #include <osmium/util/cast.hpp>
63 
64 namespace osmium {
65 
70  struct bzip2_error : public io_error {
71 
74 
75  bzip2_error(const std::string& what, int error_code) :
76  io_error(what),
77  bzip2_error_code(error_code),
78  system_errno(error_code == BZ_IO_ERROR ? errno : 0) {
79  }
80 
81  }; // struct bzip2_error
82 
83  namespace io {
84 
85  namespace detail {
86 
87  OSMIUM_NORETURN inline void throw_bzip2_error(BZFILE* bzfile, const char* msg, int bzlib_error = 0) {
88  std::string error{"bzip2 error: "};
89  error += msg;
90  error += ": ";
91  int errnum = bzlib_error;
92  if (bzlib_error) {
93  error += std::to_string(bzlib_error);
94  } else {
95  error += ::BZ2_bzerror(bzfile, &errnum);
96  }
97  throw osmium::bzip2_error{error, errnum};
98  }
99 
100  } // namespace detail
101 
102  class Bzip2Compressor : public Compressor {
103 
104  FILE* m_file;
106  BZFILE* m_bzfile;
107 
108  public:
109 
110  explicit Bzip2Compressor(int fd, fsync sync) :
111  Compressor(sync),
112  m_file(fdopen(::dup(fd), "wb")),
113  m_bzerror(BZ_OK),
114  m_bzfile(::BZ2_bzWriteOpen(&m_bzerror, m_file, 6, 0, 0)) {
115  if (!m_bzfile) {
116  detail::throw_bzip2_error(m_bzfile, "write open failed", m_bzerror);
117  }
118  }
119 
120  ~Bzip2Compressor() noexcept final {
121  try {
122  close();
123  } catch (...) {
124  // Ignore any exceptions because destructor must not throw.
125  }
126  }
127 
128  void write(const std::string& data) final {
129  int error;
130  ::BZ2_bzWrite(&error, m_bzfile, const_cast<char*>(data.data()), static_cast_with_assert<int>(data.size()));
131  if (error != BZ_OK && error != BZ_STREAM_END) {
132  detail::throw_bzip2_error(m_bzfile, "write failed", error);
133  }
134  }
135 
136  void close() final {
137  if (m_bzfile) {
138  int error;
139  ::BZ2_bzWriteClose(&error, m_bzfile, 0, nullptr, nullptr);
140  m_bzfile = nullptr;
141  if (m_file) {
142  if (do_fsync()) {
143  osmium::io::detail::reliable_fsync(::fileno(m_file));
144  }
145  if (fclose(m_file) != 0) {
146  throw std::system_error{errno, std::system_category(), "Close failed"};
147  }
148  }
149  if (error != BZ_OK) {
150  detail::throw_bzip2_error(m_bzfile, "write close failed", error);
151  }
152  }
153  }
154 
155  }; // class Bzip2Compressor
156 
158 
159  FILE* m_file;
161  BZFILE* m_bzfile;
162  bool m_stream_end {false};
163 
164  public:
165 
166  explicit Bzip2Decompressor(int fd) :
167  Decompressor(),
168  m_file(fdopen(::dup(fd), "rb")),
169  m_bzerror(BZ_OK),
170  m_bzfile(::BZ2_bzReadOpen(&m_bzerror, m_file, 0, 0, nullptr, 0)) {
171  if (!m_bzfile) {
172  detail::throw_bzip2_error(m_bzfile, "read open failed", m_bzerror);
173  }
174  }
175 
176  ~Bzip2Decompressor() noexcept final {
177  try {
178  close();
179  } catch (...) {
180  // Ignore any exceptions because destructor must not throw.
181  }
182  }
183 
184  std::string read() final {
185  std::string buffer;
186 
187  if (!m_stream_end) {
189  int error;
190  const int nread = ::BZ2_bzRead(&error, m_bzfile, const_cast<char*>(buffer.data()), static_cast_with_assert<int>(buffer.size()));
191  if (error != BZ_OK && error != BZ_STREAM_END) {
192  detail::throw_bzip2_error(m_bzfile, "read failed", error);
193  }
194  if (error == BZ_STREAM_END) {
195  void* unused;
196  int nunused;
197  if (! feof(m_file)) {
198  ::BZ2_bzReadGetUnused(&error, m_bzfile, &unused, &nunused);
199  if (error != BZ_OK) {
200  detail::throw_bzip2_error(m_bzfile, "get unused failed", error);
201  }
202  std::string unused_data(static_cast<const char*>(unused), static_cast<std::string::size_type>(nunused));
203  ::BZ2_bzReadClose(&error, m_bzfile);
204  if (error != BZ_OK) {
205  detail::throw_bzip2_error(m_bzfile, "read close failed", error);
206  }
207  m_bzfile = ::BZ2_bzReadOpen(&error, m_file, 0, 0, const_cast<void*>(static_cast<const void*>(unused_data.data())), static_cast_with_assert<int>(unused_data.size()));
208  if (error != BZ_OK) {
209  detail::throw_bzip2_error(m_bzfile, "read open failed", error);
210  }
211  } else {
212  m_stream_end = true;
213  }
214  }
215  buffer.resize(static_cast<std::string::size_type>(nread));
216  }
217 
218  set_offset(size_t(ftell(m_file)));
219 
220  return buffer;
221  }
222 
223  void close() final {
224  if (m_bzfile) {
225  int error;
226  ::BZ2_bzReadClose(&error, m_bzfile);
227  m_bzfile = nullptr;
228  if (m_file) {
229  if (fclose(m_file) != 0) {
230  throw std::system_error{errno, std::system_category(), "Close failed"};
231  }
232  }
233  if (error != BZ_OK) {
234  detail::throw_bzip2_error(m_bzfile, "read close failed", error);
235  }
236  }
237  }
238 
239  }; // class Bzip2Decompressor
240 
242 
243  const char* m_buffer;
245  bz_stream m_bzstream;
246 
247  public:
248 
249  Bzip2BufferDecompressor(const char* buffer, size_t size) :
250  m_buffer(buffer),
251  m_buffer_size(size),
252  m_bzstream() {
253  m_bzstream.next_in = const_cast<char*>(buffer);
254  m_bzstream.avail_in = static_cast_with_assert<unsigned int>(size);
255  const int result = BZ2_bzDecompressInit(&m_bzstream, 0, 0);
256  if (result != BZ_OK) {
257  std::string message{"bzip2 error: decompression init failed: "};
258  throw bzip2_error{message, result};
259  }
260  }
261 
262  ~Bzip2BufferDecompressor() noexcept final {
263  try {
264  close();
265  } catch (...) {
266  // Ignore any exceptions because destructor must not throw.
267  }
268  }
269 
270  std::string read() final {
271  std::string output;
272 
273  if (m_buffer) {
274  const size_t buffer_size = 10240;
275  output.resize(buffer_size);
276  m_bzstream.next_out = const_cast<char*>(output.data());
277  m_bzstream.avail_out = buffer_size;
278  const int result = BZ2_bzDecompress(&m_bzstream);
279 
280  if (result != BZ_OK) {
281  m_buffer = nullptr;
282  m_buffer_size = 0;
283  }
284 
285  if (result != BZ_OK && result != BZ_STREAM_END) {
286  std::string message{"bzip2 error: decompress failed: "};
287  throw bzip2_error{message, result};
288  }
289 
290  output.resize(static_cast<unsigned long>(m_bzstream.next_out - output.data()));
291  }
292 
293  return output;
294  }
295 
296  void close() final {
297  BZ2_bzDecompressEnd(&m_bzstream);
298  }
299 
300  }; // class Bzip2BufferDecompressor
301 
302  namespace detail {
303 
304  // we want the register_compression() function to run, setting
305  // the variable is only a side-effect, it will never be used
307  [](int fd, fsync sync) { return new osmium::io::Bzip2Compressor{fd, sync}; },
308  [](int fd) { return new osmium::io::Bzip2Decompressor{fd}; },
309  [](const char* buffer, size_t size) { return new osmium::io::Bzip2BufferDecompressor{buffer, size}; }
310  );
311 
312  // dummy function to silence the unused variable warning from above
313  inline bool get_registered_bzip2_compression() noexcept {
314  return registered_bzip2_compression;
315  }
316 
317  } // namespace detail
318 
319  } // namespace io
320 
321 } // namespace osmium
322 
323 #endif // OSMIUM_IO_BZIP2_COMPRESSION_HPP
bz_stream m_bzstream
Definition: bzip2_compression.hpp:245
Definition: bzip2_compression.hpp:157
int bzip2_error_code
Definition: bzip2_compression.hpp:72
Bzip2Compressor(int fd, fsync sync)
Definition: bzip2_compression.hpp:110
#define OSMIUM_NORETURN
Definition: compatibility.hpp:41
~Bzip2Decompressor() noexcept final
Definition: bzip2_compression.hpp:176
int m_bzerror
Definition: bzip2_compression.hpp:160
void close() final
Definition: bzip2_compression.hpp:296
void write(const std::string &data) final
Definition: bzip2_compression.hpp:128
const char * m_buffer
Definition: bzip2_compression.hpp:243
int m_bzerror
Definition: bzip2_compression.hpp:105
size_t m_buffer_size
Definition: bzip2_compression.hpp:244
static CompressionFactory & instance()
Definition: compression.hpp:176
~Bzip2BufferDecompressor() noexcept final
Definition: bzip2_compression.hpp:262
static constexpr unsigned int input_buffer_size
Definition: compression.hpp:94
bzip2_error(const std::string &what, int error_code)
Definition: bzip2_compression.hpp:75
std::string read() final
Definition: bzip2_compression.hpp:184
Definition: bzip2_compression.hpp:102
int system_errno
Definition: bzip2_compression.hpp:73
Definition: compression.hpp:87
FILE * m_file
Definition: bzip2_compression.hpp:104
Definition: bzip2_compression.hpp:241
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
Definition: attr.hpp:333
Bzip2BufferDecompressor(const char *buffer, size_t size)
Definition: bzip2_compression.hpp:249
fsync
Definition: writer_options.hpp:51
BZFILE * m_bzfile
Definition: bzip2_compression.hpp:161
BZFILE * m_bzfile
Definition: bzip2_compression.hpp:106
Definition: error.hpp:44
void close() final
Definition: bzip2_compression.hpp:136
void close() final
Definition: bzip2_compression.hpp:223
std::string read() final
Definition: bzip2_compression.hpp:270
Definition: compression.hpp:63
FILE * m_file
Definition: bzip2_compression.hpp:159
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:181
Bzip2Decompressor(int fd)
Definition: bzip2_compression.hpp:166
~Bzip2Compressor() noexcept final
Definition: bzip2_compression.hpp:120
Definition: bzip2_compression.hpp:70