Libosmium  2.5.3
Fast and flexible C++ library for working with OpenStreetMap data
bzip2_compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_BZIP2_COMPRESSION_HPP
2 #define OSMIUM_IO_BZIP2_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
45 #include <cerrno>
46 #include <cstddef>
47 #include <cstdio>
48 #include <stdexcept>
49 #include <string>
50 
51 #include <bzlib.h>
52 
53 #ifndef _MSC_VER
54 # include <unistd.h>
55 #endif
56 
58 #include <osmium/io/error.hpp>
61 #include <osmium/util/cast.hpp>
63 
64 namespace osmium {
65 
70  struct bzip2_error : public io_error {
71 
74 
75  bzip2_error(const std::string& what, int error_code) :
76  io_error(what),
77  bzip2_error_code(error_code),
78  system_errno(error_code == BZ_IO_ERROR ? errno : 0) {
79  }
80 
81  }; // struct bzip2_error
82 
83  namespace io {
84 
85  namespace detail {
86 
87  OSMIUM_NORETURN inline void throw_bzip2_error(BZFILE* bzfile, const char* msg, int bzlib_error = 0) {
88  std::string error("bzip2 error: ");
89  error += msg;
90  error += ": ";
91  int errnum = bzlib_error;
92  if (bzlib_error) {
93  error += std::to_string(bzlib_error);
94  } else {
95  error += ::BZ2_bzerror(bzfile, &errnum);
96  }
97  throw osmium::bzip2_error(error, errnum);
98  }
99 
100  } // namespace detail
101 
102  class Bzip2Compressor : public Compressor {
103 
104  FILE* m_file;
106  BZFILE* m_bzfile;
107 
108  public:
109 
110  explicit Bzip2Compressor(int fd, fsync sync) :
111  Compressor(sync),
112  m_file(fdopen(dup(fd), "wb")),
113  m_bzerror(BZ_OK),
114  m_bzfile(::BZ2_bzWriteOpen(&m_bzerror, m_file, 6, 0, 0)) {
115  if (!m_bzfile) {
116  detail::throw_bzip2_error(m_bzfile, "write open failed", m_bzerror);
117  }
118  }
119 
120  ~Bzip2Compressor() noexcept override final {
121  try {
122  close();
123  } catch (...) {
124  // Ignore any exceptions because destructor must not throw.
125  }
126  }
127 
128  void write(const std::string& data) override final {
129  int error;
130  ::BZ2_bzWrite(&error, m_bzfile, const_cast<char*>(data.data()), static_cast_with_assert<int>(data.size()));
131  if (error != BZ_OK && error != BZ_STREAM_END) {
132  detail::throw_bzip2_error(m_bzfile, "write failed", error);
133  }
134  }
135 
136  void close() override final {
137  if (m_bzfile) {
138  int error;
139  ::BZ2_bzWriteClose(&error, m_bzfile, 0, nullptr, nullptr);
140  m_bzfile = nullptr;
141  if (m_file) {
142  if (do_fsync()) {
143  osmium::io::detail::reliable_fsync(::fileno(m_file));
144  }
145  if (fclose(m_file) != 0) {
146  throw std::system_error(errno, std::system_category(), "Close failed");
147  }
148  }
149  if (error != BZ_OK) {
150  detail::throw_bzip2_error(m_bzfile, "write close failed", error);
151  }
152  }
153  }
154 
155  }; // class Bzip2Compressor
156 
158 
159  FILE* m_file;
161  BZFILE* m_bzfile;
162  bool m_stream_end {false};
163 
164  public:
165 
167  Decompressor(),
168  m_file(fdopen(dup(fd), "rb")),
169  m_bzerror(BZ_OK),
170  m_bzfile(::BZ2_bzReadOpen(&m_bzerror, m_file, 0, 0, nullptr, 0)) {
171  if (!m_bzfile) {
172  detail::throw_bzip2_error(m_bzfile, "read open failed", m_bzerror);
173  }
174  }
175 
176  ~Bzip2Decompressor() noexcept override final {
177  try {
178  close();
179  } catch (...) {
180  // Ignore any exceptions because destructor must not throw.
181  }
182  }
183 
184  std::string read() override final {
185  std::string buffer;
186 
187  if (!m_stream_end) {
189  int error;
190  int nread = ::BZ2_bzRead(&error, m_bzfile, const_cast<char*>(buffer.data()), static_cast_with_assert<int>(buffer.size()));
191  if (error != BZ_OK && error != BZ_STREAM_END) {
192  detail::throw_bzip2_error(m_bzfile, "read failed", error);
193  }
194  if (error == BZ_STREAM_END) {
195  void* unused;
196  int nunused;
197  if (! feof(m_file)) {
198  ::BZ2_bzReadGetUnused(&error, m_bzfile, &unused, &nunused);
199  if (error != BZ_OK) {
200  detail::throw_bzip2_error(m_bzfile, "get unused failed", error);
201  }
202  std::string unused_data(static_cast<const char*>(unused), static_cast<std::string::size_type>(nunused));
203  ::BZ2_bzReadClose(&error, m_bzfile);
204  if (error != BZ_OK) {
205  detail::throw_bzip2_error(m_bzfile, "read close failed", error);
206  }
207  m_bzfile = ::BZ2_bzReadOpen(&error, m_file, 0, 0, const_cast<void*>(static_cast<const void*>(unused_data.data())), static_cast_with_assert<int>(unused_data.size()));
208  if (error != BZ_OK) {
209  detail::throw_bzip2_error(m_bzfile, "read open failed", error);
210  }
211  } else {
212  m_stream_end = true;
213  }
214  }
215  buffer.resize(static_cast<std::string::size_type>(nread));
216  }
217 
218  return buffer;
219  }
220 
221  void close() override final {
222  if (m_bzfile) {
223  int error;
224  ::BZ2_bzReadClose(&error, m_bzfile);
225  m_bzfile = nullptr;
226  if (m_file) {
227  if (fclose(m_file) != 0) {
228  throw std::system_error(errno, std::system_category(), "Close failed");
229  }
230  }
231  if (error != BZ_OK) {
232  detail::throw_bzip2_error(m_bzfile, "read close failed", error);
233  }
234  }
235  }
236 
237  }; // class Bzip2Decompressor
238 
240 
241  const char* m_buffer;
243  bz_stream m_bzstream;
244 
245  public:
246 
247  Bzip2BufferDecompressor(const char* buffer, size_t size) :
248  m_buffer(buffer),
249  m_buffer_size(size),
250  m_bzstream() {
251  m_bzstream.next_in = const_cast<char*>(buffer);
252  m_bzstream.avail_in = static_cast_with_assert<unsigned int>(size);
253  int result = BZ2_bzDecompressInit(&m_bzstream, 0, 0);
254  if (result != BZ_OK) {
255  std::string message("bzip2 error: decompression init failed: ");
256  throw bzip2_error(message, result);
257  }
258  }
259 
260  ~Bzip2BufferDecompressor() noexcept override final {
261  try {
262  close();
263  } catch (...) {
264  // Ignore any exceptions because destructor must not throw.
265  }
266  }
267 
268  std::string read() override final {
269  std::string output;
270 
271  if (m_buffer) {
272  const size_t buffer_size = 10240;
273  output.resize(buffer_size);
274  m_bzstream.next_out = const_cast<char*>(output.data());
275  m_bzstream.avail_out = buffer_size;
276  int result = BZ2_bzDecompress(&m_bzstream);
277 
278  if (result != BZ_OK) {
279  m_buffer = nullptr;
280  m_buffer_size = 0;
281  }
282 
283  if (result != BZ_OK && result != BZ_STREAM_END) {
284  std::string message("bzip2 error: decompress failed: ");
285  throw bzip2_error(message, result);
286  }
287 
288  output.resize(static_cast<unsigned long>(m_bzstream.next_out - output.data()));
289  }
290 
291  return output;
292  }
293 
294  void close() override final {
295  BZ2_bzDecompressEnd(&m_bzstream);
296  }
297 
298  }; // class Bzip2BufferDecompressor
299 
300  namespace detail {
301 
302  // we want the register_compression() function to run, setting
303  // the variable is only a side-effect, it will never be used
305  [](int fd, fsync sync) { return new osmium::io::Bzip2Compressor(fd, sync); },
306  [](int fd) { return new osmium::io::Bzip2Decompressor(fd); },
307  [](const char* buffer, size_t size) { return new osmium::io::Bzip2BufferDecompressor(buffer, size); }
308  );
309 
310  // dummy function to silence the unused variable warning from above
311  inline bool get_registered_bzip2_compression() noexcept {
312  return registered_bzip2_compression;
313  }
314 
315  } // namespace detail
316 
317  } // namespace io
318 
319 } // namespace osmium
320 
321 #endif // OSMIUM_IO_BZIP2_COMPRESSION_HPP
bz_stream m_bzstream
Definition: bzip2_compression.hpp:243
Definition: bzip2_compression.hpp:157
std::string read() override final
Definition: bzip2_compression.hpp:268
int bzip2_error_code
Definition: bzip2_compression.hpp:72
Bzip2Compressor(int fd, fsync sync)
Definition: bzip2_compression.hpp:110
#define OSMIUM_NORETURN
Definition: compatibility.hpp:44
int m_bzerror
Definition: bzip2_compression.hpp:160
const char * m_buffer
Definition: bzip2_compression.hpp:241
int m_bzerror
Definition: bzip2_compression.hpp:105
void close() override final
Definition: bzip2_compression.hpp:221
size_t m_buffer_size
Definition: bzip2_compression.hpp:242
bool m_stream_end
Definition: bzip2_compression.hpp:162
~Bzip2BufferDecompressor() noexcept override final
Definition: bzip2_compression.hpp:260
static CompressionFactory & instance()
Definition: compression.hpp:151
void write(const std::string &data) override final
Definition: bzip2_compression.hpp:128
static constexpr unsigned int input_buffer_size
Definition: compression.hpp:91
bzip2_error(const std::string &what, int error_code)
Definition: bzip2_compression.hpp:75
Definition: bzip2_compression.hpp:102
int system_errno
Definition: bzip2_compression.hpp:73
Definition: compression.hpp:87
FILE * m_file
Definition: bzip2_compression.hpp:104
void close() override final
Definition: bzip2_compression.hpp:136
Definition: bzip2_compression.hpp:239
Namespace for everything in the Osmium library.
Definition: assembler.hpp:59
Bzip2BufferDecompressor(const char *buffer, size_t size)
Definition: bzip2_compression.hpp:247
fsync
Definition: writer_options.hpp:51
std::string read() override final
Definition: bzip2_compression.hpp:184
BZFILE * m_bzfile
Definition: bzip2_compression.hpp:161
BZFILE * m_bzfile
Definition: bzip2_compression.hpp:106
~Bzip2Decompressor() noexcept override final
Definition: bzip2_compression.hpp:176
Definition: error.hpp:44
bool do_fsync() const
Definition: compression.hpp:68
Definition: compression.hpp:62
FILE * m_file
Definition: bzip2_compression.hpp:159
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:156
void close() override final
Definition: bzip2_compression.hpp:294
~Bzip2Compressor() noexcept override final
Definition: bzip2_compression.hpp:120
Bzip2Decompressor(int fd)
Definition: bzip2_compression.hpp:166
Definition: bzip2_compression.hpp:70