Apache Mesos
gzip.hpp
Go to the documentation of this file.
1 // Licensed under the Apache License, Version 2.0 (the "License");
2 // you may not use this file except in compliance with the License.
3 // You may obtain a copy of the License at
4 //
5 // http://www.apache.org/licenses/LICENSE-2.0
6 //
7 // Unless required by applicable law or agreed to in writing, software
8 // distributed under the License is distributed on an "AS IS" BASIS,
9 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 // See the License for the specific language governing permissions and
11 // limitations under the License.
12 
13 #ifndef __STOUT_GZIP_HPP__
14 #define __STOUT_GZIP_HPP__
15 
16 #include <zlib.h>
17 
18 #include <string>
19 
20 #include <stout/abort.hpp>
21 #include <stout/error.hpp>
22 #include <stout/os/strerror.hpp>
23 #include <stout/stringify.hpp>
24 #include <stout/try.hpp>
25 
26 
27 // Compression utilities.
28 // TODO(bmahler): Provide streaming compression as well.
29 namespace gzip {
30 
31 namespace internal {
32 
33 // We use a 16KB buffer with zlib compression / decompression.
34 #define GZIP_BUFFER_SIZE 16384
35 
36 
37 class GzipError : public Error
38 {
39 public:
40  GzipError(const std::string& message, const z_stream_s& stream, int _code)
41  : Error(message + ": " + GzipError::strerror(stream, _code)), code(_code) {}
42 
43  GzipError(const std::string& message, int _code)
44  : Error(message + ": " + GzipError::strerror(_code)), code(_code) {}
45 
46  GzipError(const z_stream_s& stream, int _code)
47  : Error(GzipError::strerror(stream, _code)), code(_code) {}
48 
49  GzipError(int _code)
50  : Error(GzipError::strerror(_code)), code(_code) {}
51 
52  const int code;
53 
54 private:
55  static std::string strerror(int code)
56  {
57  // We do not attempt to interpret the error codes since
58  // their meaning depends on which zlib call was made.
59  switch (code) {
60  case Z_OK: return "Z_OK"; // Not an error.
61  case Z_STREAM_END: return "Z_STREAM_END"; // Not an error.
62  case Z_NEED_DICT: return "Z_NEED_DICT"; // Not an error.
63  case Z_ERRNO: return "Z_ERRNO: " + os::strerror(errno);
64  case Z_STREAM_ERROR: return "Z_STREAM_ERROR";
65  case Z_DATA_ERROR: return "Z_DATA_ERROR";
66  case Z_MEM_ERROR: return "Z_MEM_ERROR";
67  case Z_BUF_ERROR: return "Z_BUF_ERROR";
68  case Z_VERSION_ERROR: return "Z_VERSION_ERROR";
69  default: return "Unknown error " + stringify(code);
70  }
71  }
72 
73  static std::string strerror(const z_stream_s& stream, int code)
74  {
75  if (stream.msg == Z_NULL) {
76  return GzipError::strerror(code);
77  } else {
78  return GzipError::strerror(code) + ": " + stream.msg;
79  }
80  }
81 };
82 
83 } // namespace internal {
84 
85 
86 // Provides the ability to incrementally decompress
87 // a stream of compressed input data.
89 {
90 public:
92  : _finished(false)
93  {
94  stream.zalloc = Z_NULL;
95  stream.zfree = Z_NULL;
96  stream.opaque = Z_NULL;
97  stream.next_in = Z_NULL;
98  stream.avail_in = 0;
99 
100  int code = inflateInit2(
101  &stream,
102  MAX_WBITS + 16); // Zlib magic for gzip compression / decompression.
103 
104  if (code != Z_OK) {
105  Error error = internal::GzipError("Failed to inflateInit2", stream, code);
106  ABORT(error.message);
107  }
108  }
109 
110  Decompressor(const Decompressor&) = delete;
111  Decompressor& operator=(const Decompressor&) = delete;
112 
114  {
115  if (inflateEnd(&stream) != Z_OK) {
116  ABORT("Failed to inflateEnd");
117  }
118  }
119 
120  // Returns the next decompressed chunk of data,
121  // or an Error if decompression fails.
122  Try<std::string> decompress(const std::string& compressed)
123  {
124  stream.next_in =
125  const_cast<Bytef*>(reinterpret_cast<const Bytef*>(compressed.data()));
126  stream.avail_in = static_cast<uInt>(compressed.length());
127 
128  // Build up the decompressed result.
129  Bytef buffer[GZIP_BUFFER_SIZE];
130  std::string result;
131 
132  while (stream.avail_in > 0) {
133  stream.next_out = buffer;
134  stream.avail_out = GZIP_BUFFER_SIZE;
135 
136  int code = inflate(&stream, Z_SYNC_FLUSH);
137 
138  _finished = code == Z_STREAM_END;
139 
140  if (code != Z_OK && !_finished) {
141  return internal::GzipError("Failed to inflate", stream, code);
142  }
143 
144  if (_finished && stream.avail_in > 0) {
145  return Error("Stream finished with data unconsumed");
146  }
147 
148  // Consume output and reset the buffer.
149  result.append(
150  reinterpret_cast<char*>(buffer),
151  GZIP_BUFFER_SIZE - stream.avail_out);
152  stream.next_out = buffer;
153  stream.avail_out = GZIP_BUFFER_SIZE;
154  }
155 
156  return result;
157  }
158 
159  // Returns whether the decompression stream is finished.
160  // If set to false, more input is expected.
161  bool finished() const
162  {
163  return _finished;
164  }
165 
166 private:
167  z_stream_s stream;
168  bool _finished;
169 };
170 
171 
172 // Returns a gzip compressed version of the provided string.
173 // The compression level should be within the range [-1, 9].
174 // See zlib.h:
175 // #define Z_NO_COMPRESSION 0
176 // #define Z_BEST_SPEED 1
177 // #define Z_BEST_COMPRESSION 9
178 // #define Z_DEFAULT_COMPRESSION (-1)
180  const std::string& decompressed,
181  int level = Z_DEFAULT_COMPRESSION)
182 {
183  // Verify the level is within range.
184  if (!(level == Z_DEFAULT_COMPRESSION ||
185  (level >= Z_NO_COMPRESSION && level <= Z_BEST_COMPRESSION))) {
186  return Error("Invalid compression level: " + stringify(level));
187  }
188 
189  z_stream_s stream;
190  stream.next_in =
191  const_cast<Bytef*>(reinterpret_cast<const Bytef*>(decompressed.data()));
192  stream.avail_in = static_cast<uInt>(decompressed.length());
193  stream.zalloc = Z_NULL;
194  stream.zfree = Z_NULL;
195  stream.opaque = Z_NULL;
196 
197  int code = deflateInit2(
198  &stream,
199  level, // Compression level.
200  Z_DEFLATED, // Compression method.
201  MAX_WBITS + 16, // Zlib magic for gzip compression / decompression.
202  8, // Default memLevel value.
203  Z_DEFAULT_STRATEGY);
204 
205  if (code != Z_OK) {
206  Error error = internal::GzipError("Failed to deflateInit2", stream, code);
207  ABORT(error.message);
208  }
209 
210  // Build up the compressed result.
211  Bytef buffer[GZIP_BUFFER_SIZE];
212  std::string result;
213  do {
214  stream.next_out = buffer;
215  stream.avail_out = GZIP_BUFFER_SIZE;
216  code = deflate(&stream, stream.avail_in > 0 ? Z_NO_FLUSH : Z_FINISH);
217 
218  if (code != Z_OK && code != Z_STREAM_END) {
219  Error error = internal::GzipError("Failed to deflate", stream, code);
220  if (deflateEnd(&stream) != Z_OK) {
221  ABORT("Failed to deflateEnd");
222  }
223  return error;
224  }
225 
226  // Consume output and reset the buffer.
227  result.append(
228  reinterpret_cast<char*>(buffer),
229  GZIP_BUFFER_SIZE - stream.avail_out);
230  stream.next_out = buffer;
231  stream.avail_out = GZIP_BUFFER_SIZE;
232  } while (code != Z_STREAM_END);
233 
234  if (deflateEnd(&stream) != Z_OK) {
235  ABORT("Failed to deflateEnd");
236  }
237 
238  return result;
239 }
240 
241 
242 // Returns a gzip decompressed version of the provided string.
243 inline Try<std::string> decompress(const std::string& compressed)
244 {
245  Decompressor decompressor;
246  Try<std::string> decompressed = decompressor.decompress(compressed);
247 
248  // Ensure that the decompression stream does not expect more input.
249  if (decompressed.isSome() && !decompressor.finished()) {
250  return Error("More input is expected");
251  }
252 
253  return decompressed;
254 }
255 
256 } // namespace gzip {
257 
258 #endif // __STOUT_GZIP_HPP__
GzipError(const std::string &message, const z_stream_s &stream, int _code)
Definition: gzip.hpp:40
std::string strerror(int errno_)
A thread-safe version of strerror.
Definition: strerror.hpp:30
Definition: errorbase.hpp:36
#define ABORT(...)
Definition: abort.hpp:40
Definition: gzip.hpp:88
Definition: check.hpp:33
Error(const std::string &_message)
Definition: errorbase.hpp:39
const int code
Definition: gzip.hpp:52
GzipError(const z_stream_s &stream, int _code)
Definition: gzip.hpp:46
Try< std::string > compress(const std::string &decompressed, int level=Z_DEFAULT_COMPRESSION)
Definition: gzip.hpp:179
Try< std::string > decompress(const std::string &compressed)
Definition: gzip.hpp:122
Definition: gzip.hpp:29
GzipError(int _code)
Definition: gzip.hpp:49
Try< std::string > decompress(const std::string &compressed)
Definition: gzip.hpp:243
bool finished() const
Definition: gzip.hpp:161
bool isSome() const
Definition: try.hpp:70
GzipError(const std::string &message, int _code)
Definition: gzip.hpp:43
~Decompressor()
Definition: gzip.hpp:113
const std::string message
Definition: errorbase.hpp:46
Definition: attributes.hpp:24
std::string error(const std::string &msg, uint32_t code)
std::string stringify(int flags)
Definition: gzip.hpp:37
Decompressor()
Definition: gzip.hpp:91
#define GZIP_BUFFER_SIZE
Definition: gzip.hpp:34