Apache Mesos
protobuf.hpp
Go to the documentation of this file.
1 // Licensed under the Apache License, Version 2.0 (the "License");
2 // you may not use this file except in compliance with the License.
3 // You may obtain a copy of the License at
4 //
5 // http://www.apache.org/licenses/LICENSE-2.0
6 //
7 // Unless required by applicable law or agreed to in writing, software
8 // distributed under the License is distributed on an "AS IS" BASIS,
9 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 // See the License for the specific language governing permissions and
11 // limitations under the License.
12 
13 #ifndef __STOUT_PROTOBUF_HPP__
14 #define __STOUT_PROTOBUF_HPP__
15 
16 #include <assert.h>
17 #include <errno.h>
18 #include <stdint.h>
19 #ifndef __WINDOWS__
20 #include <unistd.h>
21 #endif // __WINDOWS__
22 
23 #include <sys/types.h>
24 
25 #include <string>
26 #include <type_traits>
27 #include <vector>
28 
29 #include <google/protobuf/descriptor.h>
30 #include <google/protobuf/descriptor.pb.h>
31 #include <google/protobuf/message.h>
32 #include <google/protobuf/repeated_field.h>
33 
34 #include <google/protobuf/io/zero_copy_stream_impl.h>
35 
36 #include <stout/abort.hpp>
37 #include <stout/base64.hpp>
38 #include <stout/error.hpp>
39 #include <stout/json.hpp>
40 #include <stout/jsonify.hpp>
41 #include <stout/none.hpp>
42 #include <stout/nothing.hpp>
43 #include <stout/representation.hpp>
44 #include <stout/result.hpp>
45 #include <stout/stringify.hpp>
46 #include <stout/try.hpp>
47 
48 #include <stout/os/close.hpp>
49 #include <stout/os/fsync.hpp>
50 #include <stout/os/int_fd.hpp>
51 #include <stout/os/lseek.hpp>
52 #include <stout/os/open.hpp>
53 #include <stout/os/read.hpp>
54 #include <stout/os/write.hpp>
55 
56 #ifdef __WINDOWS__
57 #include <stout/os/dup.hpp>
58 #endif // __WINDOWS__
59 
60 namespace protobuf {
61 
62 // TODO(bmahler): Re-use stout's 'recordio' facilities here. Note
63 // that these use a fixed size length header, whereas stout's
64 // currently uses a base-10 newline delimited header for language
65 // portability, which makes changing these a bit tricky.
66 
67 // Write out the given protobuf to the specified file descriptor by
68 // first writing out the length of the protobuf followed by the
69 // contents.
70 // NOTE: On error, this may have written partial data to the file.
71 inline Try<Nothing> write(int_fd fd, const google::protobuf::Message& message)
72 {
73  if (!message.IsInitialized()) {
74  return Error(message.InitializationErrorString() +
75  " is required but not initialized");
76  }
77 
78  // First write the size of the protobuf.
79  uint32_t size = message.ByteSize();
80  std::string bytes((char*) &size, sizeof(size));
81 
82  Try<Nothing> result = os::write(fd, bytes);
83  if (result.isError()) {
84  return Error("Failed to write size: " + result.error());
85  }
86 
87 #ifdef __WINDOWS__
88  // NOTE: On Windows, we need to explicitly allocate a CRT file
89  // descriptor because the Protobuf library requires it. Because
90  // users of `protobuf::write` are likely to call `os::close` on the
91  // `fd` we were given, we need to duplicate it before allocating the
92  // CRT fd. This is because once the CRT fd is allocated, it must be
93  // closed with `_close` instead of `os::close`. Since we need to
94  // call `_close` here, we duplicate the fd to prevent the users call
95  // of `os::close` from closing twice.
96  Try<int_fd> dup = os::dup(fd);
97  if (dup.isError()) {
98  return Error("Failed to duplicate handle: " + dup.error());
99  }
100 
101  int crt = dup->crt();
102 
103  if (!message.SerializeToFileDescriptor(crt)) {
104  ::_close(crt);
105  return Error("Failed to write/serialize message");
106  }
107  ::_close(crt);
108 #else
109  if (!message.SerializeToFileDescriptor(fd)) {
110  return Error("Failed to write/serialize message");
111  }
112 #endif
113 
114  return Nothing();
115 }
116 
117 // Write out the given sequence of protobuf messages to the
118 // specified file descriptor by repeatedly invoking write
119 // on each of the messages.
120 // NOTE: On error, this may have written partial data to the file.
121 template <typename T>
123  int_fd fd, const google::protobuf::RepeatedPtrField<T>& messages)
124 {
125  foreach (const T& message, messages) {
126  Try<Nothing> result = write(fd, message);
127  if (result.isError()) {
128  return Error(result.error());
129  }
130  }
131 
132  return Nothing();
133 }
134 
135 
136 // A wrapper function for the above `write()` with opening and closing the file.
137 // If `sync` is set to true, an `fsync()` will be called before `close()`.
138 template <typename T>
139 Try<Nothing> write(const std::string& path, const T& t, bool sync = false)
140 {
141  Try<int_fd> fd = os::open(
142  path,
143  O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC,
145 
146  if (fd.isError()) {
147  return Error("Failed to open file '" + path + "': " + fd.error());
148  }
149 
150  Try<Nothing> result = write(fd.get(), t);
151 
152  if (sync && result.isSome()) {
153  // We call `fsync()` before closing the file instead of opening it with the
154  // `O_SYNC` flag for better performance. See:
155  // http://lkml.iu.edu/hypermail/linux/kernel/0105.3/0353.html
156  result = os::fsync(fd.get());
157  }
158 
159  // We ignore the return value of `close()` because users calling this function
160  // are interested in the return value of `write()`, or `fsync()` if `sync` is
161  // set to true. Also an unsuccessful `close()` doesn't affect the write.
162  os::close(fd.get());
163 
164  return result;
165 }
166 
167 
168 // A wrapper function to append a protobuf message with opening and closing the
169 // file. If `sync` is set to true, an `fsync()` will be called before `close()`.
171  const std::string& path,
172  const google::protobuf::Message& message,
173  bool sync = false)
174 {
175  Try<int_fd> fd = os::open(
176  path,
177  O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC,
179 
180  if (fd.isError()) {
181  return Error("Failed to open file '" + path + "': " + fd.error());
182  }
183 
184  Try<Nothing> result = write(fd.get(), message);
185 
186  if (sync && result.isSome()) {
187  // We call `fsync()` before closing the file instead of opening it with the
188  // `O_SYNC` flag for better performance.
189  result = os::fsync(fd.get());
190  }
191 
192  // NOTE: We ignore the return value of close(). This is because
193  // users calling this function are interested in the return value of
194  // write(). Also an unsuccessful close() doesn't affect the write.
195  os::close(fd.get());
196 
197  return result;
198 }
199 
200 
201 template <typename T>
202 Try<T> deserialize(const std::string& value)
203 {
204  T t;
205  (void) static_cast<google::protobuf::Message*>(&t);
206 
207  // Verify that the size of `value` fits into `ArrayInputStream`'s
208  // constructor. The maximum size of a proto2 message is 64 MB, so it is
209  // unlikely that we will hit this limit, but since an arbitrary string can be
210  // passed in, we include this check to be safe.
211  CHECK_LE(value.size(), static_cast<size_t>(std::numeric_limits<int>::max()));
212  google::protobuf::io::ArrayInputStream stream(
213  value.data(),
214  static_cast<int>(value.size()));
215  if (!t.ParseFromZeroCopyStream(&stream)) {
216  return Error("Failed to deserialize " + t.GetDescriptor()->full_name());
217  }
218  return t;
219 }
220 
221 
222 template <typename T>
224 {
225  (void) static_cast<const google::protobuf::Message*>(&t);
226 
227  std::string value;
228  if (!t.SerializeToString(&value)) {
229  return Error("Failed to serialize " + t.GetDescriptor()->full_name());
230  }
231  return value;
232 }
233 
234 
235 namespace internal {
236 
237 // Reads a single message of type T from the file by first reading the
238 // "size" followed by the contents (as written by 'write' above).
239 // NOTE: This struct is used by the public 'read' function.
240 // See comments there for the reason why we need this.
241 template <typename T>
242 struct Read
243 {
244  Result<T> operator()(int_fd fd, bool ignorePartial, bool undoFailed)
245  {
246  off_t offset = 0;
247 
248  if (undoFailed) {
249  // Save the offset so we can re-adjust if something goes wrong.
250  Try<off_t> lseek = os::lseek(fd, offset, SEEK_CUR);
251  if (lseek.isError()) {
252  return Error(lseek.error());
253  }
254 
255  offset = lseek.get();
256  }
257 
258  uint32_t size;
259  Result<std::string> result = os::read(fd, sizeof(size));
260 
261  if (result.isError()) {
262  if (undoFailed) {
263  os::lseek(fd, offset, SEEK_SET);
264  }
265  return Error("Failed to read size: " + result.error());
266  } else if (result.isNone()) {
267  return None(); // No more protobufs to read.
268  } else if (result->size() < sizeof(size)) {
269  // Hit EOF unexpectedly.
270  if (undoFailed) {
271  // Restore the offset to before the size read.
272  os::lseek(fd, offset, SEEK_SET);
273  }
274  if (ignorePartial) {
275  return None();
276  }
277  return Error(
278  "Failed to read size: hit EOF unexpectedly, possible corruption");
279  }
280 
281  // Parse the size from the bytes.
282  memcpy((void*)&size, (void*)result->data(), sizeof(size));
283 
284  // NOTE: Instead of specifically checking for corruption in 'size',
285  // we simply try to read 'size' bytes. If we hit EOF early, it is an
286  // indication of corruption.
287  result = os::read(fd, size);
288 
289  if (result.isError()) {
290  if (undoFailed) {
291  // Restore the offset to before the size read.
292  os::lseek(fd, offset, SEEK_SET);
293  }
294  return Error("Failed to read message: " + result.error());
295  } else if (result.isNone() || result->size() < size) {
296  // Hit EOF unexpectedly.
297  if (undoFailed) {
298  // Restore the offset to before the size read.
299  os::lseek(fd, offset, SEEK_SET);
300  }
301  if (ignorePartial) {
302  return None();
303  }
304  return Error("Failed to read message of size " + stringify(size) +
305  " bytes: hit EOF unexpectedly, possible corruption");
306  }
307 
308  // Parse the protobuf from the string.
309  // NOTE: We need to capture a const reference to the data because it
310  // must outlive the creation of ArrayInputStream.
311  const std::string& data = result.get();
312 
313  // Verify that the size of `data` fits into `ArrayInputStream`'s
314  // constructor. The maximum size of a proto2 message is 64 MB, so it is
315  // unlikely that we will hit this limit, but since an arbitrary string can
316  // be passed in, we include this check to be safe.
317  CHECK_LE(data.size(), static_cast<size_t>(std::numeric_limits<int>::max()));
318  T message;
319  google::protobuf::io::ArrayInputStream stream(
320  data.data(),
321  static_cast<int>(data.size()));
322 
323  if (!message.ParseFromZeroCopyStream(&stream)) {
324  if (undoFailed) {
325  // Restore the offset to before the size read.
326  os::lseek(fd, offset, SEEK_SET);
327  }
328  return Error("Failed to deserialize message");
329  }
330 
331  return message;
332  }
333 };
334 
335 
336 // Partial specialization for RepeatedPtrField<T> to read a sequence
337 // of protobuf messages from a given fd by repeatedly invoking
338 // Read<T> until None is reached, which we treat as EOF.
339 // NOTE: This struct is used by the public 'read' function.
340 // See comments there for the reason why we need this.
341 template <typename T>
342 struct Read<google::protobuf::RepeatedPtrField<T>>
343 {
345  int_fd fd, bool ignorePartial, bool undoFailed)
346  {
347  google::protobuf::RepeatedPtrField<T> result;
348  for (;;) {
349  Result<T> message = Read<T>()(fd, ignorePartial, undoFailed);
350  if (message.isError()) {
351  return Error(message.error());
352  } else if (message.isNone()) {
353  break;
354  } else {
355  result.Add()->CopyFrom(message.get());
356  }
357  }
358  return result;
359  }
360 };
361 
362 } // namespace internal {
363 
364 
365 // Reads the protobuf message(s) from a given fd based on the format
366 // written by write() above. We use partial specialization of
367 // - internal::Read<T> vs
368 // - internal::Read<google::protobuf::RepeatedPtrField<T>>
369 // in order to determine whether T is a single protobuf message or
370 // a sequence of messages.
371 // If 'ignorePartial' is true, None() is returned when we unexpectedly
372 // hit EOF while reading the protobuf (e.g., partial write).
373 // If 'undoFailed' is true, failed read attempts will restore the file
374 // read/write file offset towards the initial callup position.
375 template <typename T>
376 Result<T> read(int_fd fd, bool ignorePartial = false, bool undoFailed = false)
377 {
378  return internal::Read<T>()(fd, ignorePartial, undoFailed);
379 }
380 
381 
382 // A wrapper function that wraps the above read() with open and
383 // closing the file.
384 template <typename T>
385 Result<T> read(const std::string& path)
386 {
387  Try<int_fd> fd = os::open(
388  path,
389  O_RDONLY | O_CLOEXEC,
391 
392  if (fd.isError()) {
393  return Error("Failed to open file '" + path + "': " + fd.error());
394  }
395 
396  Result<T> result = read<T>(fd.get());
397 
398  // NOTE: We ignore the return value of close(). This is because
399  // users calling this function are interested in the return value of
400  // read(). Also an unsuccessful close() doesn't affect the read.
401  os::close(fd.get());
402 
403  return result;
404 }
405 
406 
407 namespace internal {
408 
409 // Forward declaration.
411  google::protobuf::Message* message,
412  const JSON::Object& object);
413 
414 
415 struct Parser : boost::static_visitor<Try<Nothing>>
416 {
417  Parser(google::protobuf::Message* _message,
418  const google::protobuf::FieldDescriptor* _field)
419  : message(_message),
420  reflection(message->GetReflection()),
421  field(_field) {}
422 
423  Try<Nothing> operator()(const JSON::Object& object) const
424  {
425  switch (field->type()) {
426  case google::protobuf::FieldDescriptor::TYPE_MESSAGE:
427  if (field->is_map()) {
428  foreachpair (
429  const std::string& name,
430  const JSON::Value& value,
431  object.values) {
432  google::protobuf::Message* entry =
433  reflection->AddMessage(message, field);
434 
435  // A map is equivalent to:
436  //
437  // message MapFieldEntry {
438  // optional key_type key = 1;
439  // optional value_type value = 2;
440  // }
441  //
442  // repeated MapFieldEntry map_field = N;
443  //
444  // See the link below for details:
445  // https://developers.google.com/protocol-buffers/docs/proto#maps
446  const google::protobuf::FieldDescriptor* key_field =
447  entry->GetDescriptor()->FindFieldByNumber(1);
448 
449  JSON::Value key(name);
450 
451  Try<Nothing> apply =
452  boost::apply_visitor(Parser(entry, key_field), key);
453 
454  if (apply.isError()) {
455  return Error(apply.error());
456  }
457 
458  const google::protobuf::FieldDescriptor* value_field =
459  entry->GetDescriptor()->FindFieldByNumber(2);
460 
461  apply = boost::apply_visitor(Parser(entry, value_field), value);
462  if (apply.isError()) {
463  return Error(apply.error());
464  }
465  }
466  } else if (field->is_repeated()) {
467  // TODO(gilbert): We currently push up the nested error
468  // messages without wrapping the error message (due to
469  // the recursive nature of parse). We should pass along
470  // variable information in order to construct a helpful
471  // error message, e.g. "Failed to parse field 'a.b.c': ...".
472  return parse(reflection->AddMessage(message, field), object);
473  } else {
474  return parse(reflection->MutableMessage(message, field), object);
475  }
476  break;
477  default:
478  return Error("Not expecting a JSON object for field '" +
479  field->name() + "'");
480  }
481  return Nothing();
482  }
483 
484  Try<Nothing> operator()(const JSON::String& string) const
485  {
486  switch (field->type()) {
487  case google::protobuf::FieldDescriptor::TYPE_STRING:
488  if (field->is_repeated()) {
489  reflection->AddString(message, field, string.value);
490  } else {
491  reflection->SetString(message, field, string.value);
492  }
493  break;
494  case google::protobuf::FieldDescriptor::TYPE_BYTES: {
495  Try<std::string> decode = base64::decode(string.value);
496  if (decode.isError()) {
497  return Error("Failed to base64 decode bytes field"
498  " '" + field->name() + "': " + decode.error());
499  }
500 
501  if (field->is_repeated()) {
502  reflection->AddString(message, field, decode.get());
503  } else {
504  reflection->SetString(message, field, decode.get());
505  }
506  break;
507  }
508  case google::protobuf::FieldDescriptor::TYPE_ENUM: {
509  const google::protobuf::EnumValueDescriptor* descriptor =
510  field->enum_type()->FindValueByName(string.value);
511 
512  if (descriptor == nullptr) {
513  if (field->is_required()) {
514  return Error("Failed to find enum for '" + string.value + "'");
515  }
516 
517  // Unrecognized enum value will be discarded if this is not a
518  // required enum field, which makes the field's `has..` accessor
519  // return false and its getter return the first value listed in
520  // the enum definition, or the default value if one is specified.
521  //
522  // This is the deserialization behavior of proto2, see the link
523  // below for details:
524  // https://developers.google.com/protocol-buffers/docs/proto#updating
525  break;
526  }
527 
528  if (field->is_repeated()) {
529  reflection->AddEnum(message, field, descriptor);
530  } else {
531  reflection->SetEnum(message, field, descriptor);
532  }
533  break;
534  }
535  case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
536  case google::protobuf::FieldDescriptor::TYPE_FLOAT:
537  case google::protobuf::FieldDescriptor::TYPE_INT64:
538  case google::protobuf::FieldDescriptor::TYPE_SINT64:
539  case google::protobuf::FieldDescriptor::TYPE_SFIXED64:
540  case google::protobuf::FieldDescriptor::TYPE_UINT64:
541  case google::protobuf::FieldDescriptor::TYPE_FIXED64:
542  case google::protobuf::FieldDescriptor::TYPE_INT32:
543  case google::protobuf::FieldDescriptor::TYPE_SINT32:
544  case google::protobuf::FieldDescriptor::TYPE_SFIXED32:
545  case google::protobuf::FieldDescriptor::TYPE_UINT32:
546  case google::protobuf::FieldDescriptor::TYPE_FIXED32: {
547  Try<JSON::Number> number = JSON::parse<JSON::Number>(string.value);
548  if (number.isError()) {
549  return Error(
550  "Failed to parse '" + string.value + "' as a JSON number "
551  "for field '" + field->name() + "': " + number.error());
552  }
553 
554  return operator()(number.get());
555  }
556  case google::protobuf::FieldDescriptor::TYPE_BOOL: {
557  Try<JSON::Boolean> boolean = JSON::parse<JSON::Boolean>(string.value);
558  if (boolean.isError()) {
559  return Error(
560  "Failed to parse '" + string.value + "' as a JSON boolean "
561  "for field '" + field->name() + "': " + boolean.error());
562  }
563 
564  return operator()(boolean.get());
565  }
566  default:
567  return Error("Not expecting a JSON string for field '" +
568  field->name() + "'");
569  }
570  return Nothing();
571  }
572 
573  Try<Nothing> operator()(const JSON::Number& number) const
574  {
575  switch (field->type()) {
576  case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
577  if (field->is_repeated()) {
578  reflection->AddDouble(message, field, number.as<double>());
579  } else {
580  reflection->SetDouble(message, field, number.as<double>());
581  }
582  break;
583  case google::protobuf::FieldDescriptor::TYPE_FLOAT:
584  if (field->is_repeated()) {
585  reflection->AddFloat(message, field, number.as<float>());
586  } else {
587  reflection->SetFloat(message, field, number.as<float>());
588  }
589  break;
590  case google::protobuf::FieldDescriptor::TYPE_INT64:
591  case google::protobuf::FieldDescriptor::TYPE_SINT64:
592  case google::protobuf::FieldDescriptor::TYPE_SFIXED64:
593  if (field->is_repeated()) {
594  reflection->AddInt64(message, field, number.as<int64_t>());
595  } else {
596  reflection->SetInt64(message, field, number.as<int64_t>());
597  }
598  break;
599  case google::protobuf::FieldDescriptor::TYPE_UINT64:
600  case google::protobuf::FieldDescriptor::TYPE_FIXED64:
601  if (field->is_repeated()) {
602  reflection->AddUInt64(message, field, number.as<uint64_t>());
603  } else {
604  reflection->SetUInt64(message, field, number.as<uint64_t>());
605  }
606  break;
607  case google::protobuf::FieldDescriptor::TYPE_INT32:
608  case google::protobuf::FieldDescriptor::TYPE_SINT32:
609  case google::protobuf::FieldDescriptor::TYPE_SFIXED32:
610  if (field->is_repeated()) {
611  reflection->AddInt32(message, field, number.as<int32_t>());
612  } else {
613  reflection->SetInt32(message, field, number.as<int32_t>());
614  }
615  break;
616  case google::protobuf::FieldDescriptor::TYPE_UINT32:
617  case google::protobuf::FieldDescriptor::TYPE_FIXED32:
618  if (field->is_repeated()) {
619  reflection->AddUInt32(message, field, number.as<uint32_t>());
620  } else {
621  reflection->SetUInt32(message, field, number.as<uint32_t>());
622  }
623  break;
624  default:
625  return Error("Not expecting a JSON number for field '" +
626  field->name() + "'");
627  }
628  return Nothing();
629  }
630 
631  Try<Nothing> operator()(const JSON::Array& array) const
632  {
633  if (!field->is_repeated()) {
634  return Error("Not expecting a JSON array for field '" +
635  field->name() + "'");
636  }
637 
638  foreach (const JSON::Value& value, array.values) {
639  Try<Nothing> apply =
640  boost::apply_visitor(Parser(message, field), value);
641 
642  if (apply.isError()) {
643  return Error(apply.error());
644  }
645  }
646 
647  return Nothing();
648  }
649 
650  Try<Nothing> operator()(const JSON::Boolean& boolean) const
651  {
652  switch (field->type()) {
653  case google::protobuf::FieldDescriptor::TYPE_BOOL:
654  if (field->is_repeated()) {
655  reflection->AddBool(message, field, boolean.value);
656  } else {
657  reflection->SetBool(message, field, boolean.value);
658  }
659  break;
660  default:
661  return Error("Not expecting a JSON boolean for field '" +
662  field->name() + "'");
663  }
664  return Nothing();
665  }
666 
668  {
669  // We treat 'null' as an unset field. Note that we allow
670  // unset required fields here since the top-level parse
671  // function is responsible for checking 'IsInitialized'.
672  return Nothing();
673  }
674 
675 private:
676  google::protobuf::Message* message;
677  const google::protobuf::Reflection* reflection;
678  const google::protobuf::FieldDescriptor* field;
679 };
680 
681 
683  google::protobuf::Message* message,
684  const JSON::Object& object)
685 {
686  foreachpair (
687  const std::string& name, const JSON::Value& value, object.values) {
688  // Look for a field by this name.
689  const google::protobuf::FieldDescriptor* field =
690  message->GetDescriptor()->FindFieldByName(name);
691 
692  if (field != nullptr) {
693  Try<Nothing> apply =
694  boost::apply_visitor(Parser(message, field), value);
695 
696  if (apply.isError()) {
697  return Error(apply.error());
698  }
699  }
700  }
701 
702  return Nothing();
703 }
704 
705 
706 // Parses a single protobuf message of type T from a JSON::Object.
707 // NOTE: This struct is used by the public parse<T>() function below. See
708 // comments there for the reason why we opted for this design.
709 template <typename T>
710 struct Parse
711 {
713  {
714  static_assert(std::is_convertible<T*, google::protobuf::Message*>::value,
715  "T must be a protobuf message");
716 
717  const JSON::Object* object = boost::get<JSON::Object>(&value);
718  if (object == nullptr) {
719  return Error("Expecting a JSON object");
720  }
721 
722  T message;
723 
724  Try<Nothing> parse = internal::parse(&message, *object);
725  if (parse.isError()) {
726  return Error(parse.error());
727  }
728 
729  if (!message.IsInitialized()) {
730  return Error("Missing required fields: " +
731  message.InitializationErrorString());
732  }
733 
734  return message;
735  }
736 };
737 
738 
739 // Partial specialization for RepeatedPtrField<T> to parse a sequence of
740 // protobuf messages from a JSON::Array by repeatedly invoking Parse<T> to
741 // facilitate conversions like JSON::Array -> Resources.
742 // NOTE: This struct is used by the public parse<T>() function below. See
743 // comments there for the reason why we opted for this design.
744 template <typename T>
745 struct Parse<google::protobuf::RepeatedPtrField<T>>
746 {
748  const JSON::Value& value)
749  {
750  static_assert(std::is_convertible<T*, google::protobuf::Message*>::value,
751  "T must be a protobuf message");
752 
753  const JSON::Array* array = boost::get<JSON::Array>(&value);
754  if (array == nullptr) {
755  return Error("Expecting a JSON array");
756  }
757 
758  google::protobuf::RepeatedPtrField<T> collection;
759  collection.Reserve(static_cast<int>(array->values.size()));
760 
761  // Parse messages one by one and propagate an error if it happens.
762  foreach (const JSON::Value& elem, array->values) {
763  Try<T> message = Parse<T>()(elem);
764  if (message.isError()) {
765  return Error(message.error());
766  }
767 
768  collection.Add()->CopyFrom(message.get());
769  }
770 
771  return collection;
772  }
773 };
774 
775 } // namespace internal {
776 
777 // A dispatch wrapper which parses protobuf messages(s) from a given JSON value.
778 // We use partial specialization of
779 // - internal::Parse<T> for JSON::Object
780 // - internal::Parse<google::protobuf::RepeatedPtrField<T>> for JSON::Array
781 // to determine whether T is a single message or a sequence of messages.
782 // We cannot partially specialize function templates and overloaded function
783 // approach combined with std::enable_if is not that clean, hence we leverage
784 // partial specialization of class templates.
785 template <typename T>
786 Try<T> parse(const JSON::Value& value)
787 {
788  return internal::Parse<T>()(value);
789 }
790 
791 } // namespace protobuf {
792 
793 namespace JSON {
794 
795 // The representation of generic protobuf => JSON,
796 // e.g., `jsonify(JSON::Protobuf(message))`.
797 struct Protobuf : Representation<google::protobuf::Message>
798 {
800 };
801 
802 
803 // `json` function for protobuf messages. Refer to `jsonify.hpp` for details.
804 // TODO(mpark): This currently uses the default value for optional fields
805 // that are not deprecated, but we may want to revisit this decision.
806 inline void json(ObjectWriter* writer, const Protobuf& protobuf)
807 {
808  using google::protobuf::FieldDescriptor;
809 
810  const google::protobuf::Message& message = protobuf;
811 
812  const google::protobuf::Descriptor* descriptor = message.GetDescriptor();
813  const google::protobuf::Reflection* reflection = message.GetReflection();
814 
815  // We first look through all the possible fields to determine both the set
816  // fields __and__ the optional fields with a default that are not set.
817  // `Reflection::ListFields()` alone will only include set fields and
818  // is therefore insufficient.
819  int fieldCount = descriptor->field_count();
820  std::vector<const FieldDescriptor*> fields;
821  fields.reserve(fieldCount);
822  for (int i = 0; i < fieldCount; ++i) {
823  const FieldDescriptor* field = descriptor->field(i);
824  if (field->is_repeated()) {
825  if (reflection->FieldSize(message, field) > 0) {
826  // Has repeated field with members, output as JSON.
827  fields.push_back(field);
828  }
829  } else if (
830  reflection->HasField(message, field) ||
831  (field->has_default_value() && !field->options().deprecated())) {
832  // Field is set or has default, output as JSON.
833  fields.push_back(field);
834  }
835  }
836 
837  foreach (const FieldDescriptor* field, fields) {
838  if (field->is_repeated()) {
839  writer->field(
840  field->name(),
841  [&field, &reflection, &message](JSON::ArrayWriter* writer) {
842  int fieldSize = reflection->FieldSize(message, field);
843  for (int i = 0; i < fieldSize; ++i) {
844  switch (field->cpp_type()) {
845  case FieldDescriptor::CPPTYPE_BOOL:
846  writer->element(
847  reflection->GetRepeatedBool(message, field, i));
848  break;
849  case FieldDescriptor::CPPTYPE_INT32:
850  writer->element(
851  reflection->GetRepeatedInt32(message, field, i));
852  break;
853  case FieldDescriptor::CPPTYPE_INT64:
854  writer->element(
855  reflection->GetRepeatedInt64(message, field, i));
856  break;
857  case FieldDescriptor::CPPTYPE_UINT32:
858  writer->element(
859  reflection->GetRepeatedUInt32(message, field, i));
860  break;
861  case FieldDescriptor::CPPTYPE_UINT64:
862  writer->element(
863  reflection->GetRepeatedUInt64(message, field, i));
864  break;
865  case FieldDescriptor::CPPTYPE_FLOAT:
866  writer->element(
867  reflection->GetRepeatedFloat(message, field, i));
868  break;
869  case FieldDescriptor::CPPTYPE_DOUBLE:
870  writer->element(
871  reflection->GetRepeatedDouble(message, field, i));
872  break;
873  case FieldDescriptor::CPPTYPE_MESSAGE:
874  writer->element(Protobuf(
875  reflection->GetRepeatedMessage(message, field, i)));
876  break;
877  case FieldDescriptor::CPPTYPE_ENUM:
878  writer->element(
879  reflection->GetRepeatedEnum(message, field, i)->name());
880  break;
881  case FieldDescriptor::CPPTYPE_STRING:
882  const std::string& s = reflection->GetRepeatedStringReference(
883  message, field, i, nullptr);
884  if (field->type() == FieldDescriptor::TYPE_BYTES) {
885  writer->element(base64::encode(s));
886  } else {
887  writer->element(s);
888  }
889  break;
890  }
891  }
892  });
893  } else {
894  switch (field->cpp_type()) {
895  case FieldDescriptor::CPPTYPE_BOOL:
896  writer->field(field->name(), reflection->GetBool(message, field));
897  break;
898  case FieldDescriptor::CPPTYPE_INT32:
899  writer->field(field->name(), reflection->GetInt32(message, field));
900  break;
901  case FieldDescriptor::CPPTYPE_INT64:
902  writer->field(field->name(), reflection->GetInt64(message, field));
903  break;
904  case FieldDescriptor::CPPTYPE_UINT32:
905  writer->field(field->name(), reflection->GetUInt32(message, field));
906  break;
907  case FieldDescriptor::CPPTYPE_UINT64:
908  writer->field(field->name(), reflection->GetUInt64(message, field));
909  break;
910  case FieldDescriptor::CPPTYPE_FLOAT:
911  writer->field(field->name(), reflection->GetFloat(message, field));
912  break;
913  case FieldDescriptor::CPPTYPE_DOUBLE:
914  writer->field(field->name(), reflection->GetDouble(message, field));
915  break;
916  case FieldDescriptor::CPPTYPE_MESSAGE:
917  writer->field(
918  field->name(), Protobuf(reflection->GetMessage(message, field)));
919  break;
920  case FieldDescriptor::CPPTYPE_ENUM:
921  writer->field(
922  field->name(), reflection->GetEnum(message, field)->name());
923  break;
924  case FieldDescriptor::CPPTYPE_STRING:
925  const std::string& s = reflection->GetStringReference(
926  message, field, nullptr);
927  if (field->type() == FieldDescriptor::TYPE_BYTES) {
928  writer->field(field->name(), base64::encode(s));
929  } else {
930  writer->field(field->name(), s);
931  }
932  break;
933  }
934  }
935  }
936 }
937 
938 
939 // TODO(bmahler): This currently uses the default value for optional fields
940 // that are not deprecated, but we may want to revisit this decision.
941 inline Object protobuf(const google::protobuf::Message& message)
942 {
943  Object object;
944 
945  const google::protobuf::Descriptor* descriptor = message.GetDescriptor();
946  const google::protobuf::Reflection* reflection = message.GetReflection();
947 
948  auto value_for_field = [](
949  const google::protobuf::Message& message,
950  const google::protobuf::FieldDescriptor* field) -> JSON::Value {
951  const google::protobuf::Reflection* reflection = message.GetReflection();
952 
953  switch (field->type()) {
954  case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
955  return JSON::Number(reflection->GetDouble(message, field));
956  case google::protobuf::FieldDescriptor::TYPE_FLOAT:
957  return JSON::Number(reflection->GetFloat(message, field));
958  case google::protobuf::FieldDescriptor::TYPE_INT64:
959  case google::protobuf::FieldDescriptor::TYPE_SINT64:
960  case google::protobuf::FieldDescriptor::TYPE_SFIXED64:
961  return JSON::Number(reflection->GetInt64(message, field));
962  case google::protobuf::FieldDescriptor::TYPE_UINT64:
963  case google::protobuf::FieldDescriptor::TYPE_FIXED64:
964  return JSON::Number(reflection->GetUInt64(message, field));
965  case google::protobuf::FieldDescriptor::TYPE_INT32:
966  case google::protobuf::FieldDescriptor::TYPE_SINT32:
967  case google::protobuf::FieldDescriptor::TYPE_SFIXED32:
968  return JSON::Number(reflection->GetInt32(message, field));
969  case google::protobuf::FieldDescriptor::TYPE_UINT32:
970  case google::protobuf::FieldDescriptor::TYPE_FIXED32:
971  return JSON::Number(reflection->GetUInt32(message, field));
972  case google::protobuf::FieldDescriptor::TYPE_BOOL:
973  if (reflection->GetBool(message, field)) {
974  return JSON::Boolean(true);
975  } else {
976  return JSON::Boolean(false);
977  }
978  break;
979  case google::protobuf::FieldDescriptor::TYPE_STRING:
980  return JSON::String(reflection->GetString(message, field));
981  case google::protobuf::FieldDescriptor::TYPE_BYTES:
982  return JSON::String(
983  base64::encode(reflection->GetString(message, field)));
984  case google::protobuf::FieldDescriptor::TYPE_MESSAGE:
985  return protobuf(reflection->GetMessage(message, field));
986  case google::protobuf::FieldDescriptor::TYPE_ENUM:
987  return JSON::String(reflection->GetEnum(message, field)->name());
988  case google::protobuf::FieldDescriptor::TYPE_GROUP:
989  // Deprecated! We abort here instead of using a Try as return value,
990  // because we expect this code path to never be taken.
991  ABORT("Unhandled protobuf field type: " +
992  stringify(field->type()));
993  }
994 
995  UNREACHABLE();
996  };
997 
998  // We first look through all the possible fields to determine both
999  // the set fields _and_ the optional fields with a default that
1000  // are not set. Reflection::ListFields() alone will only include
1001  // set fields and is therefore insufficient.
1002  std::vector<const google::protobuf::FieldDescriptor*> fields;
1003  fields.reserve(descriptor->field_count());
1004  for (int i = 0; i < descriptor->field_count(); i++) {
1005  const google::protobuf::FieldDescriptor* field = descriptor->field(i);
1006  if (field->is_repeated()) {
1007  if (reflection->FieldSize(message, descriptor->field(i)) > 0) {
1008  // Has repeated field with members, output as JSON.
1009  fields.push_back(field);
1010  }
1011  } else if (
1012  reflection->HasField(message, field) ||
1013  (field->has_default_value() && !field->options().deprecated())) {
1014  // Field is set or has default, output as JSON.
1015  fields.push_back(field);
1016  }
1017  }
1018 
1019  foreach (const google::protobuf::FieldDescriptor* field, fields) {
1020  if (field->is_map()) {
1021  JSON::Object map;
1022 
1023  int fieldSize = reflection->FieldSize(message, field);
1024  for (int i = 0; i < fieldSize; ++i) {
1025  const google::protobuf::Message& entry =
1026  reflection->GetRepeatedMessage(message, field, i);
1027 
1028  // A map is equivalent to:
1029  //
1030  // message MapFieldEntry {
1031  // optional key_type key = 1;
1032  // optional value_type value = 2;
1033  // }
1034  //
1035  // repeated MapFieldEntry map_field = N;
1036  //
1037  // See the link below for details:
1038  // https://developers.google.com/protocol-buffers/docs/proto#maps
1039  const google::protobuf::FieldDescriptor* key_field =
1040  entry.GetDescriptor()->FindFieldByNumber(1);
1041 
1042  const google::protobuf::FieldDescriptor* value_field =
1043  entry.GetDescriptor()->FindFieldByNumber(2);
1044 
1045  JSON::Value key = value_for_field(entry, key_field);
1046 
1047  std::string name;
1048  if (key.is<JSON::String>()) {
1049  name = key.as<JSON::String>().value;
1050  } else {
1051  name = jsonify(key);
1052  }
1053 
1054  map.values[name] = value_for_field(entry, value_field);
1055  }
1056  object.values[field->name()] = map;
1057  } else if (field->is_repeated()) {
1058  JSON::Array array;
1059  int fieldSize = reflection->FieldSize(message, field);
1060  array.values.reserve(fieldSize);
1061  for (int i = 0; i < fieldSize; ++i) {
1062  switch (field->type()) {
1063  case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
1064  array.values.push_back(JSON::Number(
1065  reflection->GetRepeatedDouble(message, field, i)));
1066  break;
1067  case google::protobuf::FieldDescriptor::TYPE_FLOAT:
1068  array.values.push_back(JSON::Number(
1069  reflection->GetRepeatedFloat(message, field, i)));
1070  break;
1071  case google::protobuf::FieldDescriptor::TYPE_INT64:
1072  case google::protobuf::FieldDescriptor::TYPE_SINT64:
1073  case google::protobuf::FieldDescriptor::TYPE_SFIXED64:
1074  array.values.push_back(JSON::Number(
1075  reflection->GetRepeatedInt64(message, field, i)));
1076  break;
1077  case google::protobuf::FieldDescriptor::TYPE_UINT64:
1078  case google::protobuf::FieldDescriptor::TYPE_FIXED64:
1079  array.values.push_back(JSON::Number(
1080  reflection->GetRepeatedUInt64(message, field, i)));
1081  break;
1082  case google::protobuf::FieldDescriptor::TYPE_INT32:
1083  case google::protobuf::FieldDescriptor::TYPE_SINT32:
1084  case google::protobuf::FieldDescriptor::TYPE_SFIXED32:
1085  array.values.push_back(JSON::Number(
1086  reflection->GetRepeatedInt32(message, field, i)));
1087  break;
1088  case google::protobuf::FieldDescriptor::TYPE_UINT32:
1089  case google::protobuf::FieldDescriptor::TYPE_FIXED32:
1090  array.values.push_back(JSON::Number(
1091  reflection->GetRepeatedUInt32(message, field, i)));
1092  break;
1093  case google::protobuf::FieldDescriptor::TYPE_BOOL:
1094  if (reflection->GetRepeatedBool(message, field, i)) {
1095  array.values.push_back(JSON::Boolean(true));
1096  } else {
1097  array.values.push_back(JSON::Boolean(false));
1098  }
1099  break;
1100  case google::protobuf::FieldDescriptor::TYPE_STRING:
1101  array.values.push_back(JSON::String(
1102  reflection->GetRepeatedString(message, field, i)));
1103  break;
1104  case google::protobuf::FieldDescriptor::TYPE_BYTES:
1105  array.values.push_back(JSON::String(base64::encode(
1106  reflection->GetRepeatedString(message, field, i))));
1107  break;
1108  case google::protobuf::FieldDescriptor::TYPE_MESSAGE:
1109  array.values.push_back(protobuf(
1110  reflection->GetRepeatedMessage(message, field, i)));
1111  break;
1112  case google::protobuf::FieldDescriptor::TYPE_ENUM:
1113  array.values.push_back(JSON::String(
1114  reflection->GetRepeatedEnum(message, field, i)->name()));
1115  break;
1116  case google::protobuf::FieldDescriptor::TYPE_GROUP:
1117  // Deprecated! We abort here instead of using a Try as return value,
1118  // because we expect this code path to never be taken.
1119  ABORT("Unhandled protobuf field type: " +
1120  stringify(field->type()));
1121  }
1122  }
1123  object.values[field->name()] = array;
1124  } else {
1125  object.values[field->name()] = value_for_field(message, field);
1126  }
1127  }
1128 
1129  return object;
1130 }
1131 
1132 
1133 template <typename T>
1134 Array protobuf(const google::protobuf::RepeatedPtrField<T>& repeated)
1135 {
1136  static_assert(std::is_convertible<T*, google::protobuf::Message*>::value,
1137  "T must be a protobuf message");
1138 
1139  JSON::Array array;
1140  array.values.reserve(repeated.size());
1141  foreach (const T& elem, repeated) {
1142  array.values.emplace_back(JSON::protobuf(elem));
1143  }
1144 
1145  return array;
1146 }
1147 
1148 } // namespace JSON {
1149 
1150 #endif // __STOUT_PROTOBUF_HPP__
T as() const
Definition: json.hpp:120
Try< std::string > decode(const std::string &s)
Decode a string that is Base64-encoded with the standard Base64 alphabet.
Definition: base64.hpp:183
Try< Nothing > operator()(const JSON::Object &object) const
Definition: protobuf.hpp:423
Definition: path.hpp:26
bool isNone() const
Definition: result.hpp:112
Definition: nothing.hpp:16
Definition: errorbase.hpp:36
#define ABORT(...)
Definition: abort.hpp:40
Try< Bytes > size(const std::string &path, const FollowSymlink follow=FollowSymlink::FOLLOW_SYMLINK)
Definition: stat.hpp:121
const mode_t S_IRGRP
Definition: windows.hpp:313
T & get()&
Definition: try.hpp:73
ssize_t write(const int_fd &fd, const void *data, size_t size)
Definition: write.hpp:72
Result< Classifier > decode(const Netlink< struct rtnl_cls > &cls)
Try< Nothing > operator()(const JSON::Number &number) const
Definition: protobuf.hpp:573
Definition: protobuf.hpp:710
Definition: json.hpp:231
Definition: check.hpp:33
static Result< T > error(const std::string &message)
Definition: result.hpp:53
Try< google::protobuf::RepeatedPtrField< T > > operator()(const JSON::Value &value)
Definition: protobuf.hpp:747
const mode_t S_IWUSR
Definition: windows.hpp:306
Result< google::protobuf::RepeatedPtrField< T > > operator()(int_fd fd, bool ignorePartial, bool undoFailed)
Definition: protobuf.hpp:344
Definition: json.hpp:207
Try< T > operator()(const JSON::Value &value)
Definition: protobuf.hpp:712
Try< int_fd > open(const std::string &path, int oflag, mode_t mode=0)
Definition: open.hpp:35
Definition: protobuf.hpp:415
std::string encode(const std::string &s)
Encode a string to Base64 with the standard Base64 alphabet.
Definition: base64.hpp:170
Try< Nothing > operator()(const JSON::String &string) const
Definition: protobuf.hpp:484
Definition: json.hpp:198
const mode_t S_IRUSR
Definition: windows.hpp:305
Try< std::string > serialize(const T &t)
Definition: protobuf.hpp:223
std::map< std::string, Value > values
Definition: json.hpp:194
Definition: check.hpp:30
Definition: json.hpp:158
bool is() const
Definition: json.hpp:338
Try< Nothing > write(int_fd fd, const google::protobuf::Message &message)
Definition: protobuf.hpp:71
constexpr int O_CLOEXEC
Definition: open.hpp:41
Try< Nothing > operator()(const JSON::Array &array) const
Definition: protobuf.hpp:631
void field(const std::string &key, const T &value)
Definition: jsonify.hpp:346
Definition: protobuf.hpp:242
Try< Nothing > close(int fd)
Definition: close.hpp:24
Try< Nothing > append(const std::string &path, const google::protobuf::Message &message, bool sync=false)
Definition: protobuf.hpp:170
Try< off_t > lseek(int_fd fd, off_t offset, int whence)
Definition: lseek.hpp:25
Option< T > max(const Option< T > &left, const Option< T > &right)
Definition: option.hpp:208
Try< Nothing > operator()(const JSON::Boolean &boolean) const
Definition: protobuf.hpp:650
const T & as() const &
Definition: json.hpp:353
Definition: jsonify.hpp:325
Try< T > deserialize(const std::string &value)
Definition: protobuf.hpp:202
void json(ObjectWriter *writer, const Protobuf &protobuf)
Definition: protobuf.hpp:806
#define foreachpair(KEY, VALUE, ELEMS)
Definition: foreach.hpp:51
std::vector< Value > values
Definition: json.hpp:203
const T & get() const
Definition: result.hpp:115
Result< T > read(int_fd fd, bool ignorePartial=false, bool undoFailed=false)
Definition: protobuf.hpp:376
Definition: protobuf.hpp:60
Result< T > operator()(int_fd fd, bool ignorePartial, bool undoFailed)
Definition: protobuf.hpp:244
static Try error(const E &e)
Definition: try.hpp:42
JSON::Proxy jsonify(const T &)
Definition: jsonify.hpp:701
#define UNREACHABLE()
Definition: unreachable.hpp:22
Try< Nothing > fsync(int fd)
Definition: fsync.hpp:29
Definition: json.hpp:247
Result< std::string > read(int_fd fd, size_t size)
Definition: read.hpp:55
Parser(google::protobuf::Message *_message, const google::protobuf::FieldDescriptor *_field)
Definition: protobuf.hpp:417
Try< Nothing > operator()(const JSON::Null &) const
Definition: protobuf.hpp:667
Iterable< V > map(F &&f, const Iterable< U, Us... > &input)
Definition: lambda.hpp:46
Definition: protobuf.hpp:797
Try< Nothing > parse(google::protobuf::Message *message, const JSON::Object &object)
Definition: protobuf.hpp:682
Definition: none.hpp:27
Definition: attributes.hpp:24
bool isError() const
Definition: try.hpp:71
Definition: json.hpp:79
Object protobuf(const google::protobuf::Message &message)
Definition: protobuf.hpp:941
Definition: json.hpp:93
bool isError() const
Definition: result.hpp:113
Type utilities for the protobuf library that are not specific to particular protobuf classes...
Definition: type_utils.hpp:500
int int_fd
Definition: int_fd.hpp:35
std::string stringify(int flags)
Array protobuf(const google::protobuf::RepeatedPtrField< T > &repeated)
Definition: protobuf.hpp:1134
const mode_t S_IROTH
Definition: windows.hpp:321
constexpr const char * name
Definition: shell.hpp:43
Definition: json.hpp:51
Definition: representation.hpp:72
Definition: jsonify.hpp:295
Try< int > dup(int fd)
Definition: dup.hpp:23