Apache Mesos
protobuf.hpp
Go to the documentation of this file.
1 // Licensed under the Apache License, Version 2.0 (the "License");
2 // you may not use this file except in compliance with the License.
3 // You may obtain a copy of the License at
4 //
5 // http://www.apache.org/licenses/LICENSE-2.0
6 //
7 // Unless required by applicable law or agreed to in writing, software
8 // distributed under the License is distributed on an "AS IS" BASIS,
9 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 // See the License for the specific language governing permissions and
11 // limitations under the License.
12 
13 #ifndef __STOUT_PROTOBUF_HPP__
14 #define __STOUT_PROTOBUF_HPP__
15 
16 #include <assert.h>
17 #include <errno.h>
18 #include <stdint.h>
19 #ifndef __WINDOWS__
20 #include <unistd.h>
21 #endif // __WINDOWS__
22 
23 #include <sys/types.h>
24 
25 #include <string>
26 #include <type_traits>
27 #include <vector>
28 
29 #include <google/protobuf/descriptor.h>
30 #include <google/protobuf/descriptor.pb.h>
31 #include <google/protobuf/message.h>
32 #include <google/protobuf/repeated_field.h>
33 
34 #include <google/protobuf/io/zero_copy_stream_impl.h>
35 
36 #include <stout/abort.hpp>
37 #include <stout/base64.hpp>
38 #include <stout/error.hpp>
39 #include <stout/json.hpp>
40 #include <stout/jsonify.hpp>
41 #include <stout/none.hpp>
42 #include <stout/nothing.hpp>
43 #include <stout/representation.hpp>
44 #include <stout/result.hpp>
45 #include <stout/stringify.hpp>
46 #include <stout/try.hpp>
47 
48 #include <stout/os/close.hpp>
49 #include <stout/os/int_fd.hpp>
50 #include <stout/os/lseek.hpp>
51 #include <stout/os/open.hpp>
52 #include <stout/os/read.hpp>
53 #include <stout/os/write.hpp>
54 
55 #ifdef __WINDOWS__
56 #include <stout/os/dup.hpp>
57 #endif // __WINDOWS__
58 
59 namespace protobuf {
60 
61 // TODO(bmahler): Re-use stout's 'recordio' facilities here. Note
62 // that these use a fixed size length header, whereas stout's
63 // currently uses a base-10 newline delimited header for language
64 // portability, which makes changing these a bit tricky.
65 
66 // Write out the given protobuf to the specified file descriptor by
67 // first writing out the length of the protobuf followed by the
68 // contents.
69 // NOTE: On error, this may have written partial data to the file.
70 inline Try<Nothing> write(int_fd fd, const google::protobuf::Message& message)
71 {
72  if (!message.IsInitialized()) {
73  return Error(message.InitializationErrorString() +
74  " is required but not initialized");
75  }
76 
77  // First write the size of the protobuf.
78  uint32_t size = message.ByteSize();
79  std::string bytes((char*) &size, sizeof(size));
80 
81  Try<Nothing> result = os::write(fd, bytes);
82  if (result.isError()) {
83  return Error("Failed to write size: " + result.error());
84  }
85 
86 #ifdef __WINDOWS__
87  // NOTE: On Windows, we need to explicitly allocate a CRT file
88  // descriptor because the Protobuf library requires it. Because
89  // users of `protobuf::write` are likely to call `os::close` on the
90  // `fd` we were given, we need to duplicate it before allocating the
91  // CRT fd. This is because once the CRT fd is allocated, it must be
92  // closed with `_close` instead of `os::close`. Since we need to
93  // call `_close` here, we duplicate the fd to prevent the users call
94  // of `os::close` from closing twice.
95  Try<int_fd> dup = os::dup(fd);
96  if (dup.isError()) {
97  return Error("Failed to duplicate handle: " + dup.error());
98  }
99 
100  int crt = dup->crt();
101 
102  if (!message.SerializeToFileDescriptor(crt)) {
103  ::_close(crt);
104  return Error("Failed to write/serialize message");
105  }
106  ::_close(crt);
107 #else
108  if (!message.SerializeToFileDescriptor(fd)) {
109  return Error("Failed to write/serialize message");
110  }
111 #endif
112 
113  return Nothing();
114 }
115 
116 // Write out the given sequence of protobuf messages to the
117 // specified file descriptor by repeatedly invoking write
118 // on each of the messages.
119 // NOTE: On error, this may have written partial data to the file.
120 template <typename T>
122  int_fd fd, const google::protobuf::RepeatedPtrField<T>& messages)
123 {
124  foreach (const T& message, messages) {
125  Try<Nothing> result = write(fd, message);
126  if (result.isError()) {
127  return Error(result.error());
128  }
129  }
130 
131  return Nothing();
132 }
133 
134 
135 template <typename T>
136 Try<Nothing> write(const std::string& path, const T& t)
137 {
138  Try<int_fd> fd = os::open(
139  path,
140  O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC,
142 
143  if (fd.isError()) {
144  return Error("Failed to open file '" + path + "': " + fd.error());
145  }
146 
147  Try<Nothing> result = write(fd.get(), t);
148 
149  // NOTE: We ignore the return value of close(). This is because
150  // users calling this function are interested in the return value of
151  // write(). Also an unsuccessful close() doesn't affect the write.
152  os::close(fd.get());
153 
154  return result;
155 }
156 
157 
159  const std::string& path,
160  const google::protobuf::Message& message)
161 {
162  Try<int_fd> fd = os::open(
163  path,
164  O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC,
166 
167  if (fd.isError()) {
168  return Error("Failed to open file '" + path + "': " + fd.error());
169  }
170 
171  Try<Nothing> result = write(fd.get(), message);
172 
173  // NOTE: We ignore the return value of close(). This is because
174  // users calling this function are interested in the return value of
175  // write(). Also an unsuccessful close() doesn't affect the write.
176  os::close(fd.get());
177 
178  return result;
179 }
180 
181 
182 template <typename T>
183 Try<T> deserialize(const std::string& value)
184 {
185  T t;
186  (void) static_cast<google::protobuf::Message*>(&t);
187 
188  // Verify that the size of `value` fits into `ArrayInputStream`'s
189  // constructor. The maximum size of a proto2 message is 64 MB, so it is
190  // unlikely that we will hit this limit, but since an arbitrary string can be
191  // passed in, we include this check to be safe.
192  CHECK_LE(value.size(), static_cast<size_t>(std::numeric_limits<int>::max()));
193  google::protobuf::io::ArrayInputStream stream(
194  value.data(),
195  static_cast<int>(value.size()));
196  if (!t.ParseFromZeroCopyStream(&stream)) {
197  return Error("Failed to deserialize " + t.GetDescriptor()->full_name());
198  }
199  return t;
200 }
201 
202 
203 template <typename T>
205 {
206  (void) static_cast<const google::protobuf::Message*>(&t);
207 
208  std::string value;
209  if (!t.SerializeToString(&value)) {
210  return Error("Failed to serialize " + t.GetDescriptor()->full_name());
211  }
212  return value;
213 }
214 
215 
216 namespace internal {
217 
218 // Reads a single message of type T from the file by first reading the
219 // "size" followed by the contents (as written by 'write' above).
220 // NOTE: This struct is used by the public 'read' function.
221 // See comments there for the reason why we need this.
222 template <typename T>
223 struct Read
224 {
225  Result<T> operator()(int_fd fd, bool ignorePartial, bool undoFailed)
226  {
227  off_t offset = 0;
228 
229  if (undoFailed) {
230  // Save the offset so we can re-adjust if something goes wrong.
231  Try<off_t> lseek = os::lseek(fd, offset, SEEK_CUR);
232  if (lseek.isError()) {
233  return Error(lseek.error());
234  }
235 
236  offset = lseek.get();
237  }
238 
239  uint32_t size;
240  Result<std::string> result = os::read(fd, sizeof(size));
241 
242  if (result.isError()) {
243  if (undoFailed) {
244  os::lseek(fd, offset, SEEK_SET);
245  }
246  return Error("Failed to read size: " + result.error());
247  } else if (result.isNone()) {
248  return None(); // No more protobufs to read.
249  } else if (result->size() < sizeof(size)) {
250  // Hit EOF unexpectedly.
251  if (undoFailed) {
252  // Restore the offset to before the size read.
253  os::lseek(fd, offset, SEEK_SET);
254  }
255  if (ignorePartial) {
256  return None();
257  }
258  return Error(
259  "Failed to read size: hit EOF unexpectedly, possible corruption");
260  }
261 
262  // Parse the size from the bytes.
263  memcpy((void*)&size, (void*)result->data(), sizeof(size));
264 
265  // NOTE: Instead of specifically checking for corruption in 'size',
266  // we simply try to read 'size' bytes. If we hit EOF early, it is an
267  // indication of corruption.
268  result = os::read(fd, size);
269 
270  if (result.isError()) {
271  if (undoFailed) {
272  // Restore the offset to before the size read.
273  os::lseek(fd, offset, SEEK_SET);
274  }
275  return Error("Failed to read message: " + result.error());
276  } else if (result.isNone() || result->size() < size) {
277  // Hit EOF unexpectedly.
278  if (undoFailed) {
279  // Restore the offset to before the size read.
280  os::lseek(fd, offset, SEEK_SET);
281  }
282  if (ignorePartial) {
283  return None();
284  }
285  return Error("Failed to read message of size " + stringify(size) +
286  " bytes: hit EOF unexpectedly, possible corruption");
287  }
288 
289  // Parse the protobuf from the string.
290  // NOTE: We need to capture a const reference to the data because it
291  // must outlive the creation of ArrayInputStream.
292  const std::string& data = result.get();
293 
294  // Verify that the size of `data` fits into `ArrayInputStream`'s
295  // constructor. The maximum size of a proto2 message is 64 MB, so it is
296  // unlikely that we will hit this limit, but since an arbitrary string can
297  // be passed in, we include this check to be safe.
298  CHECK_LE(data.size(), static_cast<size_t>(std::numeric_limits<int>::max()));
299  T message;
300  google::protobuf::io::ArrayInputStream stream(
301  data.data(),
302  static_cast<int>(data.size()));
303 
304  if (!message.ParseFromZeroCopyStream(&stream)) {
305  if (undoFailed) {
306  // Restore the offset to before the size read.
307  os::lseek(fd, offset, SEEK_SET);
308  }
309  return Error("Failed to deserialize message");
310  }
311 
312  return message;
313  }
314 };
315 
316 
317 // Partial specialization for RepeatedPtrField<T> to read a sequence
318 // of protobuf messages from a given fd by repeatedly invoking
319 // Read<T> until None is reached, which we treat as EOF.
320 // NOTE: This struct is used by the public 'read' function.
321 // See comments there for the reason why we need this.
322 template <typename T>
323 struct Read<google::protobuf::RepeatedPtrField<T>>
324 {
326  int_fd fd, bool ignorePartial, bool undoFailed)
327  {
328  google::protobuf::RepeatedPtrField<T> result;
329  for (;;) {
330  Result<T> message = Read<T>()(fd, ignorePartial, undoFailed);
331  if (message.isError()) {
332  return Error(message.error());
333  } else if (message.isNone()) {
334  break;
335  } else {
336  result.Add()->CopyFrom(message.get());
337  }
338  }
339  return result;
340  }
341 };
342 
343 } // namespace internal {
344 
345 
346 // Reads the protobuf message(s) from a given fd based on the format
347 // written by write() above. We use partial specialization of
348 // - internal::Read<T> vs
349 // - internal::Read<google::protobuf::RepeatedPtrField<T>>
350 // in order to determine whether T is a single protobuf message or
351 // a sequence of messages.
352 // If 'ignorePartial' is true, None() is returned when we unexpectedly
353 // hit EOF while reading the protobuf (e.g., partial write).
354 // If 'undoFailed' is true, failed read attempts will restore the file
355 // read/write file offset towards the initial callup position.
356 template <typename T>
357 Result<T> read(int_fd fd, bool ignorePartial = false, bool undoFailed = false)
358 {
359  return internal::Read<T>()(fd, ignorePartial, undoFailed);
360 }
361 
362 
363 // A wrapper function that wraps the above read() with open and
364 // closing the file.
365 template <typename T>
366 Result<T> read(const std::string& path)
367 {
368  Try<int_fd> fd = os::open(
369  path,
370  O_RDONLY | O_CLOEXEC,
372 
373  if (fd.isError()) {
374  return Error("Failed to open file '" + path + "': " + fd.error());
375  }
376 
377  Result<T> result = read<T>(fd.get());
378 
379  // NOTE: We ignore the return value of close(). This is because
380  // users calling this function are interested in the return value of
381  // read(). Also an unsuccessful close() doesn't affect the read.
382  os::close(fd.get());
383 
384  return result;
385 }
386 
387 
388 namespace internal {
389 
390 // Forward declaration.
392  google::protobuf::Message* message,
393  const JSON::Object& object);
394 
395 
396 struct Parser : boost::static_visitor<Try<Nothing>>
397 {
398  Parser(google::protobuf::Message* _message,
399  const google::protobuf::FieldDescriptor* _field)
400  : message(_message),
401  reflection(message->GetReflection()),
402  field(_field) {}
403 
404  Try<Nothing> operator()(const JSON::Object& object) const
405  {
406  switch (field->type()) {
407  case google::protobuf::FieldDescriptor::TYPE_MESSAGE:
408  if (field->is_map()) {
409  foreachpair (
410  const std::string& name,
411  const JSON::Value& value,
412  object.values) {
413  google::protobuf::Message* entry =
414  reflection->AddMessage(message, field);
415 
416  // A map is equivalent to:
417  //
418  // message MapFieldEntry {
419  // optional key_type key = 1;
420  // optional value_type value = 2;
421  // }
422  //
423  // repeated MapFieldEntry map_field = N;
424  //
425  // See the link below for details:
426  // https://developers.google.com/protocol-buffers/docs/proto#maps
427  const google::protobuf::FieldDescriptor* key_field =
428  entry->GetDescriptor()->FindFieldByNumber(1);
429 
430  JSON::Value key(name);
431 
432  Try<Nothing> apply =
433  boost::apply_visitor(Parser(entry, key_field), key);
434 
435  if (apply.isError()) {
436  return Error(apply.error());
437  }
438 
439  const google::protobuf::FieldDescriptor* value_field =
440  entry->GetDescriptor()->FindFieldByNumber(2);
441 
442  apply = boost::apply_visitor(Parser(entry, value_field), value);
443  if (apply.isError()) {
444  return Error(apply.error());
445  }
446  }
447  } else if (field->is_repeated()) {
448  // TODO(gilbert): We currently push up the nested error
449  // messages without wrapping the error message (due to
450  // the recursive nature of parse). We should pass along
451  // variable information in order to construct a helpful
452  // error message, e.g. "Failed to parse field 'a.b.c': ...".
453  return parse(reflection->AddMessage(message, field), object);
454  } else {
455  return parse(reflection->MutableMessage(message, field), object);
456  }
457  break;
458  default:
459  return Error("Not expecting a JSON object for field '" +
460  field->name() + "'");
461  }
462  return Nothing();
463  }
464 
465  Try<Nothing> operator()(const JSON::String& string) const
466  {
467  switch (field->type()) {
468  case google::protobuf::FieldDescriptor::TYPE_STRING:
469  if (field->is_repeated()) {
470  reflection->AddString(message, field, string.value);
471  } else {
472  reflection->SetString(message, field, string.value);
473  }
474  break;
475  case google::protobuf::FieldDescriptor::TYPE_BYTES: {
476  Try<std::string> decode = base64::decode(string.value);
477  if (decode.isError()) {
478  return Error("Failed to base64 decode bytes field"
479  " '" + field->name() + "': " + decode.error());
480  }
481 
482  if (field->is_repeated()) {
483  reflection->AddString(message, field, decode.get());
484  } else {
485  reflection->SetString(message, field, decode.get());
486  }
487  break;
488  }
489  case google::protobuf::FieldDescriptor::TYPE_ENUM: {
490  const google::protobuf::EnumValueDescriptor* descriptor =
491  field->enum_type()->FindValueByName(string.value);
492 
493  if (descriptor == nullptr) {
494  if (field->is_required()) {
495  return Error("Failed to find enum for '" + string.value + "'");
496  }
497 
498  // Unrecognized enum value will be discarded if this is not a
499  // required enum field, which makes the field's `has..` accessor
500  // return false and its getter return the first value listed in
501  // the enum definition, or the default value if one is specified.
502  //
503  // This is the deserialization behavior of proto2, see the link
504  // below for details:
505  // https://developers.google.com/protocol-buffers/docs/proto#updating
506  break;
507  }
508 
509  if (field->is_repeated()) {
510  reflection->AddEnum(message, field, descriptor);
511  } else {
512  reflection->SetEnum(message, field, descriptor);
513  }
514  break;
515  }
516  case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
517  case google::protobuf::FieldDescriptor::TYPE_FLOAT:
518  case google::protobuf::FieldDescriptor::TYPE_INT64:
519  case google::protobuf::FieldDescriptor::TYPE_SINT64:
520  case google::protobuf::FieldDescriptor::TYPE_SFIXED64:
521  case google::protobuf::FieldDescriptor::TYPE_UINT64:
522  case google::protobuf::FieldDescriptor::TYPE_FIXED64:
523  case google::protobuf::FieldDescriptor::TYPE_INT32:
524  case google::protobuf::FieldDescriptor::TYPE_SINT32:
525  case google::protobuf::FieldDescriptor::TYPE_SFIXED32:
526  case google::protobuf::FieldDescriptor::TYPE_UINT32:
527  case google::protobuf::FieldDescriptor::TYPE_FIXED32: {
528  Try<JSON::Number> number = JSON::parse<JSON::Number>(string.value);
529  if (number.isError()) {
530  return Error(
531  "Failed to parse '" + string.value + "' as a JSON number "
532  "for field '" + field->name() + "': " + number.error());
533  }
534 
535  return operator()(number.get());
536  }
537  case google::protobuf::FieldDescriptor::TYPE_BOOL: {
538  Try<JSON::Boolean> boolean = JSON::parse<JSON::Boolean>(string.value);
539  if (boolean.isError()) {
540  return Error(
541  "Failed to parse '" + string.value + "' as a JSON boolean "
542  "for field '" + field->name() + "': " + boolean.error());
543  }
544 
545  return operator()(boolean.get());
546  }
547  default:
548  return Error("Not expecting a JSON string for field '" +
549  field->name() + "'");
550  }
551  return Nothing();
552  }
553 
554  Try<Nothing> operator()(const JSON::Number& number) const
555  {
556  switch (field->type()) {
557  case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
558  if (field->is_repeated()) {
559  reflection->AddDouble(message, field, number.as<double>());
560  } else {
561  reflection->SetDouble(message, field, number.as<double>());
562  }
563  break;
564  case google::protobuf::FieldDescriptor::TYPE_FLOAT:
565  if (field->is_repeated()) {
566  reflection->AddFloat(message, field, number.as<float>());
567  } else {
568  reflection->SetFloat(message, field, number.as<float>());
569  }
570  break;
571  case google::protobuf::FieldDescriptor::TYPE_INT64:
572  case google::protobuf::FieldDescriptor::TYPE_SINT64:
573  case google::protobuf::FieldDescriptor::TYPE_SFIXED64:
574  if (field->is_repeated()) {
575  reflection->AddInt64(message, field, number.as<int64_t>());
576  } else {
577  reflection->SetInt64(message, field, number.as<int64_t>());
578  }
579  break;
580  case google::protobuf::FieldDescriptor::TYPE_UINT64:
581  case google::protobuf::FieldDescriptor::TYPE_FIXED64:
582  if (field->is_repeated()) {
583  reflection->AddUInt64(message, field, number.as<uint64_t>());
584  } else {
585  reflection->SetUInt64(message, field, number.as<uint64_t>());
586  }
587  break;
588  case google::protobuf::FieldDescriptor::TYPE_INT32:
589  case google::protobuf::FieldDescriptor::TYPE_SINT32:
590  case google::protobuf::FieldDescriptor::TYPE_SFIXED32:
591  if (field->is_repeated()) {
592  reflection->AddInt32(message, field, number.as<int32_t>());
593  } else {
594  reflection->SetInt32(message, field, number.as<int32_t>());
595  }
596  break;
597  case google::protobuf::FieldDescriptor::TYPE_UINT32:
598  case google::protobuf::FieldDescriptor::TYPE_FIXED32:
599  if (field->is_repeated()) {
600  reflection->AddUInt32(message, field, number.as<uint32_t>());
601  } else {
602  reflection->SetUInt32(message, field, number.as<uint32_t>());
603  }
604  break;
605  default:
606  return Error("Not expecting a JSON number for field '" +
607  field->name() + "'");
608  }
609  return Nothing();
610  }
611 
612  Try<Nothing> operator()(const JSON::Array& array) const
613  {
614  if (!field->is_repeated()) {
615  return Error("Not expecting a JSON array for field '" +
616  field->name() + "'");
617  }
618 
619  foreach (const JSON::Value& value, array.values) {
620  Try<Nothing> apply =
621  boost::apply_visitor(Parser(message, field), value);
622 
623  if (apply.isError()) {
624  return Error(apply.error());
625  }
626  }
627 
628  return Nothing();
629  }
630 
631  Try<Nothing> operator()(const JSON::Boolean& boolean) const
632  {
633  switch (field->type()) {
634  case google::protobuf::FieldDescriptor::TYPE_BOOL:
635  if (field->is_repeated()) {
636  reflection->AddBool(message, field, boolean.value);
637  } else {
638  reflection->SetBool(message, field, boolean.value);
639  }
640  break;
641  default:
642  return Error("Not expecting a JSON boolean for field '" +
643  field->name() + "'");
644  }
645  return Nothing();
646  }
647 
649  {
650  // We treat 'null' as an unset field. Note that we allow
651  // unset required fields here since the top-level parse
652  // function is responsible for checking 'IsInitialized'.
653  return Nothing();
654  }
655 
656 private:
657  google::protobuf::Message* message;
658  const google::protobuf::Reflection* reflection;
659  const google::protobuf::FieldDescriptor* field;
660 };
661 
662 
664  google::protobuf::Message* message,
665  const JSON::Object& object)
666 {
667  foreachpair (
668  const std::string& name, const JSON::Value& value, object.values) {
669  // Look for a field by this name.
670  const google::protobuf::FieldDescriptor* field =
671  message->GetDescriptor()->FindFieldByName(name);
672 
673  if (field != nullptr) {
674  Try<Nothing> apply =
675  boost::apply_visitor(Parser(message, field), value);
676 
677  if (apply.isError()) {
678  return Error(apply.error());
679  }
680  }
681  }
682 
683  return Nothing();
684 }
685 
686 
687 // Parses a single protobuf message of type T from a JSON::Object.
688 // NOTE: This struct is used by the public parse<T>() function below. See
689 // comments there for the reason why we opted for this design.
690 template <typename T>
691 struct Parse
692 {
694  {
695  static_assert(std::is_convertible<T*, google::protobuf::Message*>::value,
696  "T must be a protobuf message");
697 
698  const JSON::Object* object = boost::get<JSON::Object>(&value);
699  if (object == nullptr) {
700  return Error("Expecting a JSON object");
701  }
702 
703  T message;
704 
705  Try<Nothing> parse = internal::parse(&message, *object);
706  if (parse.isError()) {
707  return Error(parse.error());
708  }
709 
710  if (!message.IsInitialized()) {
711  return Error("Missing required fields: " +
712  message.InitializationErrorString());
713  }
714 
715  return message;
716  }
717 };
718 
719 
720 // Partial specialization for RepeatedPtrField<T> to parse a sequence of
721 // protobuf messages from a JSON::Array by repeatedly invoking Parse<T> to
722 // facilitate conversions like JSON::Array -> Resources.
723 // NOTE: This struct is used by the public parse<T>() function below. See
724 // comments there for the reason why we opted for this design.
725 template <typename T>
726 struct Parse<google::protobuf::RepeatedPtrField<T>>
727 {
729  const JSON::Value& value)
730  {
731  static_assert(std::is_convertible<T*, google::protobuf::Message*>::value,
732  "T must be a protobuf message");
733 
734  const JSON::Array* array = boost::get<JSON::Array>(&value);
735  if (array == nullptr) {
736  return Error("Expecting a JSON array");
737  }
738 
739  google::protobuf::RepeatedPtrField<T> collection;
740  collection.Reserve(static_cast<int>(array->values.size()));
741 
742  // Parse messages one by one and propagate an error if it happens.
743  foreach (const JSON::Value& elem, array->values) {
744  Try<T> message = Parse<T>()(elem);
745  if (message.isError()) {
746  return Error(message.error());
747  }
748 
749  collection.Add()->CopyFrom(message.get());
750  }
751 
752  return collection;
753  }
754 };
755 
756 } // namespace internal {
757 
758 // A dispatch wrapper which parses protobuf messages(s) from a given JSON value.
759 // We use partial specialization of
760 // - internal::Parse<T> for JSON::Object
761 // - internal::Parse<google::protobuf::RepeatedPtrField<T>> for JSON::Array
762 // to determine whether T is a single message or a sequence of messages.
763 // We cannot partially specialize function templates and overloaded function
764 // approach combined with std::enable_if is not that clean, hence we leverage
765 // partial specialization of class templates.
766 template <typename T>
767 Try<T> parse(const JSON::Value& value)
768 {
769  return internal::Parse<T>()(value);
770 }
771 
772 } // namespace protobuf {
773 
774 namespace JSON {
775 
776 // The representation of generic protobuf => JSON,
777 // e.g., `jsonify(JSON::Protobuf(message))`.
778 struct Protobuf : Representation<google::protobuf::Message>
779 {
781 };
782 
783 
784 // `json` function for protobuf messages. Refer to `jsonify.hpp` for details.
785 // TODO(mpark): This currently uses the default value for optional fields
786 // that are not deprecated, but we may want to revisit this decision.
787 inline void json(ObjectWriter* writer, const Protobuf& protobuf)
788 {
789  using google::protobuf::FieldDescriptor;
790 
791  const google::protobuf::Message& message = protobuf;
792 
793  const google::protobuf::Descriptor* descriptor = message.GetDescriptor();
794  const google::protobuf::Reflection* reflection = message.GetReflection();
795 
796  // We first look through all the possible fields to determine both the set
797  // fields __and__ the optional fields with a default that are not set.
798  // `Reflection::ListFields()` alone will only include set fields and
799  // is therefore insufficient.
800  int fieldCount = descriptor->field_count();
801  std::vector<const FieldDescriptor*> fields;
802  fields.reserve(fieldCount);
803  for (int i = 0; i < fieldCount; ++i) {
804  const FieldDescriptor* field = descriptor->field(i);
805  if (field->is_repeated()) {
806  if (reflection->FieldSize(message, field) > 0) {
807  // Has repeated field with members, output as JSON.
808  fields.push_back(field);
809  }
810  } else if (
811  reflection->HasField(message, field) ||
812  (field->has_default_value() && !field->options().deprecated())) {
813  // Field is set or has default, output as JSON.
814  fields.push_back(field);
815  }
816  }
817 
818  foreach (const FieldDescriptor* field, fields) {
819  if (field->is_repeated()) {
820  writer->field(
821  field->name(),
822  [&field, &reflection, &message](JSON::ArrayWriter* writer) {
823  int fieldSize = reflection->FieldSize(message, field);
824  for (int i = 0; i < fieldSize; ++i) {
825  switch (field->cpp_type()) {
826  case FieldDescriptor::CPPTYPE_BOOL:
827  writer->element(
828  reflection->GetRepeatedBool(message, field, i));
829  break;
830  case FieldDescriptor::CPPTYPE_INT32:
831  writer->element(
832  reflection->GetRepeatedInt32(message, field, i));
833  break;
834  case FieldDescriptor::CPPTYPE_INT64:
835  writer->element(
836  reflection->GetRepeatedInt64(message, field, i));
837  break;
838  case FieldDescriptor::CPPTYPE_UINT32:
839  writer->element(
840  reflection->GetRepeatedUInt32(message, field, i));
841  break;
842  case FieldDescriptor::CPPTYPE_UINT64:
843  writer->element(
844  reflection->GetRepeatedUInt64(message, field, i));
845  break;
846  case FieldDescriptor::CPPTYPE_FLOAT:
847  writer->element(
848  reflection->GetRepeatedFloat(message, field, i));
849  break;
850  case FieldDescriptor::CPPTYPE_DOUBLE:
851  writer->element(
852  reflection->GetRepeatedDouble(message, field, i));
853  break;
854  case FieldDescriptor::CPPTYPE_MESSAGE:
855  writer->element(Protobuf(
856  reflection->GetRepeatedMessage(message, field, i)));
857  break;
858  case FieldDescriptor::CPPTYPE_ENUM:
859  writer->element(
860  reflection->GetRepeatedEnum(message, field, i)->name());
861  break;
862  case FieldDescriptor::CPPTYPE_STRING:
863  const std::string& s = reflection->GetRepeatedStringReference(
864  message, field, i, nullptr);
865  if (field->type() == FieldDescriptor::TYPE_BYTES) {
866  writer->element(base64::encode(s));
867  } else {
868  writer->element(s);
869  }
870  break;
871  }
872  }
873  });
874  } else {
875  switch (field->cpp_type()) {
876  case FieldDescriptor::CPPTYPE_BOOL:
877  writer->field(field->name(), reflection->GetBool(message, field));
878  break;
879  case FieldDescriptor::CPPTYPE_INT32:
880  writer->field(field->name(), reflection->GetInt32(message, field));
881  break;
882  case FieldDescriptor::CPPTYPE_INT64:
883  writer->field(field->name(), reflection->GetInt64(message, field));
884  break;
885  case FieldDescriptor::CPPTYPE_UINT32:
886  writer->field(field->name(), reflection->GetUInt32(message, field));
887  break;
888  case FieldDescriptor::CPPTYPE_UINT64:
889  writer->field(field->name(), reflection->GetUInt64(message, field));
890  break;
891  case FieldDescriptor::CPPTYPE_FLOAT:
892  writer->field(field->name(), reflection->GetFloat(message, field));
893  break;
894  case FieldDescriptor::CPPTYPE_DOUBLE:
895  writer->field(field->name(), reflection->GetDouble(message, field));
896  break;
897  case FieldDescriptor::CPPTYPE_MESSAGE:
898  writer->field(
899  field->name(), Protobuf(reflection->GetMessage(message, field)));
900  break;
901  case FieldDescriptor::CPPTYPE_ENUM:
902  writer->field(
903  field->name(), reflection->GetEnum(message, field)->name());
904  break;
905  case FieldDescriptor::CPPTYPE_STRING:
906  const std::string& s = reflection->GetStringReference(
907  message, field, nullptr);
908  if (field->type() == FieldDescriptor::TYPE_BYTES) {
909  writer->field(field->name(), base64::encode(s));
910  } else {
911  writer->field(field->name(), s);
912  }
913  break;
914  }
915  }
916  }
917 }
918 
919 
920 // TODO(bmahler): This currently uses the default value for optional fields
921 // that are not deprecated, but we may want to revisit this decision.
922 inline Object protobuf(const google::protobuf::Message& message)
923 {
924  Object object;
925 
926  const google::protobuf::Descriptor* descriptor = message.GetDescriptor();
927  const google::protobuf::Reflection* reflection = message.GetReflection();
928 
929  auto value_for_field = [](
930  const google::protobuf::Message& message,
931  const google::protobuf::FieldDescriptor* field) -> JSON::Value {
932  const google::protobuf::Reflection* reflection = message.GetReflection();
933 
934  switch (field->type()) {
935  case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
936  return JSON::Number(reflection->GetDouble(message, field));
937  case google::protobuf::FieldDescriptor::TYPE_FLOAT:
938  return JSON::Number(reflection->GetFloat(message, field));
939  case google::protobuf::FieldDescriptor::TYPE_INT64:
940  case google::protobuf::FieldDescriptor::TYPE_SINT64:
941  case google::protobuf::FieldDescriptor::TYPE_SFIXED64:
942  return JSON::Number(reflection->GetInt64(message, field));
943  case google::protobuf::FieldDescriptor::TYPE_UINT64:
944  case google::protobuf::FieldDescriptor::TYPE_FIXED64:
945  return JSON::Number(reflection->GetUInt64(message, field));
946  case google::protobuf::FieldDescriptor::TYPE_INT32:
947  case google::protobuf::FieldDescriptor::TYPE_SINT32:
948  case google::protobuf::FieldDescriptor::TYPE_SFIXED32:
949  return JSON::Number(reflection->GetInt32(message, field));
950  case google::protobuf::FieldDescriptor::TYPE_UINT32:
951  case google::protobuf::FieldDescriptor::TYPE_FIXED32:
952  return JSON::Number(reflection->GetUInt32(message, field));
953  case google::protobuf::FieldDescriptor::TYPE_BOOL:
954  if (reflection->GetBool(message, field)) {
955  return JSON::Boolean(true);
956  } else {
957  return JSON::Boolean(false);
958  }
959  break;
960  case google::protobuf::FieldDescriptor::TYPE_STRING:
961  return JSON::String(reflection->GetString(message, field));
962  case google::protobuf::FieldDescriptor::TYPE_BYTES:
963  return JSON::String(
964  base64::encode(reflection->GetString(message, field)));
965  case google::protobuf::FieldDescriptor::TYPE_MESSAGE:
966  return protobuf(reflection->GetMessage(message, field));
967  case google::protobuf::FieldDescriptor::TYPE_ENUM:
968  return JSON::String(reflection->GetEnum(message, field)->name());
969  case google::protobuf::FieldDescriptor::TYPE_GROUP:
970  // Deprecated! We abort here instead of using a Try as return value,
971  // because we expect this code path to never be taken.
972  ABORT("Unhandled protobuf field type: " +
973  stringify(field->type()));
974  }
975 
976  UNREACHABLE();
977  };
978 
979  // We first look through all the possible fields to determine both
980  // the set fields _and_ the optional fields with a default that
981  // are not set. Reflection::ListFields() alone will only include
982  // set fields and is therefore insufficient.
983  std::vector<const google::protobuf::FieldDescriptor*> fields;
984  fields.reserve(descriptor->field_count());
985  for (int i = 0; i < descriptor->field_count(); i++) {
986  const google::protobuf::FieldDescriptor* field = descriptor->field(i);
987  if (field->is_repeated()) {
988  if (reflection->FieldSize(message, descriptor->field(i)) > 0) {
989  // Has repeated field with members, output as JSON.
990  fields.push_back(field);
991  }
992  } else if (
993  reflection->HasField(message, field) ||
994  (field->has_default_value() && !field->options().deprecated())) {
995  // Field is set or has default, output as JSON.
996  fields.push_back(field);
997  }
998  }
999 
1000  foreach (const google::protobuf::FieldDescriptor* field, fields) {
1001  if (field->is_map()) {
1002  JSON::Object map;
1003 
1004  int fieldSize = reflection->FieldSize(message, field);
1005  for (int i = 0; i < fieldSize; ++i) {
1006  const google::protobuf::Message& entry =
1007  reflection->GetRepeatedMessage(message, field, i);
1008 
1009  // A map is equivalent to:
1010  //
1011  // message MapFieldEntry {
1012  // optional key_type key = 1;
1013  // optional value_type value = 2;
1014  // }
1015  //
1016  // repeated MapFieldEntry map_field = N;
1017  //
1018  // See the link below for details:
1019  // https://developers.google.com/protocol-buffers/docs/proto#maps
1020  const google::protobuf::FieldDescriptor* key_field =
1021  entry.GetDescriptor()->FindFieldByNumber(1);
1022 
1023  const google::protobuf::FieldDescriptor* value_field =
1024  entry.GetDescriptor()->FindFieldByNumber(2);
1025 
1026  JSON::Value key = value_for_field(entry, key_field);
1027 
1028  std::string name;
1029  if (key.is<JSON::String>()) {
1030  name = key.as<JSON::String>().value;
1031  } else {
1032  name = jsonify(key);
1033  }
1034 
1035  map.values[name] = value_for_field(entry, value_field);
1036  }
1037  object.values[field->name()] = map;
1038  } else if (field->is_repeated()) {
1039  JSON::Array array;
1040  int fieldSize = reflection->FieldSize(message, field);
1041  array.values.reserve(fieldSize);
1042  for (int i = 0; i < fieldSize; ++i) {
1043  switch (field->type()) {
1044  case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
1045  array.values.push_back(JSON::Number(
1046  reflection->GetRepeatedDouble(message, field, i)));
1047  break;
1048  case google::protobuf::FieldDescriptor::TYPE_FLOAT:
1049  array.values.push_back(JSON::Number(
1050  reflection->GetRepeatedFloat(message, field, i)));
1051  break;
1052  case google::protobuf::FieldDescriptor::TYPE_INT64:
1053  case google::protobuf::FieldDescriptor::TYPE_SINT64:
1054  case google::protobuf::FieldDescriptor::TYPE_SFIXED64:
1055  array.values.push_back(JSON::Number(
1056  reflection->GetRepeatedInt64(message, field, i)));
1057  break;
1058  case google::protobuf::FieldDescriptor::TYPE_UINT64:
1059  case google::protobuf::FieldDescriptor::TYPE_FIXED64:
1060  array.values.push_back(JSON::Number(
1061  reflection->GetRepeatedUInt64(message, field, i)));
1062  break;
1063  case google::protobuf::FieldDescriptor::TYPE_INT32:
1064  case google::protobuf::FieldDescriptor::TYPE_SINT32:
1065  case google::protobuf::FieldDescriptor::TYPE_SFIXED32:
1066  array.values.push_back(JSON::Number(
1067  reflection->GetRepeatedInt32(message, field, i)));
1068  break;
1069  case google::protobuf::FieldDescriptor::TYPE_UINT32:
1070  case google::protobuf::FieldDescriptor::TYPE_FIXED32:
1071  array.values.push_back(JSON::Number(
1072  reflection->GetRepeatedUInt32(message, field, i)));
1073  break;
1074  case google::protobuf::FieldDescriptor::TYPE_BOOL:
1075  if (reflection->GetRepeatedBool(message, field, i)) {
1076  array.values.push_back(JSON::Boolean(true));
1077  } else {
1078  array.values.push_back(JSON::Boolean(false));
1079  }
1080  break;
1081  case google::protobuf::FieldDescriptor::TYPE_STRING:
1082  array.values.push_back(JSON::String(
1083  reflection->GetRepeatedString(message, field, i)));
1084  break;
1085  case google::protobuf::FieldDescriptor::TYPE_BYTES:
1086  array.values.push_back(JSON::String(base64::encode(
1087  reflection->GetRepeatedString(message, field, i))));
1088  break;
1089  case google::protobuf::FieldDescriptor::TYPE_MESSAGE:
1090  array.values.push_back(protobuf(
1091  reflection->GetRepeatedMessage(message, field, i)));
1092  break;
1093  case google::protobuf::FieldDescriptor::TYPE_ENUM:
1094  array.values.push_back(JSON::String(
1095  reflection->GetRepeatedEnum(message, field, i)->name()));
1096  break;
1097  case google::protobuf::FieldDescriptor::TYPE_GROUP:
1098  // Deprecated! We abort here instead of using a Try as return value,
1099  // because we expect this code path to never be taken.
1100  ABORT("Unhandled protobuf field type: " +
1101  stringify(field->type()));
1102  }
1103  }
1104  object.values[field->name()] = array;
1105  } else {
1106  object.values[field->name()] = value_for_field(message, field);
1107  }
1108  }
1109 
1110  return object;
1111 }
1112 
1113 
1114 template <typename T>
1115 Array protobuf(const google::protobuf::RepeatedPtrField<T>& repeated)
1116 {
1117  static_assert(std::is_convertible<T*, google::protobuf::Message*>::value,
1118  "T must be a protobuf message");
1119 
1120  JSON::Array array;
1121  array.values.reserve(repeated.size());
1122  foreach (const T& elem, repeated) {
1123  array.values.emplace_back(JSON::protobuf(elem));
1124  }
1125 
1126  return array;
1127 }
1128 
1129 } // namespace JSON {
1130 
1131 #endif // __STOUT_PROTOBUF_HPP__
T as() const
Definition: json.hpp:116
Try< std::string > decode(const std::string &s)
Decode a string that is Base64-encoded with the standard Base64 alphabet.
Definition: base64.hpp:183
Try< Nothing > operator()(const JSON::Object &object) const
Definition: protobuf.hpp:404
Definition: path.hpp:26
bool isNone() const
Definition: result.hpp:112
Definition: nothing.hpp:16
const T & as() const
Definition: json.hpp:340
Definition: errorbase.hpp:36
#define ABORT(...)
Definition: abort.hpp:40
Try< Bytes > size(const std::string &path, const FollowSymlink follow=FollowSymlink::FOLLOW_SYMLINK)
Definition: stat.hpp:121
const mode_t S_IRGRP
Definition: windows.hpp:313
T & get()&
Definition: try.hpp:73
ssize_t write(const int_fd &fd, const void *data, size_t size)
Definition: write.hpp:72
Result< Classifier > decode(const Netlink< struct rtnl_cls > &cls)
Try< Nothing > operator()(const JSON::Number &number) const
Definition: protobuf.hpp:554
Definition: protobuf.hpp:691
Definition: json.hpp:227
Definition: check.hpp:33
static Result< T > error(const std::string &message)
Definition: result.hpp:53
Try< google::protobuf::RepeatedPtrField< T > > operator()(const JSON::Value &value)
Definition: protobuf.hpp:728
const mode_t S_IWUSR
Definition: windows.hpp:306
Result< google::protobuf::RepeatedPtrField< T > > operator()(int_fd fd, bool ignorePartial, bool undoFailed)
Definition: protobuf.hpp:325
Definition: json.hpp:203
Try< T > operator()(const JSON::Value &value)
Definition: protobuf.hpp:693
Try< int_fd > open(const std::string &path, int oflag, mode_t mode=0)
Definition: open.hpp:35
Definition: protobuf.hpp:396
std::string encode(const std::string &s)
Encode a string to Base64 with the standard Base64 alphabet.
Definition: base64.hpp:170
Try< Nothing > operator()(const JSON::String &string) const
Definition: protobuf.hpp:465
Definition: json.hpp:194
const mode_t S_IRUSR
Definition: windows.hpp:305
Try< std::string > serialize(const T &t)
Definition: protobuf.hpp:204
std::map< std::string, Value > values
Definition: json.hpp:190
Definition: check.hpp:30
Definition: json.hpp:154
bool is() const
Definition: json.hpp:325
Try< Nothing > write(int_fd fd, const google::protobuf::Message &message)
Definition: protobuf.hpp:70
constexpr int O_CLOEXEC
Definition: open.hpp:41
Try< Nothing > operator()(const JSON::Array &array) const
Definition: protobuf.hpp:612
void field(const std::string &key, const T &value)
Definition: jsonify.hpp:435
Definition: protobuf.hpp:223
Try< Nothing > close(int fd)
Definition: close.hpp:24
Try< off_t > lseek(int_fd fd, off_t offset, int whence)
Definition: lseek.hpp:25
Option< T > max(const Option< T > &left, const Option< T > &right)
Definition: option.hpp:208
Try< Nothing > operator()(const JSON::Boolean &boolean) const
Definition: protobuf.hpp:631
Definition: jsonify.hpp:418
Try< T > deserialize(const std::string &value)
Definition: protobuf.hpp:183
void json(ObjectWriter *writer, const Protobuf &protobuf)
Definition: protobuf.hpp:787
#define foreachpair(KEY, VALUE, ELEMS)
Definition: foreach.hpp:51
std::vector< Value > values
Definition: json.hpp:199
const T & get() const
Definition: result.hpp:115
Result< T > read(int_fd fd, bool ignorePartial=false, bool undoFailed=false)
Definition: protobuf.hpp:357
Definition: protobuf.hpp:59
Result< T > operator()(int_fd fd, bool ignorePartial, bool undoFailed)
Definition: protobuf.hpp:225
static Try error(const E &e)
Definition: try.hpp:42
JSON::Proxy jsonify(const T &)
Definition: jsonify.hpp:779
#define UNREACHABLE()
Definition: unreachable.hpp:22
Definition: json.hpp:243
Result< std::string > read(int_fd fd, size_t size)
Definition: read.hpp:55
Parser(google::protobuf::Message *_message, const google::protobuf::FieldDescriptor *_field)
Definition: protobuf.hpp:398
Try< Nothing > operator()(const JSON::Null &) const
Definition: protobuf.hpp:648
Iterable< V > map(F &&f, const Iterable< U, Us... > &input)
Definition: lambda.hpp:46
Definition: protobuf.hpp:778
Try< Nothing > parse(google::protobuf::Message *message, const JSON::Object &object)
Definition: protobuf.hpp:663
Definition: none.hpp:27
Definition: attributes.hpp:24
bool isError() const
Definition: try.hpp:71
Definition: json.hpp:75
Object protobuf(const google::protobuf::Message &message)
Definition: protobuf.hpp:922
Definition: json.hpp:89
bool isError() const
Definition: result.hpp:113
Definition: protobuf.hpp:68
Try< Nothing > append(const std::string &path, const google::protobuf::Message &message)
Definition: protobuf.hpp:158
int int_fd
Definition: int_fd.hpp:35
std::string stringify(int flags)
Array protobuf(const google::protobuf::RepeatedPtrField< T > &repeated)
Definition: protobuf.hpp:1115
const mode_t S_IROTH
Definition: windows.hpp:321
constexpr const char * name
Definition: shell.hpp:43
Definition: json.hpp:47
Definition: representation.hpp:72
Definition: jsonify.hpp:384
Try< int > dup(int fd)
Definition: dup.hpp:23