Apache Mesos
protobuf.hpp
Go to the documentation of this file.
1 // Licensed under the Apache License, Version 2.0 (the "License");
2 // you may not use this file except in compliance with the License.
3 // You may obtain a copy of the License at
4 //
5 // http://www.apache.org/licenses/LICENSE-2.0
6 //
7 // Unless required by applicable law or agreed to in writing, software
8 // distributed under the License is distributed on an "AS IS" BASIS,
9 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 // See the License for the specific language governing permissions and
11 // limitations under the License.
12 
13 #ifndef __STOUT_PROTOBUF_HPP__
14 #define __STOUT_PROTOBUF_HPP__
15 
16 #include <assert.h>
17 #include <errno.h>
18 #include <stdint.h>
19 #ifndef __WINDOWS__
20 #include <unistd.h>
21 #endif // __WINDOWS__
22 
23 #include <sys/types.h>
24 
25 #include <string>
26 #include <type_traits>
27 #include <vector>
28 
29 #include <google/protobuf/descriptor.h>
30 #include <google/protobuf/descriptor.pb.h>
31 #include <google/protobuf/message.h>
32 #include <google/protobuf/repeated_field.h>
33 
34 #include <google/protobuf/io/zero_copy_stream_impl.h>
35 
36 #include <stout/abort.hpp>
37 #include <stout/base64.hpp>
38 #include <stout/error.hpp>
39 #include <stout/json.hpp>
40 #include <stout/jsonify.hpp>
41 #include <stout/none.hpp>
42 #include <stout/nothing.hpp>
43 #include <stout/representation.hpp>
44 #include <stout/result.hpp>
45 #include <stout/stringify.hpp>
46 #include <stout/try.hpp>
47 
48 #include <stout/os/close.hpp>
49 #include <stout/os/int_fd.hpp>
50 #include <stout/os/lseek.hpp>
51 #include <stout/os/open.hpp>
52 #include <stout/os/read.hpp>
53 #include <stout/os/write.hpp>
54 
55 namespace protobuf {
56 
57 // TODO(bmahler): Re-use stout's 'recordio' facilities here. Note
58 // that these use a fixed size length header, whereas stout's
59 // currently uses a base-10 newline delimited header for language
60 // portability, which makes changing these a bit tricky.
61 
62 // Write out the given protobuf to the specified file descriptor by
63 // first writing out the length of the protobuf followed by the
64 // contents.
65 // NOTE: On error, this may have written partial data to the file.
66 inline Try<Nothing> write(int_fd fd, const google::protobuf::Message& message)
67 {
68  if (!message.IsInitialized()) {
69  return Error(message.InitializationErrorString() +
70  " is required but not initialized");
71  }
72 
73  // First write the size of the protobuf.
74  uint32_t size = message.ByteSize();
75  std::string bytes((char*) &size, sizeof(size));
76 
77  Try<Nothing> result = os::write(fd, bytes);
78  if (result.isError()) {
79  return Error("Failed to write size: " + result.error());
80  }
81 
82 #ifdef __WINDOWS__
83  if (!message.SerializeToFileDescriptor(fd.crt())) {
84 #else
85  if (!message.SerializeToFileDescriptor(fd)) {
86 #endif
87  return Error("Failed to write/serialize message");
88  }
89 
90  return Nothing();
91 }
92 
93 // Write out the given sequence of protobuf messages to the
94 // specified file descriptor by repeatedly invoking write
95 // on each of the messages.
96 // NOTE: On error, this may have written partial data to the file.
97 template <typename T>
99  int_fd fd, const google::protobuf::RepeatedPtrField<T>& messages)
100 {
101  foreach (const T& message, messages) {
102  Try<Nothing> result = write(fd, message);
103  if (result.isError()) {
104  return Error(result.error());
105  }
106  }
107 
108  return Nothing();
109 }
110 
111 
112 template <typename T>
113 Try<Nothing> write(const std::string& path, const T& t)
114 {
115  Try<int_fd> fd = os::open(
116  path,
119 
120  if (fd.isError()) {
121  return Error("Failed to open file '" + path + "': " + fd.error());
122  }
123 
124  Try<Nothing> result = write(fd.get(), t);
125 
126  // NOTE: We ignore the return value of close(). This is because
127  // users calling this function are interested in the return value of
128  // write(). Also an unsuccessful close() doesn't affect the write.
129  os::close(fd.get());
130 
131  return result;
132 }
133 
134 
136  const std::string& path,
137  const google::protobuf::Message& message)
138 {
139  Try<int_fd> fd = os::open(
140  path,
143 
144  if (fd.isError()) {
145  return Error("Failed to open file '" + path + "': " + fd.error());
146  }
147 
148  Try<Nothing> result = write(fd.get(), message);
149 
150  // NOTE: We ignore the return value of close(). This is because
151  // users calling this function are interested in the return value of
152  // write(). Also an unsuccessful close() doesn't affect the write.
153  os::close(fd.get());
154 
155  return result;
156 }
157 
158 
159 template <typename T>
160 Try<T> deserialize(const std::string& value)
161 {
162  T t;
163  (void) static_cast<google::protobuf::Message*>(&t);
164 
165  // Verify that the size of `value` fits into `ArrayInputStream`'s
166  // constructor. The maximum size of a proto2 message is 64 MB, so it is
167  // unlikely that we will hit this limit, but since an arbitrary string can be
168  // passed in, we include this check to be safe.
169  CHECK_LE(value.size(), static_cast<size_t>(std::numeric_limits<int>::max()));
170  google::protobuf::io::ArrayInputStream stream(
171  value.data(),
172  static_cast<int>(value.size()));
173  if (!t.ParseFromZeroCopyStream(&stream)) {
174  return Error("Failed to deserialize " + t.GetDescriptor()->full_name());
175  }
176  return t;
177 }
178 
179 
180 template <typename T>
182 {
183  (void) static_cast<const google::protobuf::Message*>(&t);
184 
185  std::string value;
186  if (!t.SerializeToString(&value)) {
187  return Error("Failed to serialize " + t.GetDescriptor()->full_name());
188  }
189  return value;
190 }
191 
192 
193 namespace internal {
194 
195 // Reads a single message of type T from the file by first reading the
196 // "size" followed by the contents (as written by 'write' above).
197 // NOTE: This struct is used by the public 'read' function.
198 // See comments there for the reason why we need this.
199 template <typename T>
200 struct Read
201 {
202  Result<T> operator()(int_fd fd, bool ignorePartial, bool undoFailed)
203  {
204  off_t offset = 0;
205 
206  if (undoFailed) {
207  // Save the offset so we can re-adjust if something goes wrong.
208  Try<off_t> lseek = os::lseek(fd, offset, SEEK_CUR);
209  if (lseek.isError()) {
210  return Error(lseek.error());
211  }
212 
213  offset = lseek.get();
214  }
215 
216  uint32_t size;
217  Result<std::string> result = os::read(fd, sizeof(size));
218 
219  if (result.isError()) {
220  if (undoFailed) {
221  os::lseek(fd, offset, SEEK_SET);
222  }
223  return Error("Failed to read size: " + result.error());
224  } else if (result.isNone()) {
225  return None(); // No more protobufs to read.
226  } else if (result.get().size() < sizeof(size)) {
227  // Hit EOF unexpectedly.
228  if (undoFailed) {
229  // Restore the offset to before the size read.
230  os::lseek(fd, offset, SEEK_SET);
231  }
232  if (ignorePartial) {
233  return None();
234  }
235  return Error(
236  "Failed to read size: hit EOF unexpectedly, possible corruption");
237  }
238 
239  // Parse the size from the bytes.
240  memcpy((void*) &size, (void*) result.get().data(), sizeof(size));
241 
242  // NOTE: Instead of specifically checking for corruption in 'size',
243  // we simply try to read 'size' bytes. If we hit EOF early, it is an
244  // indication of corruption.
245  result = os::read(fd, size);
246 
247  if (result.isError()) {
248  if (undoFailed) {
249  // Restore the offset to before the size read.
250  os::lseek(fd, offset, SEEK_SET);
251  }
252  return Error("Failed to read message: " + result.error());
253  } else if (result.isNone() || result.get().size() < size) {
254  // Hit EOF unexpectedly.
255  if (undoFailed) {
256  // Restore the offset to before the size read.
257  os::lseek(fd, offset, SEEK_SET);
258  }
259  if (ignorePartial) {
260  return None();
261  }
262  return Error("Failed to read message of size " + stringify(size) +
263  " bytes: hit EOF unexpectedly, possible corruption");
264  }
265 
266  // Parse the protobuf from the string.
267  // NOTE: We need to capture a const reference to the data because it
268  // must outlive the creation of ArrayInputStream.
269  const std::string& data = result.get();
270 
271  // Verify that the size of `data` fits into `ArrayInputStream`'s
272  // constructor. The maximum size of a proto2 message is 64 MB, so it is
273  // unlikely that we will hit this limit, but since an arbitrary string can
274  // be passed in, we include this check to be safe.
275  CHECK_LE(data.size(), static_cast<size_t>(std::numeric_limits<int>::max()));
276  T message;
277  google::protobuf::io::ArrayInputStream stream(
278  data.data(),
279  static_cast<int>(data.size()));
280 
281  if (!message.ParseFromZeroCopyStream(&stream)) {
282  if (undoFailed) {
283  // Restore the offset to before the size read.
284  os::lseek(fd, offset, SEEK_SET);
285  }
286  return Error("Failed to deserialize message");
287  }
288 
289  return message;
290  }
291 };
292 
293 
294 // Partial specialization for RepeatedPtrField<T> to read a sequence
295 // of protobuf messages from a given fd by repeatedly invoking
296 // Read<T> until None is reached, which we treat as EOF.
297 // NOTE: This struct is used by the public 'read' function.
298 // See comments there for the reason why we need this.
299 template <typename T>
300 struct Read<google::protobuf::RepeatedPtrField<T>>
301 {
303  int_fd fd, bool ignorePartial, bool undoFailed)
304  {
305  google::protobuf::RepeatedPtrField<T> result;
306  for (;;) {
307  Result<T> message = Read<T>()(fd, ignorePartial, undoFailed);
308  if (message.isError()) {
309  return Error(message.error());
310  } else if (message.isNone()) {
311  break;
312  } else {
313  result.Add()->CopyFrom(message.get());
314  }
315  }
316  return result;
317  }
318 };
319 
320 } // namespace internal {
321 
322 
323 // Reads the protobuf message(s) from a given fd based on the format
324 // written by write() above. We use partial specialization of
325 // - internal::Read<T> vs
326 // - internal::Read<google::protobuf::RepeatedPtrField<T>>
327 // in order to determine whether T is a single protobuf message or
328 // a sequence of messages.
329 // If 'ignorePartial' is true, None() is returned when we unexpectedly
330 // hit EOF while reading the protobuf (e.g., partial write).
331 // If 'undoFailed' is true, failed read attempts will restore the file
332 // read/write file offset towards the initial callup position.
333 template <typename T>
334 Result<T> read(int_fd fd, bool ignorePartial = false, bool undoFailed = false)
335 {
336  return internal::Read<T>()(fd, ignorePartial, undoFailed);
337 }
338 
339 
340 // A wrapper function that wraps the above read() with open and
341 // closing the file.
342 template <typename T>
343 Try<T> read(const std::string& path)
344 {
345  Try<int_fd> fd = os::open(
346  path,
349 
350  if (fd.isError()) {
351  return Error("Failed to open file '" + path + "': " + fd.error());
352  }
353 
354  Result<T> result = read<T>(fd.get());
355 
356  // NOTE: We ignore the return value of close(). This is because
357  // users calling this function are interested in the return value of
358  // read(). Also an unsuccessful close() doesn't affect the read.
359  os::close(fd.get());
360 
361  if (result.isSome()) {
362  return result.get();
363  }
364 
365  // `read(fd)` returning `None` here means that the file is empty.
366  // Since this is a partial read of `T`, we report it as an error.
367  return Error(result.isError() ? result.error() : "Found an empty file");
368 }
369 
370 
371 namespace internal {
372 
373 // Forward declaration.
375  google::protobuf::Message* message,
376  const JSON::Object& object);
377 
378 
379 struct Parser : boost::static_visitor<Try<Nothing>>
380 {
381  Parser(google::protobuf::Message* _message,
382  const google::protobuf::FieldDescriptor* _field)
383  : message(_message),
384  reflection(message->GetReflection()),
385  field(_field) {}
386 
387  Try<Nothing> operator()(const JSON::Object& object) const
388  {
389  switch (field->type()) {
390  case google::protobuf::FieldDescriptor::TYPE_MESSAGE:
391  // TODO(gilbert): We currently push up the nested error
392  // messages without wrapping the error message (due to
393  // the recursive nature of parse). We should pass along
394  // variable information in order to construct a helpful
395  // error message, e.g. "Failed to parse field 'a.b.c': ...".
396  if (field->is_repeated()) {
397  return parse(reflection->AddMessage(message, field), object);
398  } else {
399  return parse(reflection->MutableMessage(message, field), object);
400  }
401  break;
402  default:
403  return Error("Not expecting a JSON object for field '" +
404  field->name() + "'");
405  }
406  return Nothing();
407  }
408 
409  Try<Nothing> operator()(const JSON::String& string) const
410  {
411  switch (field->type()) {
412  case google::protobuf::FieldDescriptor::TYPE_STRING:
413  if (field->is_repeated()) {
414  reflection->AddString(message, field, string.value);
415  } else {
416  reflection->SetString(message, field, string.value);
417  }
418  break;
419  case google::protobuf::FieldDescriptor::TYPE_BYTES: {
420  Try<std::string> decode = base64::decode(string.value);
421 
422  if (decode.isError()) {
423  return Error("Failed to base64 decode bytes field"
424  " '" + field->name() + "': " + decode.error());
425  }
426 
427  if (field->is_repeated()) {
428  reflection->AddString(message, field, decode.get());
429  } else {
430  reflection->SetString(message, field, decode.get());
431  }
432  break;
433  }
434  case google::protobuf::FieldDescriptor::TYPE_ENUM: {
435  const google::protobuf::EnumValueDescriptor* descriptor =
436  field->enum_type()->FindValueByName(string.value);
437 
438  if (descriptor == nullptr) {
439  if (field->is_required()) {
440  return Error("Failed to find enum for '" + string.value + "'");
441  }
442 
443  // Unrecognized enum value will be discarded if this is not a
444  // required enum field, which makes the field's `has..` accessor
445  // return false and its getter return the first value listed in
446  // the enum definition, or the default value if one is specified.
447  //
448  // This is the deserialization behavior of proto2, see the link
449  // below for details:
450  // https://developers.google.com/protocol-buffers/docs/proto#updating
451  break;
452  }
453 
454  if (field->is_repeated()) {
455  reflection->AddEnum(message, field, descriptor);
456  } else {
457  reflection->SetEnum(message, field, descriptor);
458  }
459  break;
460  }
461  default:
462  return Error("Not expecting a JSON string for field '" +
463  field->name() + "'");
464  }
465  return Nothing();
466  }
467 
468  Try<Nothing> operator()(const JSON::Number& number) const
469  {
470  switch (field->type()) {
471  case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
472  if (field->is_repeated()) {
473  reflection->AddDouble(message, field, number.as<double>());
474  } else {
475  reflection->SetDouble(message, field, number.as<double>());
476  }
477  break;
478  case google::protobuf::FieldDescriptor::TYPE_FLOAT:
479  if (field->is_repeated()) {
480  reflection->AddFloat(message, field, number.as<float>());
481  } else {
482  reflection->SetFloat(message, field, number.as<float>());
483  }
484  break;
485  case google::protobuf::FieldDescriptor::TYPE_INT64:
486  case google::protobuf::FieldDescriptor::TYPE_SINT64:
487  case google::protobuf::FieldDescriptor::TYPE_SFIXED64:
488  if (field->is_repeated()) {
489  reflection->AddInt64(message, field, number.as<int64_t>());
490  } else {
491  reflection->SetInt64(message, field, number.as<int64_t>());
492  }
493  break;
494  case google::protobuf::FieldDescriptor::TYPE_UINT64:
495  case google::protobuf::FieldDescriptor::TYPE_FIXED64:
496  if (field->is_repeated()) {
497  reflection->AddUInt64(message, field, number.as<uint64_t>());
498  } else {
499  reflection->SetUInt64(message, field, number.as<uint64_t>());
500  }
501  break;
502  case google::protobuf::FieldDescriptor::TYPE_INT32:
503  case google::protobuf::FieldDescriptor::TYPE_SINT32:
504  case google::protobuf::FieldDescriptor::TYPE_SFIXED32:
505  if (field->is_repeated()) {
506  reflection->AddInt32(message, field, number.as<int32_t>());
507  } else {
508  reflection->SetInt32(message, field, number.as<int32_t>());
509  }
510  break;
511  case google::protobuf::FieldDescriptor::TYPE_UINT32:
512  case google::protobuf::FieldDescriptor::TYPE_FIXED32:
513  if (field->is_repeated()) {
514  reflection->AddUInt32(message, field, number.as<uint32_t>());
515  } else {
516  reflection->SetUInt32(message, field, number.as<uint32_t>());
517  }
518  break;
519  default:
520  return Error("Not expecting a JSON number for field '" +
521  field->name() + "'");
522  }
523  return Nothing();
524  }
525 
526  Try<Nothing> operator()(const JSON::Array& array) const
527  {
528  if (!field->is_repeated()) {
529  return Error("Not expecting a JSON array for field '" +
530  field->name() + "'");
531  }
532 
533  foreach (const JSON::Value& value, array.values) {
534  Try<Nothing> apply =
535  boost::apply_visitor(Parser(message, field), value);
536 
537  if (apply.isError()) {
538  return Error(apply.error());
539  }
540  }
541 
542  return Nothing();
543  }
544 
545  Try<Nothing> operator()(const JSON::Boolean& boolean) const
546  {
547  switch (field->type()) {
548  case google::protobuf::FieldDescriptor::TYPE_BOOL:
549  if (field->is_repeated()) {
550  reflection->AddBool(message, field, boolean.value);
551  } else {
552  reflection->SetBool(message, field, boolean.value);
553  }
554  break;
555  default:
556  return Error("Not expecting a JSON boolean for field '" +
557  field->name() + "'");
558  }
559  return Nothing();
560  }
561 
563  {
564  // We treat 'null' as an unset field. Note that we allow
565  // unset required fields here since the top-level parse
566  // function is responsible for checking 'IsInitialized'.
567  return Nothing();
568  }
569 
570 private:
571  google::protobuf::Message* message;
572  const google::protobuf::Reflection* reflection;
573  const google::protobuf::FieldDescriptor* field;
574 };
575 
576 
578  google::protobuf::Message* message,
579  const JSON::Object& object)
580 {
581  foreachpair (
582  const std::string& name, const JSON::Value& value, object.values) {
583  // Look for a field by this name.
584  const google::protobuf::FieldDescriptor* field =
585  message->GetDescriptor()->FindFieldByName(name);
586 
587  if (field != nullptr) {
588  Try<Nothing> apply =
589  boost::apply_visitor(Parser(message, field), value);
590 
591  if (apply.isError()) {
592  return Error(apply.error());
593  }
594  }
595  }
596 
597  return Nothing();
598 }
599 
600 
601 // Parses a single protobuf message of type T from a JSON::Object.
602 // NOTE: This struct is used by the public parse<T>() function below. See
603 // comments there for the reason why we opted for this design.
604 template <typename T>
605 struct Parse
606 {
608  {
609  static_assert(std::is_convertible<T*, google::protobuf::Message*>::value,
610  "T must be a protobuf message");
611 
612  const JSON::Object* object = boost::get<JSON::Object>(&value);
613  if (object == nullptr) {
614  return Error("Expecting a JSON object");
615  }
616 
617  T message;
618 
619  Try<Nothing> parse = internal::parse(&message, *object);
620  if (parse.isError()) {
621  return Error(parse.error());
622  }
623 
624  if (!message.IsInitialized()) {
625  return Error("Missing required fields: " +
626  message.InitializationErrorString());
627  }
628 
629  return message;
630  }
631 };
632 
633 
634 // Partial specialization for RepeatedPtrField<T> to parse a sequence of
635 // protobuf messages from a JSON::Array by repeatedly invoking Parse<T> to
636 // facilitate conversions like JSON::Array -> Resources.
637 // NOTE: This struct is used by the public parse<T>() function below. See
638 // comments there for the reason why we opted for this design.
639 template <typename T>
640 struct Parse<google::protobuf::RepeatedPtrField<T>>
641 {
643  const JSON::Value& value)
644  {
645  static_assert(std::is_convertible<T*, google::protobuf::Message*>::value,
646  "T must be a protobuf message");
647 
648  const JSON::Array* array = boost::get<JSON::Array>(&value);
649  if (array == nullptr) {
650  return Error("Expecting a JSON array");
651  }
652 
653  google::protobuf::RepeatedPtrField<T> collection;
654  collection.Reserve(static_cast<int>(array->values.size()));
655 
656  // Parse messages one by one and propagate an error if it happens.
657  foreach (const JSON::Value& elem, array->values) {
658  Try<T> message = Parse<T>()(elem);
659  if (message.isError()) {
660  return Error(message.error());
661  }
662 
663  collection.Add()->CopyFrom(message.get());
664  }
665 
666  return collection;
667  }
668 };
669 
670 } // namespace internal {
671 
672 // A dispatch wrapper which parses protobuf messages(s) from a given JSON value.
673 // We use partial specialization of
674 // - internal::Parse<T> for JSON::Object
675 // - internal::Parse<google::protobuf::RepeatedPtrField<T>> for JSON::Array
676 // to determine whether T is a single message or a sequence of messages.
677 // We cannot partially specialize function templates and overloaded function
678 // approach combined with std::enable_if is not that clean, hence we leverage
679 // partial specialization of class templates.
680 template <typename T>
681 Try<T> parse(const JSON::Value& value)
682 {
683  return internal::Parse<T>()(value);
684 }
685 
686 } // namespace protobuf {
687 
688 namespace JSON {
689 
690 // The representation of generic protobuf => JSON,
691 // e.g., `jsonify(JSON::Protobuf(message))`.
692 struct Protobuf : Representation<google::protobuf::Message>
693 {
695 };
696 
697 
698 // `json` function for protobuf messages. Refer to `jsonify.hpp` for details.
699 // TODO(mpark): This currently uses the default value for optional fields
700 // that are not deprecated, but we may want to revisit this decision.
701 inline void json(ObjectWriter* writer, const Protobuf& protobuf)
702 {
703  using google::protobuf::FieldDescriptor;
704 
705  const google::protobuf::Message& message = protobuf;
706 
707  const google::protobuf::Descriptor* descriptor = message.GetDescriptor();
708  const google::protobuf::Reflection* reflection = message.GetReflection();
709 
710  // We first look through all the possible fields to determine both the set
711  // fields __and__ the optional fields with a default that are not set.
712  // `Reflection::ListFields()` alone will only include set fields and
713  // is therefore insufficient.
714  int fieldCount = descriptor->field_count();
715  std::vector<const FieldDescriptor*> fields;
716  fields.reserve(fieldCount);
717  for (int i = 0; i < fieldCount; ++i) {
718  const FieldDescriptor* field = descriptor->field(i);
719  if (field->is_repeated()) {
720  if (reflection->FieldSize(message, field) > 0) {
721  // Has repeated field with members, output as JSON.
722  fields.push_back(field);
723  }
724  } else if (
725  reflection->HasField(message, field) ||
726  (field->has_default_value() && !field->options().deprecated())) {
727  // Field is set or has default, output as JSON.
728  fields.push_back(field);
729  }
730  }
731 
732  foreach (const FieldDescriptor* field, fields) {
733  if (field->is_repeated()) {
734  writer->field(
735  field->name(),
736  [&field, &reflection, &message](JSON::ArrayWriter* writer) {
737  int fieldSize = reflection->FieldSize(message, field);
738  for (int i = 0; i < fieldSize; ++i) {
739  switch (field->cpp_type()) {
740  case FieldDescriptor::CPPTYPE_BOOL:
741  writer->element(
742  reflection->GetRepeatedBool(message, field, i));
743  break;
744  case FieldDescriptor::CPPTYPE_INT32:
745  writer->element(
746  reflection->GetRepeatedInt32(message, field, i));
747  break;
748  case FieldDescriptor::CPPTYPE_INT64:
749  writer->element(
750  reflection->GetRepeatedInt64(message, field, i));
751  break;
752  case FieldDescriptor::CPPTYPE_UINT32:
753  writer->element(
754  reflection->GetRepeatedUInt32(message, field, i));
755  break;
756  case FieldDescriptor::CPPTYPE_UINT64:
757  writer->element(
758  reflection->GetRepeatedUInt64(message, field, i));
759  break;
760  case FieldDescriptor::CPPTYPE_FLOAT:
761  writer->element(
762  reflection->GetRepeatedFloat(message, field, i));
763  break;
764  case FieldDescriptor::CPPTYPE_DOUBLE:
765  writer->element(
766  reflection->GetRepeatedDouble(message, field, i));
767  break;
768  case FieldDescriptor::CPPTYPE_MESSAGE:
769  writer->element(Protobuf(
770  reflection->GetRepeatedMessage(message, field, i)));
771  break;
772  case FieldDescriptor::CPPTYPE_ENUM:
773  writer->element(
774  reflection->GetRepeatedEnum(message, field, i)->name());
775  break;
776  case FieldDescriptor::CPPTYPE_STRING:
777  const std::string& s = reflection->GetRepeatedStringReference(
778  message, field, i, nullptr);
779  if (field->type() == FieldDescriptor::TYPE_BYTES) {
780  writer->element(base64::encode(s));
781  } else {
782  writer->element(s);
783  }
784  break;
785  }
786  }
787  });
788  } else {
789  switch (field->cpp_type()) {
790  case FieldDescriptor::CPPTYPE_BOOL:
791  writer->field(field->name(), reflection->GetBool(message, field));
792  break;
793  case FieldDescriptor::CPPTYPE_INT32:
794  writer->field(field->name(), reflection->GetInt32(message, field));
795  break;
796  case FieldDescriptor::CPPTYPE_INT64:
797  writer->field(field->name(), reflection->GetInt64(message, field));
798  break;
799  case FieldDescriptor::CPPTYPE_UINT32:
800  writer->field(field->name(), reflection->GetUInt32(message, field));
801  break;
802  case FieldDescriptor::CPPTYPE_UINT64:
803  writer->field(field->name(), reflection->GetUInt64(message, field));
804  break;
805  case FieldDescriptor::CPPTYPE_FLOAT:
806  writer->field(field->name(), reflection->GetFloat(message, field));
807  break;
808  case FieldDescriptor::CPPTYPE_DOUBLE:
809  writer->field(field->name(), reflection->GetDouble(message, field));
810  break;
811  case FieldDescriptor::CPPTYPE_MESSAGE:
812  writer->field(
813  field->name(), Protobuf(reflection->GetMessage(message, field)));
814  break;
815  case FieldDescriptor::CPPTYPE_ENUM:
816  writer->field(
817  field->name(), reflection->GetEnum(message, field)->name());
818  break;
819  case FieldDescriptor::CPPTYPE_STRING:
820  const std::string& s = reflection->GetStringReference(
821  message, field, nullptr);
822  if (field->type() == FieldDescriptor::TYPE_BYTES) {
823  writer->field(field->name(), base64::encode(s));
824  } else {
825  writer->field(field->name(), s);
826  }
827  break;
828  }
829  }
830  }
831 }
832 
833 
834 // TODO(bmahler): This currently uses the default value for optional fields
835 // that are not deprecated, but we may want to revisit this decision.
836 inline Object protobuf(const google::protobuf::Message& message)
837 {
838  Object object;
839 
840  const google::protobuf::Descriptor* descriptor = message.GetDescriptor();
841  const google::protobuf::Reflection* reflection = message.GetReflection();
842 
843  // We first look through all the possible fields to determine both
844  // the set fields _and_ the optional fields with a default that
845  // are not set. Reflection::ListFields() alone will only include
846  // set fields and is therefore insufficient.
847  std::vector<const google::protobuf::FieldDescriptor*> fields;
848  fields.reserve(descriptor->field_count());
849  for (int i = 0; i < descriptor->field_count(); i++) {
850  const google::protobuf::FieldDescriptor* field = descriptor->field(i);
851  if (field->is_repeated()) {
852  if (reflection->FieldSize(message, descriptor->field(i)) > 0) {
853  // Has repeated field with members, output as JSON.
854  fields.push_back(field);
855  }
856  } else if (
857  reflection->HasField(message, field) ||
858  (field->has_default_value() && !field->options().deprecated())) {
859  // Field is set or has default, output as JSON.
860  fields.push_back(field);
861  }
862  }
863 
864  foreach (const google::protobuf::FieldDescriptor* field, fields) {
865  if (field->is_repeated()) {
866  JSON::Array array;
867  int fieldSize = reflection->FieldSize(message, field);
868  array.values.reserve(fieldSize);
869  for (int i = 0; i < fieldSize; ++i) {
870  switch (field->type()) {
871  case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
872  array.values.push_back(JSON::Number(
873  reflection->GetRepeatedDouble(message, field, i)));
874  break;
875  case google::protobuf::FieldDescriptor::TYPE_FLOAT:
876  array.values.push_back(JSON::Number(
877  reflection->GetRepeatedFloat(message, field, i)));
878  break;
879  case google::protobuf::FieldDescriptor::TYPE_INT64:
880  case google::protobuf::FieldDescriptor::TYPE_SINT64:
881  case google::protobuf::FieldDescriptor::TYPE_SFIXED64:
882  array.values.push_back(JSON::Number(
883  reflection->GetRepeatedInt64(message, field, i)));
884  break;
885  case google::protobuf::FieldDescriptor::TYPE_UINT64:
886  case google::protobuf::FieldDescriptor::TYPE_FIXED64:
887  array.values.push_back(JSON::Number(
888  reflection->GetRepeatedUInt64(message, field, i)));
889  break;
890  case google::protobuf::FieldDescriptor::TYPE_INT32:
891  case google::protobuf::FieldDescriptor::TYPE_SINT32:
892  case google::protobuf::FieldDescriptor::TYPE_SFIXED32:
893  array.values.push_back(JSON::Number(
894  reflection->GetRepeatedInt32(message, field, i)));
895  break;
896  case google::protobuf::FieldDescriptor::TYPE_UINT32:
897  case google::protobuf::FieldDescriptor::TYPE_FIXED32:
898  array.values.push_back(JSON::Number(
899  reflection->GetRepeatedUInt32(message, field, i)));
900  break;
901  case google::protobuf::FieldDescriptor::TYPE_BOOL:
902  if (reflection->GetRepeatedBool(message, field, i)) {
903  array.values.push_back(JSON::True());
904  } else {
905  array.values.push_back(JSON::False());
906  }
907  break;
908  case google::protobuf::FieldDescriptor::TYPE_STRING:
909  array.values.push_back(JSON::String(
910  reflection->GetRepeatedString(message, field, i)));
911  break;
912  case google::protobuf::FieldDescriptor::TYPE_BYTES:
913  array.values.push_back(JSON::String(base64::encode(
914  reflection->GetRepeatedString(message, field, i))));
915  break;
916  case google::protobuf::FieldDescriptor::TYPE_MESSAGE:
917  array.values.push_back(protobuf(
918  reflection->GetRepeatedMessage(message, field, i)));
919  break;
920  case google::protobuf::FieldDescriptor::TYPE_ENUM:
921  array.values.push_back(JSON::String(
922  reflection->GetRepeatedEnum(message, field, i)->name()));
923  break;
924  case google::protobuf::FieldDescriptor::TYPE_GROUP:
925  // Deprecated! We abort here instead of using a Try as return value,
926  // because we expect this code path to never be taken.
927  ABORT("Unhandled protobuf field type: " +
928  stringify(field->type()));
929  }
930  }
931  object.values[field->name()] = array;
932  } else {
933  switch (field->type()) {
934  case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
935  object.values[field->name()] =
936  JSON::Number(reflection->GetDouble(message, field));
937  break;
938  case google::protobuf::FieldDescriptor::TYPE_FLOAT:
939  object.values[field->name()] =
940  JSON::Number(reflection->GetFloat(message, field));
941  break;
942  case google::protobuf::FieldDescriptor::TYPE_INT64:
943  case google::protobuf::FieldDescriptor::TYPE_SINT64:
944  case google::protobuf::FieldDescriptor::TYPE_SFIXED64:
945  object.values[field->name()] =
946  JSON::Number(reflection->GetInt64(message, field));
947  break;
948  case google::protobuf::FieldDescriptor::TYPE_UINT64:
949  case google::protobuf::FieldDescriptor::TYPE_FIXED64:
950  object.values[field->name()] =
951  JSON::Number(reflection->GetUInt64(message, field));
952  break;
953  case google::protobuf::FieldDescriptor::TYPE_INT32:
954  case google::protobuf::FieldDescriptor::TYPE_SINT32:
955  case google::protobuf::FieldDescriptor::TYPE_SFIXED32:
956  object.values[field->name()] =
957  JSON::Number(reflection->GetInt32(message, field));
958  break;
959  case google::protobuf::FieldDescriptor::TYPE_UINT32:
960  case google::protobuf::FieldDescriptor::TYPE_FIXED32:
961  object.values[field->name()] =
962  JSON::Number(reflection->GetUInt32(message, field));
963  break;
964  case google::protobuf::FieldDescriptor::TYPE_BOOL:
965  if (reflection->GetBool(message, field)) {
966  object.values[field->name()] = JSON::True();
967  } else {
968  object.values[field->name()] = JSON::False();
969  }
970  break;
971  case google::protobuf::FieldDescriptor::TYPE_STRING:
972  object.values[field->name()] =
973  JSON::String(reflection->GetString(message, field));
974  break;
975  case google::protobuf::FieldDescriptor::TYPE_BYTES:
976  object.values[field->name()] = JSON::String(
977  base64::encode(reflection->GetString(message, field)));
978  break;
979  case google::protobuf::FieldDescriptor::TYPE_MESSAGE:
980  object.values[field->name()] =
981  protobuf(reflection->GetMessage(message, field));
982  break;
983  case google::protobuf::FieldDescriptor::TYPE_ENUM:
984  object.values[field->name()] =
985  JSON::String(reflection->GetEnum(message, field)->name());
986  break;
987  case google::protobuf::FieldDescriptor::TYPE_GROUP:
988  // Deprecated! We abort here instead of using a Try as return value,
989  // because we expect this code path to never be taken.
990  ABORT("Unhandled protobuf field type: " +
991  stringify(field->type()));
992  }
993  }
994  }
995 
996  return object;
997 }
998 
999 
1000 template <typename T>
1001 Array protobuf(const google::protobuf::RepeatedPtrField<T>& repeated)
1002 {
1003  static_assert(std::is_convertible<T*, google::protobuf::Message*>::value,
1004  "T must be a protobuf message");
1005 
1006  JSON::Array array;
1007  array.values.reserve(repeated.size());
1008  foreach (const T& elem, repeated) {
1009  array.values.emplace_back(JSON::protobuf(elem));
1010  }
1011 
1012  return array;
1013 }
1014 
1015 } // namespace JSON {
1016 
1017 #endif // __STOUT_PROTOBUF_HPP__
T as() const
Definition: json.hpp:116
Try< std::string > decode(const std::string &s)
Decode a string that is Base64-encoded with the standard Base64 alphabet.
Definition: base64.hpp:172
Try< Nothing > operator()(const JSON::Object &object) const
Definition: protobuf.hpp:387
bool isNone() const
Definition: result.hpp:112
#define O_WRONLY
Definition: fcntl.hpp:26
Definition: nothing.hpp:16
Definition: errorbase.hpp:35
#define ABORT(...)
Definition: abort.hpp:40
#define O_TRUNC
Definition: fcntl.hpp:29
Try< Bytes > size(const std::string &path, const FollowSymlink follow=FollowSymlink::FOLLOW_SYMLINK)
Definition: stat.hpp:100
const mode_t S_IRGRP
Definition: windows.hpp:319
#define O_APPEND
Definition: fcntl.hpp:30
Try< T > parse(const JSON::Value &value)
Definition: protobuf.hpp:681
Result< Classifier > decode(const Netlink< struct rtnl_cls > &cls)
Try< Nothing > operator()(const JSON::Number &number) const
Definition: protobuf.hpp:468
Definition: protobuf.hpp:605
Definition: json.hpp:227
Definition: try.hpp:34
#define O_CLOEXEC
Definition: fcntl.hpp:31
ssize_t write(const WindowsFD &fd, const void *data, size_t size)
Definition: write.hpp:29
static Result< T > error(const std::string &message)
Definition: result.hpp:53
Try< google::protobuf::RepeatedPtrField< T > > operator()(const JSON::Value &value)
Definition: protobuf.hpp:642
const mode_t S_IWUSR
Definition: windows.hpp:312
Result< google::protobuf::RepeatedPtrField< T > > operator()(int_fd fd, bool ignorePartial, bool undoFailed)
Definition: protobuf.hpp:302
Definition: json.hpp:203
Try< T > operator()(const JSON::Value &value)
Definition: protobuf.hpp:607
Definition: json.hpp:221
Try< off_t > lseek(int_fd fd, off_t offset, int whence)
Definition: lseek.hpp:29
Definition: protobuf.hpp:379
std::string encode(const std::string &s)
Encode a string to Base64 with the standard Base64 alphabet.
Definition: base64.hpp:159
Try< Nothing > operator()(const JSON::String &string) const
Definition: protobuf.hpp:409
#define O_CREAT
Definition: fcntl.hpp:28
Definition: json.hpp:194
const mode_t S_IRUSR
Definition: windows.hpp:311
Try< std::string > serialize(const T &t)
Definition: protobuf.hpp:181
std::map< std::string, Value > values
Definition: json.hpp:190
Definition: result.hpp:40
Definition: json.hpp:154
Try< Nothing > write(int_fd fd, const google::protobuf::Message &message)
Definition: protobuf.hpp:66
Try< Nothing > operator()(const JSON::Array &array) const
Definition: protobuf.hpp:526
void field(const std::string &key, const T &value)
Definition: jsonify.hpp:435
Definition: json.hpp:213
Definition: protobuf.hpp:200
Try< Nothing > close(int fd)
Definition: close.hpp:24
Option< T > max(const Option< T > &left, const Option< T > &right)
Definition: option.hpp:199
void json(JSON::ObjectWriter *writer, const Task &task)
Try< Nothing > operator()(const JSON::Boolean &boolean) const
Definition: protobuf.hpp:545
Definition: jsonify.hpp:418
Try< T > deserialize(const std::string &value)
Definition: protobuf.hpp:160
#define foreachpair(KEY, VALUE, ELEMS)
Definition: foreach.hpp:51
std::vector< Value > values
Definition: json.hpp:199
const T & get() const
Definition: result.hpp:115
Result< T > read(int_fd fd, bool ignorePartial=false, bool undoFailed=false)
Definition: protobuf.hpp:334
Result< T > operator()(int_fd fd, bool ignorePartial, bool undoFailed)
Definition: protobuf.hpp:202
static Try error(const E &e)
Definition: try.hpp:42
Definition: json.hpp:245
Result< std::string > read(int_fd fd, size_t size)
Definition: read.hpp:50
Parser(google::protobuf::Message *_message, const google::protobuf::FieldDescriptor *_field)
Definition: protobuf.hpp:381
Try< Nothing > operator()(const JSON::Null &) const
Definition: protobuf.hpp:562
Definition: protobuf.hpp:692
Try< Nothing > parse(google::protobuf::Message *message, const JSON::Object &object)
Definition: protobuf.hpp:577
Definition: none.hpp:27
bool isError() const
Definition: try.hpp:71
Try< int_fd > open(const std::string &path, int oflag, mode_t mode=0)
Definition: open.hpp:39
Definition: json.hpp:75
Object protobuf(const google::protobuf::Message &message)
Definition: protobuf.hpp:836
Definition: json.hpp:89
bool isSome() const
Definition: result.hpp:111
bool isError() const
Definition: result.hpp:113
Try< Nothing > append(const std::string &path, const google::protobuf::Message &message)
Definition: protobuf.hpp:135
int int_fd
Definition: int_fd.hpp:35
std::string stringify(int flags)
#define O_RDONLY
Definition: fcntl.hpp:25
const mode_t S_IROTH
Definition: windows.hpp:327
const T & get() const
Definition: try.hpp:73
constexpr const char * name
Definition: shell.hpp:41
Definition: representation.hpp:72
Definition: jsonify.hpp:384