Apache Mesos
http.hpp
Go to the documentation of this file.
1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 
17 #ifndef __COMMON_HTTP_HPP__
18 #define __COMMON_HTTP_HPP__
19 
20 #include <vector>
21 
22 #include <mesos/http.hpp>
23 #include <mesos/mesos.hpp>
24 
26 
27 #include <mesos/quota/quota.hpp>
28 
30 #include <process/future.hpp>
31 #include <process/http.hpp>
32 #include <process/owned.hpp>
33 
34 #include <stout/hashmap.hpp>
35 #include <stout/hashset.hpp>
36 #include <stout/json.hpp>
37 #include <stout/jsonify.hpp>
38 #include <stout/protobuf.hpp>
39 #include <stout/recordio.hpp>
40 #include <stout/unreachable.hpp>
41 #include <stout/uuid.hpp>
42 
43 #include "internal/evolve.hpp"
44 
45 // TODO(benh): Remove this once we get C++14 as an enum should have a
46 // default hash.
47 namespace std {
48 
49 template <>
50 struct hash<mesos::authorization::Action>
51 {
52  typedef size_t result_type;
53 
54  typedef mesos::authorization::Action argument_type;
55 
56  result_type operator()(const argument_type& action) const
57  {
58  size_t seed = 0;
59  boost::hash_combine(
60  seed, static_cast<std::underlying_type<argument_type>::type>(action));
61  return seed;
62  }
63 };
64 
65 } // namespace std {
66 
67 namespace mesos {
68 
69 class Attributes;
70 class Resources;
71 class Task;
72 
73 namespace internal {
74 
75 // Name of the default, basic authenticator.
76 constexpr char DEFAULT_BASIC_HTTP_AUTHENTICATOR[] = "basic";
77 
78 // Name of the default, basic authenticatee.
79 constexpr char DEFAULT_BASIC_HTTP_AUTHENTICATEE[] = "basic";
80 
81 // Name of the default, JWT authenticator.
82 constexpr char DEFAULT_JWT_HTTP_AUTHENTICATOR[] = "jwt";
83 
84 
85 // Contains the media types corresponding to some of the "Content-*",
86 // "Accept-*" and "Message-*" prefixed request headers in our internal
87 // representation.
89 {
90  ContentType content; // 'Content-Type' header.
91  ContentType accept; // 'Accept' header.
92  Option<ContentType> messageContent; // 'Message-Content-Type' header.
93  Option<ContentType> messageAccept; // 'Message-Accept' header.
94 };
95 
96 
97 // Serializes a protobuf message for transmission
98 // based on the HTTP content type.
99 // NOTE: For streaming `contentType`, `message` would not
100 // be serialized in "Record-IO" format.
101 std::string serialize(
102  ContentType contentType,
103  const google::protobuf::Message& message);
104 
105 
106 // Deserializes a string message into a protobuf message based on the
107 // HTTP content type.
108 template <typename Message>
110  ContentType contentType,
111  const std::string& body)
112 {
113  switch (contentType) {
114  case ContentType::PROTOBUF: {
115  Message message;
116  if (!message.ParseFromString(body)) {
117  return Error("Failed to parse body into a protobuf object");
118  }
119  return message;
120  }
121  case ContentType::JSON: {
122  Try<JSON::Value> value = JSON::parse(body);
123  if (value.isError()) {
124  return Error("Failed to parse body into JSON: " + value.error());
125  }
126 
127  return ::protobuf::parse<Message>(value.get());
128  }
129  case ContentType::RECORDIO: {
130  return Error("Deserializing a RecordIO stream is not supported");
131  }
132  }
133 
134  UNREACHABLE();
135 }
136 
137 
138 // Returns true if the media type can be used for
139 // streaming requests/responses.
140 bool streamingMediaType(ContentType contentType);
141 
142 
143 // Represents the streaming HTTP connection to a client, such as a framework,
144 // executor, or operator subscribed to the '/api/vX' endpoint.
145 // The `Event` template is the evolved message being sent to the client,
146 // e.g. `v1::scheduler::Event`, `v1::master::Event`, or `v1::executor::Event`.
147 template <typename Event>
149 {
151  const process::http::Pipe::Writer& _writer,
152  ContentType _contentType,
153  id::UUID _streamId = id::UUID::random())
154  : writer(_writer),
155  contentType(_contentType),
156  streamId(_streamId) {}
157 
158  template <typename Message>
159  bool send(const Message& message)
160  {
161  // TODO(bmahler): Remove this evolve(). Could we still
162  // somehow assert that evolve(message) produces a result
163  // of type Event without calling evolve()?
164  Event e = evolve(message);
165 
166  std::string record = serialize(contentType, e);
167 
168  return writer.write(::recordio::encode(record));
169  }
170 
171  // Like the above send, but for already serialized data.
172  bool send(const std::string& event)
173  {
174  return writer.write(::recordio::encode(event));
175  }
176 
177  bool close()
178  {
179  return writer.close();
180  }
181 
183  {
184  return writer.readerClosed();
185  }
186 
190 };
191 
192 
193 // The representation of generic v0 protobuf => v1 protobuf as JSON,
194 // e.g., `jsonify(asV1Protobuf(message))`.
195 //
196 // Specifically, this acts the same as JSON::Protobuf, except that
197 // it remaps "slave" to "agent" in field names and enum values.
198 struct asV1Protobuf : Representation<google::protobuf::Message>
199 {
201 };
202 
203 void json(JSON::ObjectWriter* writer, const asV1Protobuf& protobuf);
204 
205 
206 JSON::Object model(const Resources& resources);
208 JSON::Object model(const Attributes& attributes);
209 JSON::Object model(const CommandInfo& command);
210 JSON::Object model(const ExecutorInfo& executorInfo);
211 JSON::Array model(const Labels& labels);
212 JSON::Object model(const Task& task);
213 JSON::Object model(const FileInfo& fileInfo);
214 JSON::Object model(const google::protobuf::Map<std::string, Value_Scalar>& map);
215 
216 void json(JSON::ObjectWriter* writer, const Task& task);
217 
218 
219 // NOTE: The `metrics` object provided as an argument must outlive
220 // the returned function, since the function captures `metrics`
221 // by reference to avoid a really expensive map copy. This is a
222 // rather unsafe approach, but in typical jsonify usage this is
223 // not an issue.
224 //
225 // TODO(bmahler): Use std::enable_if with std::is_same to check
226 // that T is either the master or agent GetMetrics.
227 template <typename T>
228 std::function<void(JSON::ObjectWriter*)> jsonifyGetMetrics(
229  const std::map<std::string, double>& metrics)
230 {
231  // Serialize the following message:
232  //
233  // mesos::master::Response::GetMetrics getMetrics;
234  // // or: mesos::agent::Response::GetMetrics getMetrics;
235  //
236  // foreachpair (const string& key, double value, metrics) {
237  // Metric* metric = getMetrics->add_metrics();
238  // metric->set_name(key);
239  // metric->set_value(value);
240  // }
241 
242  return [&](JSON::ObjectWriter* writer) {
243  const google::protobuf::Descriptor* descriptor = T::descriptor();
244 
245  int field = T::kMetricsFieldNumber;
246 
247  writer->field(
248  descriptor->FindFieldByNumber(field)->name(),
249  [&](JSON::ArrayWriter* writer) {
250  foreachpair (const std::string& key, double value, metrics) {
251  writer->element([&](JSON::ObjectWriter* writer) {
252  const google::protobuf::Descriptor* descriptor =
253  v1::Metric::descriptor();
254 
255  int field;
256 
257  field = v1::Metric::kNameFieldNumber;
258  writer->field(
259  descriptor->FindFieldByNumber(field)->name(), key);
260 
261  field = v1::Metric::kValueFieldNumber;
262  writer->field(
263  descriptor->FindFieldByNumber(field)->name(), value);
264  });
265  }
266  });
267  };
268 }
269 
270 
271 // TODO(bmahler): Use std::enable_if with std::is_same to check
272 // that T is either the master or agent GetMetrics.
273 template <typename T>
275  const std::map<std::string, double>& metrics)
276 {
277  // Serialize the following message:
278  //
279  // v1::master::Response::GetMetrics getMetrics;
280  // // or: v1::agent::Response::GetMetrics getMetrics;
281  //
282  // foreachpair (const string& key, double value, metrics) {
283  // Metric* metric = getMetrics->add_metrics();
284  // metric->set_name(key);
285  // metric->set_value(value);
286  // }
287 
288  auto serializeMetric = [](const std::string& key, double value) {
289  std::string output;
290  google::protobuf::io::StringOutputStream stream(&output);
291  google::protobuf::io::CodedOutputStream writer(&stream);
292 
293  google::protobuf::internal::WireFormatLite::WriteString(
294  v1::Metric::kNameFieldNumber, key, &writer);
295  google::protobuf::internal::WireFormatLite::WriteDouble(
296  v1::Metric::kValueFieldNumber, value, &writer);
297 
298  // While an explicit Trim() isn't necessary (since the coded
299  // output stream is destructed before the string is returned),
300  // it's a quite tricky bug to diagnose if Trim() is missed, so
301  // we always do it explicitly to signal the reader about this
302  // subtlety.
303  writer.Trim();
304  return output;
305  };
306 
307  std::string output;
308  google::protobuf::io::StringOutputStream stream(&output);
309  google::protobuf::io::CodedOutputStream writer(&stream);
310 
311  foreachpair (const std::string& key, double value, metrics) {
312  google::protobuf::internal::WireFormatLite::WriteBytes(
313  T::kMetricsFieldNumber,
314  serializeMetric(key, value),
315  &writer);
316  }
317 
318  // While an explicit Trim() isn't necessary (since the coded
319  // output stream is destructed before the string is returned),
320  // it's a quite tricky bug to diagnose if Trim() is missed, so
321  // we always do it explicitly to signal the reader about this
322  // subtlety.
323  writer.Trim();
324  return output;
325 }
326 
327 
328 } // namespace internal {
329 
330 void json(JSON::ObjectWriter* writer, const Attributes& attributes);
331 void json(JSON::ObjectWriter* writer, const CommandInfo& command);
332 void json(JSON::ObjectWriter* writer, const DomainInfo& domainInfo);
333 void json(JSON::ObjectWriter* writer, const ExecutorInfo& executorInfo);
334 void json(
335  JSON::StringWriter* writer, const FrameworkInfo::Capability& capability);
336 void json(JSON::ArrayWriter* writer, const Labels& labels);
337 void json(JSON::ObjectWriter* writer, const MasterInfo& info);
338 void json(
339  JSON::StringWriter* writer, const MasterInfo::Capability& capability);
340 void json(JSON::ObjectWriter* writer, const Offer& offer);
341 void json(JSON::ObjectWriter* writer, const Resources& resources);
342 void json(
343  JSON::ObjectWriter* writer,
344  const google::protobuf::RepeatedPtrField<Resource>& resources);
345 void json(JSON::ObjectWriter* writer, const ResourceQuantities& quantities);
346 void json(JSON::ObjectWriter* writer, const ResourceLimits& limits);
347 void json(JSON::ObjectWriter* writer, const SlaveInfo& slaveInfo);
348 void json(
349  JSON::StringWriter* writer, const SlaveInfo::Capability& capability);
350 void json(JSON::ObjectWriter* writer, const Task& task);
351 void json(JSON::ObjectWriter* writer, const TaskStatus& status);
352 
353 
354 // Implementation of the `ObjectApprover` interface authorizing all objects.
356 {
357 public:
359  const Option<ObjectApprover::Object>& object) const noexcept override
360  {
361  return true;
362  }
363 };
364 
365 
367 {
368 public:
370  const Option<Authorizer*>& authorizer,
372  std::initializer_list<authorization::Action> actions);
373 
375  authorization::Action action,
376  const ObjectApprover::Object& object) const
377  {
378  if (!approvers.contains(action)) {
379  LOG(WARNING)
380  << "Attempted to authorize principal "
381  << " '" << (principal.isSome() ? stringify(*principal) : "") << "'"
382  << " for unexpected action " << authorization::Action_Name(action);
383 
384  return false;
385  }
386 
387  return approvers.at(action)->approved(object);
388  }
389 
390  // Constructs one (or more) authorization objects, depending on the
391  // action, and returns true if all action-object pairs are authorized.
392  //
393  // NOTE: This template has specializations that actually check
394  // more than one action-object pair.
395  template <authorization::Action action, typename... Args>
396  bool approved(const Args&... args) const
397  {
398  const Try<bool> approval =
399  approved(action, ObjectApprover::Object(args...));
400 
401  if (approval.isError()) {
402  // NOTE: Silently dropping errors here creates a potential for
403  // _transient_ authorization errors to make API events subscriber's view
404  // inconsistent (see MESOS-10085). Also, this creates potential for an
405  // object to silently disappear from Operator API endpoint response in
406  // case of an authorization error (see MESOS-10099).
407  //
408  // TODO(joerg84): Expose these errors back to the caller.
409  LOG(WARNING) << "Failed to authorize principal "
410  << " '" << (principal.isSome() ? stringify(*principal) : "")
411  << "' for action " << authorization::Action_Name(action)
412  << ": " << approval.error();
413 
414  return false;
415  }
416 
417  return approval.get();
418  }
419 
421 
422 private:
424  hashmap<
425  authorization::Action,
426  std::shared_ptr<const ObjectApprover>>&& _approvers,
428  : principal(_principal),
429  approvers(std::move(_approvers)) {}
430 
432  approvers;
433 };
434 
435 
436 template <>
437 inline bool ObjectApprovers::approved<authorization::VIEW_ROLE>(
438  const Resource& resource) const
439 {
440  // Necessary because recovered agents are presented in old format.
441  if (resource.has_role() && resource.role() != "*" &&
442  !approved<authorization::VIEW_ROLE>(resource.role())) {
443  return false;
444  }
445 
446  // Reservations follow a path model where each entry is a child of the
447  // previous one. Therefore, to accept the resource the acceptor has to
448  // accept all entries.
449  foreach (Resource::ReservationInfo reservation, resource.reservations()) {
450  if (!approved<authorization::VIEW_ROLE>(reservation.role())) {
451  return false;
452  }
453  }
454 
455  if (resource.has_allocation_info() &&
456  !approved<authorization::VIEW_ROLE>(
457  resource.allocation_info().role())) {
458  return false;
459  }
460 
461  return true;
462 }
463 
464 
470 template <typename T>
472 {
473 public:
475  {
476  if (id.isSome()) {
477  T targetId_;
478  targetId_.set_value(id.get());
479  targetId = targetId_;
480  }
481  }
482 
483  bool accept(const T& candidateId) const
484  {
485  if (targetId.isNone()) {
486  return true;
487  }
488 
489  return candidateId.value() == targetId->value();
490  }
491 
492 protected:
494 };
495 
496 
497 // Authorizes access to an HTTP endpoint. The `method` parameter
498 // determines which ACL action will be used in the authorization.
499 // It is expected that the caller has validated that `method` is
500 // supported by this function. Currently "GET" is supported.
501 //
502 // TODO(nfnt): Prefer types instead of strings
503 // for `endpoint` and `method`, see MESOS-5300.
505  const std::string& endpoint,
506  const std::string& method,
507  const Option<Authorizer*>& authorizer,
509 
510 
524  const std::string& realm,
525  const std::vector<std::string>& httpAuthenticatorNames,
526  const Option<Credentials>& credentials = None(),
527  const Option<std::string>& jwtSecretKey = None());
528 
529 
530 // Logs the request. Route handlers can compose this with the
531 // desired request handler to get consistent request logging.
533 
534 
535 // Log the response for the corresponding request together with the request
536 // processing time. Route handlers can compose this with the desired request
537 // handler to get consistent request/response logging.
538 //
539 // TODO(alexr): Consider taking `response` as a future to allow logging for
540 // cases when response has not been generated.
541 void logResponse(
543  const process::http::Response& response);
544 
545 } // namespace mesos {
546 
547 #endif // __COMMON_HTTP_HPP__
size_t result_type
Definition: http.hpp:52
Definition: http.hpp:355
ContentType accept
Definition: http.hpp:91
ContentType
Definition: http.hpp:43
Definition: errorbase.hpp:36
Definition: resource_quantities.hpp:192
Future< Response > request(const Request &request, bool streamedResponse=false)
Asynchronously sends an HTTP request to the process and returns the HTTP response once the entire res...
T & get()&
Definition: try.hpp:80
Definition: authorizer.hpp:53
id::UUID streamId
Definition: http.hpp:189
Definition: check.hpp:33
std::string encode(const std::string &record)
Returns the "Record-IO" encoded record.
Definition: recordio.hpp:63
bool approved(const Args &...args) const
Definition: http.hpp:396
Definition: http.hpp:198
bool streamingMediaType(ContentType contentType)
Definition: resource_quantities.hpp:63
std::string serializeGetMetrics(const std::map< std::string, double > &metrics)
Definition: http.hpp:274
constexpr char DEFAULT_BASIC_HTTP_AUTHENTICATEE[]
Definition: http.hpp:79
process::Future< Nothing > closed() const
Definition: http.hpp:182
Result< ProcessStatus > status(pid_t pid)
Definition: proc.hpp:166
Definition: resources.hpp:83
Definition: type_utils.hpp:598
Definition: json.hpp:198
result_type operator()(const argument_type &action) const
Definition: http.hpp:56
Capability
Definition: capabilities.hpp:35
Try< bool > approved(authorization::Action action, const ObjectApprover::Object &object) const
Definition: http.hpp:374
Definition: jsonify.hpp:255
bool close()
Definition: http.hpp:177
Option< T > targetId
Definition: http.hpp:493
bool isSome() const
Definition: option.hpp:116
Definition: http.hpp:533
Definition: json.hpp:158
mesos::v1::scheduler::Event Event
Definition: mesos.hpp:2798
Definition: hashmap.hpp:38
constexpr char DEFAULT_JWT_HTTP_AUTHENTICATOR[]
Definition: http.hpp:82
void logResponse(const process::http::Request &request, const process::http::Response &response)
void logRequest(const process::http::Request &request)
Used to filter results for API handlers.
Definition: http.hpp:471
static UUID random()
Definition: uuid.hpp:38
void field(const std::string &key, const T &value)
Definition: jsonify.hpp:347
Try< Message > deserialize(ContentType contentType, const std::string &body)
Definition: http.hpp:109
Definition: http.hpp:88
Definition: http.hpp:354
void json(JSON::ObjectWriter *writer, const TaskStatus &status)
bool send(const Message &message)
Definition: http.hpp:159
JSON::Object model(const google::protobuf::Map< std::string, Value_Scalar > &map)
Definition: uuid.hpp:35
StreamingHttpConnection(const process::http::Pipe::Writer &_writer, ContentType _contentType, id::UUID _streamId=id::UUID::random())
Definition: http.hpp:150
Definition: agent.hpp:25
Definition: http.hpp:366
std::function< void(JSON::ObjectWriter *)> jsonifyGetMetrics(const std::map< std::string, double > &metrics)
Definition: http.hpp:228
Definition: jsonify.hpp:326
ContentType contentType
Definition: http.hpp:188
#define foreachpair(KEY, VALUE, ELEMS)
Definition: foreach.hpp:51
bool accept(const T &candidateId) const
Definition: http.hpp:483
Try< Value > parse(const std::string &s)
Returns the OCI v1 descriptor, image index, image manifest and image configuration from the given str...
Definition: json.hpp:978
ContentType content
Definition: http.hpp:90
Option< ContentType > messageContent
Definition: http.hpp:92
Definition: protobuf.hpp:61
static Try error(const E &e)
Definition: try.hpp:43
Try< Nothing > initializeHttpAuthenticators(const std::string &realm, const std::vector< std::string > &httpAuthenticatorNames, const Option< Credentials > &credentials=None(), const Option< std::string > &jwtSecretKey=None())
Helper function to create HTTP authenticators for a given realm and register in libprocess.
#define UNREACHABLE()
Definition: unreachable.hpp:22
constexpr char DEFAULT_BASIC_HTTP_AUTHENTICATOR[]
Definition: http.hpp:76
Iterable< V > map(F &&f, const Iterable< U, Us... > &input)
Definition: lambda.hpp:46
Definition: none.hpp:27
Definition: attributes.hpp:24
bool isError() const
Definition: try.hpp:78
Definition: http.hpp:612
VolumeCapability evolve(const types::VolumeCapability &capability)
bool send(const std::string &event)
Definition: http.hpp:172
Try< uint32_t > type(const std::string &path)
IDAcceptor(const Option< std::string > &id=None())
Definition: http.hpp:474
Option< ContentType > messageAccept
Definition: http.hpp:93
Try< Nothing > create(const std::string &hierarchy, const std::string &cgroup, bool recursive=false)
std::string serialize(ContentType contentType, const google::protobuf::Message &message)
process::Future< bool > authorizeEndpoint(const std::string &endpoint, const std::string &method, const Option< Authorizer * > &authorizer, const Option< process::http::authentication::Principal > &principal)
This interface represents a function object returned by the authorizer which can be used locally (and...
Definition: authorizer.hpp:47
std::string stringify(int flags)
process::http::Pipe::Writer writer
Definition: http.hpp:187
mesos::authorization::Action argument_type
Definition: http.hpp:54
PID< MetricsProcess > metrics
const Option< process::http::authentication::Principal > principal
Definition: http.hpp:420
Definition: representation.hpp:72
Try< bool > approved(const Option< ObjectApprover::Object > &object) const noexceptoverride
This method returns whether access to the specified object is authorized or not, or Error...
Definition: http.hpp:358
Definition: jsonify.hpp:296
Definition: attributes.hpp:32