Apache Mesos
protobuf_utils.hpp
Go to the documentation of this file.
1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 
17 #ifndef __PROTOBUF_UTILS_HPP__
18 #define __PROTOBUF_UTILS_HPP__
19 
20 #include <initializer_list>
21 #include <ostream>
22 #include <set>
23 #include <string>
24 
25 #include <sys/stat.h>
26 
27 #include <google/protobuf/map.h>
28 
29 #include <mesos/mesos.hpp>
30 
32 
33 #include <mesos/master/master.hpp>
34 
35 #include <mesos/slave/isolator.hpp>
36 
37 #include <process/time.hpp>
38 
39 #include <stout/duration.hpp>
40 #include <stout/ip.hpp>
41 #include <stout/none.hpp>
42 #include <stout/option.hpp>
43 #include <stout/try.hpp>
44 #include <stout/uuid.hpp>
45 
46 #include "messages/messages.hpp"
47 
48 // Forward declaration (in lieu of an include).
49 namespace process {
50 struct UPID;
51 }
52 
53 namespace mesos {
54 
55 class ObjectApprovers;
56 
57 namespace internal {
58 
59 namespace master {
60 // Forward declaration (in lieu of an include).
61 struct Framework;
62 struct Slave;
63 } // namespace master {
64 
65 namespace protobuf {
66 
68  const FrameworkInfo& framework,
70 
71 
72 // Returns whether the task state is terminal. Terminal states
73 // mean that the resources are released and the task cannot
74 // transition back to a non-terminal state. Note that
75 // `TASK_UNREACHABLE` is not a terminal state, but still
76 // releases the resources.
77 bool isTerminalState(const TaskState& state);
78 
79 
80 // See TaskStatus for more information about these fields. Note
81 // that the 'uuid' must be provided for updates that need
82 // acknowledgement. Currently, all slave and executor generated
83 // updates require acknowledgement, whereas master generated
84 // and scheduler driver generated updates do not.
85 StatusUpdate createStatusUpdate(
86  const FrameworkID& frameworkId,
87  const Option<SlaveID>& slaveId,
88  const TaskID& taskId,
89  const TaskState& state,
90  const TaskStatus::Source& source,
91  const Option<id::UUID>& uuid,
92  const std::string& message = "",
93  const Option<TaskStatus::Reason>& reason = None(),
94  const Option<ExecutorID>& executorId = None(),
95  const Option<bool>& healthy = None(),
96  const Option<CheckStatusInfo>& checkStatus = None(),
97  const Option<Labels>& labels = None(),
98  const Option<ContainerStatus>& containerStatus = None(),
99  const Option<TimeInfo>& unreachableTime = None(),
100  const Option<Resources>& limitedResources = None());
101 
102 
103 StatusUpdate createStatusUpdate(
104  const FrameworkID& frameworkId,
105  const TaskStatus& status,
106  const Option<SlaveID>& slaveId);
107 
108 
109 // Helper function that creates a new task status from scratch with
110 // obligatory fields set.
111 TaskStatus createTaskStatus(
112  const TaskID& taskId,
113  const TaskState& state,
114  const id::UUID& uuid,
115  double timestamp);
116 
117 
118 // Helper function that creates a new task status from the given task
119 // status. Specific fields in `status` can be overridden in the new
120 // status by specifying the appropriate argument. Fields `task_id`,
121 // `slave_id`, `executor_id`, cannot be changed; while `timestamp`
122 // and `uuid` cannot be preserved.
123 //
124 // NOTE: A task status update may be used for guaranteed delivery of
125 // some task-related information, e.g., task's health update. In this
126 // case, it is often desirable to preserve specific fields from the
127 // previous status update to avoid shadowing information that was
128 // delivered previously.
129 TaskStatus createTaskStatus(
130  TaskStatus status,
131  const id::UUID& uuid,
132  double timestamp,
133  const Option<TaskState>& state = None(),
134  const Option<std::string>& message = None(),
135  const Option<TaskStatus::Source>& source = None(),
136  const Option<TaskStatus::Reason>& reason = None(),
137  const Option<std::string>& data = None(),
138  const Option<bool>& healthy = None(),
139  const Option<CheckStatusInfo>& checkStatus = None(),
140  const Option<Labels>& labels = None(),
141  const Option<ContainerStatus>& containerStatus = None(),
142  const Option<TimeInfo>& unreachableTime = None());
143 
144 
145 Task createTask(
146  const TaskInfo& task,
147  const TaskState& state,
148  const FrameworkID& frameworkId);
149 
150 
151 Option<bool> getTaskHealth(const Task& task);
152 
153 
155 
156 
158 
159 
160 bool isTerminalState(const OperationState& state);
161 
162 
163 OperationStatus createOperationStatus(
164  const OperationState& state,
165  const Option<OperationID>& operationId = None(),
166  const Option<std::string>& message = None(),
167  const Option<Resources>& convertedResources = None(),
168  const Option<id::UUID>& statusUUID = None());
169 
170 
172  const Offer::Operation& info,
173  const OperationStatus& latestStatus,
174  const Option<FrameworkID>& frameworkId,
175  const Option<SlaveID>& slaveId,
176  const Option<UUID>& operationUUID = None());
177 
178 
179 UpdateOperationStatusMessage createUpdateOperationStatusMessage(
180  const UUID& operationUUID,
181  const OperationStatus& status,
182  const Option<OperationStatus>& latestStatus = None(),
183  const Option<FrameworkID>& frameworkId = None(),
184  const Option<SlaveID>& slaveId = None());
185 
186 
187 // Create a `UUID`. If `uuid` is given it is used to initialize
188 // the created `UUID`; otherwise a random `UUID` is returned.
189 UUID createUUID(const Option<id::UUID>& uuid = None());
190 
191 
192 // Helper function that creates a MasterInfo from UPID.
193 MasterInfo createMasterInfo(const process::UPID& pid);
194 
195 
196 Label createLabel(
197  const std::string& key,
198  const Option<std::string>& value = None());
199 
200 
201 // Helper function to convert a protobuf string map to `Labels`.
203  const google::protobuf::Map<std::string, std::string>& map);
204 
205 
206 // Helper function to convert a `Labels` to a protobuf string map.
208  const Labels& labels);
209 
210 
211 // Previously, `Resource` did not contain `AllocationInfo`.
212 // So for backwards compatibility with old schedulers and
213 // tooling, we must allow operations to contain `Resource`s
214 // without an `AllocationInfo`. This allows the master to
215 // inject the offer's `AllocationInfo` into the operation's
216 // resources.
218  Offer::Operation* operation,
219  const Resource::AllocationInfo& allocationInfo);
220 
221 
222 // This strips the Resource::AllocationInfo from all
223 // Resource objects contained within the operation.
224 void stripAllocationInfo(Offer::Operation* operation);
225 
226 
227 bool isSpeculativeOperation(const Offer::Operation& operation);
228 
229 
230 // Helper function to pack a protobuf list of resource versions.
231 google::protobuf::RepeatedPtrField<ResourceVersionUUID> createResourceVersions(
232  const hashmap<Option<ResourceProviderID>, UUID>& resourceVersions);
233 
234 
235 // Helper function to unpack a protobuf list of resource versions.
237  const google::protobuf::RepeatedPtrField<ResourceVersionUUID>&
238  resourceVersionUUIDs);
239 
240 
241 // Helper function that fills in a TimeInfo from the current time.
242 TimeInfo getCurrentTime();
243 
244 
245 // Helper function that creates a `FileInfo` from data returned by `stat()`.
246 FileInfo createFileInfo(const std::string& path, const struct stat& s);
247 
248 
249 ContainerID getRootContainerId(const ContainerID& containerId);
250 
251 
252 ContainerID parseContainerId(const std::string& value);
253 
254 
256 
257 namespace slave {
258 
259 // TODO(bmahler): Store the repeated field within this so that we
260 // don't drop unknown capabilities.
262 {
263  Capabilities() = default;
264 
265  template <typename Iterable>
266  Capabilities(const Iterable& capabilities)
267  {
268  foreach (const SlaveInfo::Capability& capability, capabilities) {
269  switch (capability.type()) {
271  break;
272  case SlaveInfo::Capability::MULTI_ROLE:
273  multiRole = true;
274  break;
275  case SlaveInfo::Capability::HIERARCHICAL_ROLE:
276  hierarchicalRole = true;
277  break;
278  case SlaveInfo::Capability::RESERVATION_REFINEMENT:
279  reservationRefinement = true;
280  break;
281  case SlaveInfo::Capability::RESOURCE_PROVIDER:
282  resourceProvider = true;
283  break;
284  // If adding another case here be sure to update the
285  // equality operator.
286  }
287  }
288  }
289 
290  // See mesos.proto for the meaning of agent capabilities.
291  bool multiRole = false;
292  bool hierarchicalRole = false;
293  bool reservationRefinement = false;
294  bool resourceProvider = false;
295 
296  google::protobuf::RepeatedPtrField<SlaveInfo::Capability>
298  {
299  google::protobuf::RepeatedPtrField<SlaveInfo::Capability> result;
300  if (multiRole) {
301  result.Add()->set_type(SlaveInfo::Capability::MULTI_ROLE);
302  }
303  if (hierarchicalRole) {
304  result.Add()->set_type(SlaveInfo::Capability::HIERARCHICAL_ROLE);
305  }
306  if (reservationRefinement) {
307  result.Add()->set_type(SlaveInfo::Capability::RESERVATION_REFINEMENT);
308  }
309  if (resourceProvider) {
310  result.Add()->set_type(SlaveInfo::Capability::RESOURCE_PROVIDER);
311  }
312 
313  return result;
314  }
315 };
316 
317 
318 bool operator==(const Capabilities& left, const Capabilities& right);
319 bool operator!=(const Capabilities& left, const Capabilities& right);
320 std::ostream& operator<<(std::ostream& stream, const Capabilities& c);
321 
322 
323 mesos::slave::ContainerLimitation createContainerLimitation(
324  const Resources& resources,
325  const std::string& message,
326  const TaskStatus::Reason& reason);
327 
328 
329 mesos::slave::ContainerState createContainerState(
330  const Option<ExecutorInfo>& executorInfo,
331  const ContainerID& id,
332  pid_t pid,
333  const std::string& directory);
334 
335 } // namespace slave {
336 
337 namespace maintenance {
338 
342 Unavailability createUnavailability(
343  const process::Time& start,
344  const Option<Duration>& duration = None());
345 
346 
350 google::protobuf::RepeatedPtrField<MachineID> createMachineList(
351  std::initializer_list<MachineID> ids);
352 
353 
358 mesos::maintenance::Window createWindow(
359  std::initializer_list<MachineID> ids,
360  const Unavailability& unavailability);
361 
362 
367 mesos::maintenance::Schedule createSchedule(
368  std::initializer_list<mesos::maintenance::Window> windows);
369 
370 } // namespace maintenance {
371 
372 
373 namespace master {
374 
375 // TODO(bmahler): Store the repeated field within this so that we
376 // don't drop unknown capabilities.
378 {
379  Capabilities() = default;
380 
381  template <typename Iterable>
382  Capabilities(const Iterable& capabilities)
383  {
384  foreach (const MasterInfo::Capability& capability, capabilities) {
385  switch (capability.type()) {
387  break;
388  case MasterInfo::Capability::AGENT_UPDATE:
389  agentUpdate = true;
390  break;
391  }
392  }
393  }
394 
395  bool agentUpdate = false;
396 };
397 
398 namespace event {
399 
400 // Helper for creating a `TASK_UPDATED` event from a `Task`, its
401 // latest state according to the agent, and its status corresponding
402 // to the last status update acknowledged from the scheduler.
404  const Task& task,
405  const TaskState& state,
406  const TaskStatus& status);
407 
408 
409 // Helper for creating a `TASK_ADDED` event from a `Task`.
410 mesos::master::Event createTaskAdded(const Task& task);
411 
412 
413 // Helper for creating a 'FRAMEWORK_ADDED' event from a `Framework`.
415  const mesos::internal::master::Framework& framework);
416 
417 
418 // Helper for creating a 'FRAMEWORK_UPDATED' event from a `Framework`.
420  const mesos::internal::master::Framework& framework);
421 
422 
423 // Helper for creating a 'FRAMEWORK_REMOVED' event from a `FrameworkInfo`.
424 mesos::master::Event createFrameworkRemoved(const FrameworkInfo& frameworkInfo);
425 
426 
427 // Helper for creating an `Agent` response.
428 mesos::master::Response::GetAgents::Agent createAgentResponse(
429  const mesos::internal::master::Slave& slave,
430  const Option<process::Owned<ObjectApprovers>>& approvers = None());
431 
432 
433 // Helper for creating an `AGENT_ADDED` event from a `Slave`.
435  const mesos::internal::master::Slave& slave);
436 
437 
438 // Helper for creating an `AGENT_REMOVED` event from a `SlaveID`.
439 mesos::master::Event createAgentRemoved(const SlaveID& slaveId);
440 
441 } // namespace event {
442 } // namespace master {
443 
444 namespace framework {
445 
446 // TODO(bmahler): Store the repeated field within this so that we
447 // don't drop unknown capabilities.
449 {
450  Capabilities() = default;
451 
452  template <typename Iterable>
453  Capabilities(const Iterable& capabilities)
454  {
455  foreach (const FrameworkInfo::Capability& capability, capabilities) {
456  switch (capability.type()) {
458  break;
459  case FrameworkInfo::Capability::REVOCABLE_RESOURCES:
460  revocableResources = true;
461  break;
462  case FrameworkInfo::Capability::TASK_KILLING_STATE:
463  taskKillingState = true;
464  break;
465  case FrameworkInfo::Capability::GPU_RESOURCES:
466  gpuResources = true;
467  break;
468  case FrameworkInfo::Capability::SHARED_RESOURCES:
469  sharedResources = true;
470  break;
471  case FrameworkInfo::Capability::PARTITION_AWARE:
472  partitionAware = true;
473  break;
474  case FrameworkInfo::Capability::MULTI_ROLE:
475  multiRole = true;
476  break;
477  case FrameworkInfo::Capability::RESERVATION_REFINEMENT:
478  reservationRefinement = true;
479  break;
480  case FrameworkInfo::Capability::REGION_AWARE:
481  regionAware = true;
482  break;
483  }
484  }
485  }
486 
487  // See mesos.proto for the meaning of these capabilities.
488  bool revocableResources = false;
489  bool taskKillingState = false;
490  bool gpuResources = false;
491  bool sharedResources = false;
492  bool partitionAware = false;
493  bool multiRole = false;
494  bool reservationRefinement = false;
495  bool regionAware = false;
496 };
497 
498 
499 // Helper to get roles from FrameworkInfo based on the
500 // presence of the MULTI_ROLE capability.
501 std::set<std::string> getRoles(const FrameworkInfo& frameworkInfo);
502 
503 } // namespace framework {
504 
505 } // namespace protobuf {
506 } // namespace internal {
507 } // namespace mesos {
508 
509 #endif // __PROTOBUF_UTILS_HPP__
hashmap< Option< ResourceProviderID >, UUID > parseResourceVersions(const google::protobuf::RepeatedPtrField< ResourceVersionUUID > &resourceVersionUUIDs)
Definition: path.hpp:26
UpdateOperationStatusMessage createUpdateOperationStatusMessage(const UUID &operationUUID, const OperationStatus &status, const Option< OperationStatus > &latestStatus=None(), const Option< FrameworkID > &frameworkId=None(), const Option< SlaveID > &slaveId=None())
Capabilities(const Iterable &capabilities)
Definition: protobuf_utils.hpp:382
Try< Resources > getConsumedResources(const Offer::Operation &operation)
Option< bool > getTaskHealth(const Task &task)
mesos::master::Event createFrameworkAdded(const mesos::internal::master::Framework &framework)
std::ostream & operator<<(std::ostream &stream, const Future< T > &future)
Definition: future.hpp:1831
Definition: option.hpp:28
void stripAllocationInfo(Offer::Operation *operation)
bool operator==(const std::string &s, const UPID::ID &id)
Definition: pid.hpp:216
Definition: master.hpp:27
Definition: check.hpp:33
Labels convertStringMapToLabels(const google::protobuf::Map< std::string, std::string > &map)
google::protobuf::RepeatedPtrField< MachineID > createMachineList(std::initializer_list< MachineID > ids)
Helper for constructing a list of MachineID.
void injectAllocationInfo(Offer::Operation *operation, const Resource::AllocationInfo &allocationInfo)
Definition: protobuf_utils.hpp:261
Unavailability createUnavailability(const process::Time &start, const Option< Duration > &duration=None())
Helper for constructing an unavailability from a Time and Duration.
std::set< std::string > getRoles(const FrameworkInfo &frameworkInfo)
Capabilities(const Iterable &capabilities)
Definition: protobuf_utils.hpp:266
Result< ProcessStatus > status(pid_t pid)
Definition: proc.hpp:166
mesos::slave::ContainerState createContainerState(const Option< ExecutorInfo > &executorInfo, const ContainerID &id, pid_t pid, const std::string &directory)
Definition: resources.hpp:79
mesos::master::Event createFrameworkUpdated(const mesos::internal::master::Framework &framework)
ContainerID getRootContainerId(const ContainerID &containerId)
mesos::maintenance::Schedule createSchedule(std::initializer_list< mesos::maintenance::Window > windows)
Helper for constructing a maintenance Schedule.
Capability
Definition: capabilities.hpp:35
Try< google::protobuf::Map< std::string, std::string > > convertLabelsToStringMap(const Labels &labels)
Operation
Definition: cgroups.hpp:441
mesos::master::Event createAgentAdded(const mesos::internal::master::Slave &slave)
Try< Nothing > start(const std::string &name)
Starts the slice with the given name (via &#39;systemctl start <name>&#39;).
Definition: protobuf_utils.hpp:377
Capabilities(const Iterable &capabilities)
Definition: protobuf_utils.hpp:453
mesos::v1::scheduler::Event Event
Definition: mesos.hpp:2556
Definition: hashmap.hpp:38
mesos::master::Event createTaskAdded(const Task &task)
DWORD pid_t
Definition: windows.hpp:187
mesos::slave::ContainerLimitation createContainerLimitation(const Resources &resources, const std::string &message, const TaskStatus::Reason &reason)
FileInfo createFileInfo(const std::string &path, const struct stat &s)
mesos::master::Event createTaskUpdated(const Task &task, const TaskState &state, const TaskStatus &status)
mesos::master::Event createAgentRemoved(const SlaveID &slaveId)
Try< Nothing > unavailability(const Unavailability &unavailability)
An "untyped" PID, used to encapsulate the process ID for lower-layer abstractions (eg...
Definition: pid.hpp:39
StatusUpdate createStatusUpdate(const FrameworkID &frameworkId, const TaskStatus &status, const Option< SlaveID > &slaveId)
Option< ContainerStatus > getTaskContainerStatus(const Task &task)
Definition: uuid.hpp:35
Definition: protobuf_utils.hpp:448
Definition: spec.hpp:30
Task createTask(const TaskInfo &task, const TaskState &state, const FrameworkID &frameworkId)
const int UNKNOWN
Definition: diagnosis.hpp:39
google::protobuf::RepeatedPtrField< SlaveInfo::Capability > toRepeatedPtrField() const
Definition: protobuf_utils.hpp:297
Definition: protobuf.hpp:55
Try< hashmap< std::string, uint64_t > > stat(const std::string &hierarchy, const std::string &cgroup, const std::string &file)
Definition: time.hpp:23
Operation createOperation(const Offer::Operation &info, const OperationStatus &latestStatus, const Option< FrameworkID > &frameworkId, const Option< SlaveID > &slaveId, const Option< UUID > &operationUUID=None())
bool isTerminalState(const OperationState &state)
Iterable< V > map(F &&f, const Iterable< U, Us... > &input)
Definition: lambda.hpp:46
Definition: none.hpp:27
Definition: attributes.hpp:24
bool operator!=(const std::string &s, const UPID::ID &id)
Definition: pid.hpp:222
mesos::master::Event createFrameworkRemoved(const FrameworkInfo &frameworkInfo)
mesos::master::Response::GetAgents::Agent createAgentResponse(const mesos::internal::master::Slave &slave, const Option< process::Owned< ObjectApprovers >> &approvers=None())
Definition: executor.hpp:47
Type
Definition: capabilities.hpp:79
Definition: master.hpp:117
mesos::maintenance::Window createWindow(std::initializer_list< MachineID > ids, const Unavailability &unavailability)
Helper for constructing a maintenance Window.
TaskStatus createTaskStatus(TaskStatus status, const id::UUID &uuid, double timestamp, const Option< TaskState > &state=None(), const Option< std::string > &message=None(), const Option< TaskStatus::Source > &source=None(), const Option< TaskStatus::Reason > &reason=None(), const Option< std::string > &data=None(), const Option< bool > &healthy=None(), const Option< CheckStatusInfo > &checkStatus=None(), const Option< Labels > &labels=None(), const Option< ContainerStatus > &containerStatus=None(), const Option< TimeInfo > &unreachableTime=None())
bool frameworkHasCapability(const FrameworkInfo &framework, FrameworkInfo::Capability::Type capability)
bool isSpeculativeOperation(const Offer::Operation &operation)
Label createLabel(const std::string &key, const Option< std::string > &value=None())
Definition: owned.hpp:36
Definition: master.hpp:2170
MasterInfo createMasterInfo(const process::UPID &pid)
Option< CheckStatusInfo > getTaskCheckStatus(const Task &task)
OperationStatus createOperationStatus(const OperationState &state, const Option< OperationID > &operationId=None(), const Option< std::string > &message=None(), const Option< Resources > &convertedResources=None(), const Option< id::UUID > &statusUUID=None())
ContainerID parseContainerId(const std::string &value)
UUID createUUID(const Option< id::UUID > &uuid=None())
google::protobuf::RepeatedPtrField< ResourceVersionUUID > createResourceVersions(const hashmap< Option< ResourceProviderID >, UUID > &resourceVersions)