Apache Mesos
protobuf_utils.hpp
Go to the documentation of this file.
1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 
17 #ifndef __PROTOBUF_UTILS_HPP__
18 #define __PROTOBUF_UTILS_HPP__
19 
20 #include <initializer_list>
21 #include <ostream>
22 #include <set>
23 #include <string>
24 
25 #include <sys/stat.h>
26 
27 #include <google/protobuf/map.h>
28 
29 #include <mesos/mesos.hpp>
30 
32 
33 #include <mesos/master/master.hpp>
34 
35 #include <mesos/slave/isolator.hpp>
36 
37 #include <process/time.hpp>
38 
39 #include <stout/duration.hpp>
40 #include <stout/ip.hpp>
41 #include <stout/none.hpp>
42 #include <stout/option.hpp>
43 #include <stout/try.hpp>
44 #include <stout/uuid.hpp>
45 
46 #include "messages/messages.hpp"
47 
48 // Forward declaration (in lieu of an include).
49 namespace process {
50 struct UPID;
51 }
52 
53 namespace mesos {
54 
55 class ObjectApprovers;
56 
57 namespace internal {
58 
59 namespace master {
60 // Forward declaration (in lieu of an include).
61 struct Framework;
62 struct Slave;
63 } // namespace master {
64 
65 namespace protobuf {
66 
68  const FrameworkInfo& framework,
70 
71 
72 // Returns whether the task state is terminal. Terminal states
73 // mean that the resources are released and the task cannot
74 // transition back to a non-terminal state. Note that
75 // `TASK_UNREACHABLE` is not a terminal state, but still
76 // releases the resources.
77 bool isTerminalState(const TaskState& state);
78 
79 
80 // See TaskStatus for more information about these fields. Note
81 // that the 'uuid' must be provided for updates that need
82 // acknowledgement. Currently, all slave and executor generated
83 // updates require acknowledgement, whereas master generated
84 // and scheduler driver generated updates do not.
85 StatusUpdate createStatusUpdate(
86  const FrameworkID& frameworkId,
87  const Option<SlaveID>& slaveId,
88  const TaskID& taskId,
89  const TaskState& state,
90  const TaskStatus::Source& source,
91  const Option<id::UUID>& uuid,
92  const std::string& message = "",
93  const Option<TaskStatus::Reason>& reason = None(),
94  const Option<ExecutorID>& executorId = None(),
95  const Option<bool>& healthy = None(),
96  const Option<CheckStatusInfo>& checkStatus = None(),
97  const Option<Labels>& labels = None(),
98  const Option<ContainerStatus>& containerStatus = None(),
99  const Option<TimeInfo>& unreachableTime = None(),
100  const Option<Resources>& limitedResources = None());
101 
102 
103 StatusUpdate createStatusUpdate(
104  const FrameworkID& frameworkId,
105  const TaskStatus& status,
106  const Option<SlaveID>& slaveId);
107 
108 
109 // Helper function that creates a new task status from scratch with
110 // obligatory fields set.
111 TaskStatus createTaskStatus(
112  const TaskID& taskId,
113  const TaskState& state,
114  const id::UUID& uuid,
115  double timestamp);
116 
117 
118 // Helper function that creates a new task status from the given task
119 // status. Specific fields in `status` can be overridden in the new
120 // status by specifying the appropriate argument. Fields `task_id`,
121 // `slave_id`, `executor_id`, cannot be changed; while `timestamp`
122 // and `uuid` cannot be preserved.
123 //
124 // NOTE: A task status update may be used for guaranteed delivery of
125 // some task-related information, e.g., task's health update. In this
126 // case, it is often desirable to preserve specific fields from the
127 // previous status update to avoid shadowing information that was
128 // delivered previously.
129 TaskStatus createTaskStatus(
130  TaskStatus status,
131  const id::UUID& uuid,
132  double timestamp,
133  const Option<TaskState>& state = None(),
134  const Option<std::string>& message = None(),
135  const Option<TaskStatus::Source>& source = None(),
136  const Option<TaskStatus::Reason>& reason = None(),
137  const Option<std::string>& data = None(),
138  const Option<bool>& healthy = None(),
139  const Option<CheckStatusInfo>& checkStatus = None(),
140  const Option<Labels>& labels = None(),
141  const Option<ContainerStatus>& containerStatus = None(),
142  const Option<TimeInfo>& unreachableTime = None());
143 
144 
145 Task createTask(
146  const TaskInfo& task,
147  const TaskState& state,
148  const FrameworkID& frameworkId);
149 
150 
151 Option<bool> getTaskHealth(const Task& task);
152 
153 
155 
156 
158 
159 
160 bool isTerminalState(const OperationState& state);
161 
162 
163 OperationStatus createOperationStatus(
164  const OperationState& state,
165  const Option<OperationID>& operationId = None(),
166  const Option<std::string>& message = None(),
167  const Option<Resources>& convertedResources = None(),
168  const Option<id::UUID>& statusUUID = None());
169 
170 
172  const Offer::Operation& info,
173  const OperationStatus& latestStatus,
174  const Option<FrameworkID>& frameworkId,
175  const Option<SlaveID>& slaveId,
176  const Option<UUID>& operationUUID = None());
177 
178 
179 UpdateOperationStatusMessage createUpdateOperationStatusMessage(
180  const UUID& operationUUID,
181  const OperationStatus& status,
182  const Option<OperationStatus>& latestStatus = None(),
183  const Option<FrameworkID>& frameworkId = None(),
184  const Option<SlaveID>& slaveId = None());
185 
186 
187 // Create a `UUID`. If `uuid` is given it is used to initialize
188 // the created `UUID`; otherwise a random `UUID` is returned.
189 UUID createUUID(const Option<id::UUID>& uuid = None());
190 
191 
192 // Helper function that creates a MasterInfo from UPID.
193 MasterInfo createMasterInfo(const process::UPID& pid);
194 
195 
196 Label createLabel(
197  const std::string& key,
198  const Option<std::string>& value = None());
199 
200 
201 // Helper function to convert a protobuf string map to `Labels`.
203  const google::protobuf::Map<std::string, std::string>& map);
204 
205 
206 // Helper function to convert a `Labels` to a protobuf string map.
208  const Labels& labels);
209 
210 
211 // Previously, `Resource` did not contain `AllocationInfo`.
212 // So for backwards compatibility with old schedulers and
213 // tooling, we must allow operations to contain `Resource`s
214 // without an `AllocationInfo`. This allows the master to
215 // inject the offer's `AllocationInfo` into the operation's
216 // resources.
218  Offer::Operation* operation,
219  const Resource::AllocationInfo& allocationInfo);
220 
221 
222 // This strips the Resource::AllocationInfo from all
223 // Resource objects contained within the operation.
224 void stripAllocationInfo(Offer::Operation* operation);
225 
226 
227 bool isSpeculativeOperation(const Offer::Operation& operation);
228 
229 
230 // Helper function to pack a protobuf list of resource versions.
231 google::protobuf::RepeatedPtrField<ResourceVersionUUID> createResourceVersions(
232  const hashmap<Option<ResourceProviderID>, UUID>& resourceVersions);
233 
234 
235 // Helper function to unpack a protobuf list of resource versions.
237  const google::protobuf::RepeatedPtrField<ResourceVersionUUID>&
238  resourceVersionUUIDs);
239 
240 
241 // Helper function that fills in a TimeInfo from the current time.
242 TimeInfo getCurrentTime();
243 
244 
245 // Helper function that creates a `FileInfo` from data returned by `stat()`.
246 FileInfo createFileInfo(const std::string& path, const struct stat& s);
247 
248 
249 ContainerID getRootContainerId(const ContainerID& containerId);
250 
251 
252 ContainerID parseContainerId(const std::string& value);
253 
254 
256 
257 namespace slave {
258 
259 // TODO(bmahler): Store the repeated field within this so that we
260 // don't drop unknown capabilities.
262 {
263  Capabilities() = default;
264 
265  template <typename Iterable>
266  Capabilities(const Iterable& capabilities)
267  {
268  foreach (const SlaveInfo::Capability& capability, capabilities) {
269  switch (capability.type()) {
271  break;
272  case SlaveInfo::Capability::MULTI_ROLE:
273  multiRole = true;
274  break;
275  case SlaveInfo::Capability::HIERARCHICAL_ROLE:
276  hierarchicalRole = true;
277  break;
278  case SlaveInfo::Capability::RESERVATION_REFINEMENT:
279  reservationRefinement = true;
280  break;
281  case SlaveInfo::Capability::RESOURCE_PROVIDER:
282  resourceProvider = true;
283  break;
284  case SlaveInfo::Capability::RESIZE_VOLUME:
285  resizeVolume = true;
286  break;
287  // If adding another case here be sure to update the
288  // equality operator.
289  }
290  }
291  }
292 
293  // See mesos.proto for the meaning of agent capabilities.
294  bool multiRole = false;
295  bool hierarchicalRole = false;
296  bool reservationRefinement = false;
297  bool resourceProvider = false;
298  bool resizeVolume = false;
299 
300  google::protobuf::RepeatedPtrField<SlaveInfo::Capability>
302  {
303  google::protobuf::RepeatedPtrField<SlaveInfo::Capability> result;
304  if (multiRole) {
305  result.Add()->set_type(SlaveInfo::Capability::MULTI_ROLE);
306  }
307  if (hierarchicalRole) {
308  result.Add()->set_type(SlaveInfo::Capability::HIERARCHICAL_ROLE);
309  }
310  if (reservationRefinement) {
311  result.Add()->set_type(SlaveInfo::Capability::RESERVATION_REFINEMENT);
312  }
313  if (resourceProvider) {
314  result.Add()->set_type(SlaveInfo::Capability::RESOURCE_PROVIDER);
315  }
316  if (resizeVolume) {
317  result.Add()->set_type(SlaveInfo::Capability::RESIZE_VOLUME);
318  }
319 
320  return result;
321  }
322 };
323 
324 
325 bool operator==(const Capabilities& left, const Capabilities& right);
326 bool operator!=(const Capabilities& left, const Capabilities& right);
327 std::ostream& operator<<(std::ostream& stream, const Capabilities& c);
328 
329 
330 mesos::slave::ContainerLimitation createContainerLimitation(
331  const Resources& resources,
332  const std::string& message,
333  const TaskStatus::Reason& reason);
334 
335 
336 mesos::slave::ContainerState createContainerState(
337  const Option<ExecutorInfo>& executorInfo,
338  const ContainerID& id,
339  pid_t pid,
340  const std::string& directory);
341 
342 } // namespace slave {
343 
344 namespace maintenance {
345 
349 Unavailability createUnavailability(
350  const process::Time& start,
351  const Option<Duration>& duration = None());
352 
353 
357 google::protobuf::RepeatedPtrField<MachineID> createMachineList(
358  std::initializer_list<MachineID> ids);
359 
360 
365 mesos::maintenance::Window createWindow(
366  std::initializer_list<MachineID> ids,
367  const Unavailability& unavailability);
368 
369 
374 mesos::maintenance::Schedule createSchedule(
375  std::initializer_list<mesos::maintenance::Window> windows);
376 
377 } // namespace maintenance {
378 
379 
380 namespace master {
381 
382 // TODO(bmahler): Store the repeated field within this so that we
383 // don't drop unknown capabilities.
385 {
386  Capabilities() = default;
387 
388  template <typename Iterable>
389  Capabilities(const Iterable& capabilities)
390  {
391  foreach (const MasterInfo::Capability& capability, capabilities) {
392  switch (capability.type()) {
394  break;
395  case MasterInfo::Capability::AGENT_UPDATE:
396  agentUpdate = true;
397  break;
398  }
399  }
400  }
401 
402  bool agentUpdate = false;
403 };
404 
405 namespace event {
406 
407 // Helper for creating a `TASK_UPDATED` event from a `Task`, its
408 // latest state according to the agent, and its status corresponding
409 // to the last status update acknowledged from the scheduler.
411  const Task& task,
412  const TaskState& state,
413  const TaskStatus& status);
414 
415 
416 // Helper for creating a `TASK_ADDED` event from a `Task`.
417 mesos::master::Event createTaskAdded(const Task& task);
418 
419 
420 // Helper for creating a 'FRAMEWORK_ADDED' event from a `Framework`.
422  const mesos::internal::master::Framework& framework);
423 
424 
425 // Helper for creating a 'FRAMEWORK_UPDATED' event from a `Framework`.
427  const mesos::internal::master::Framework& framework);
428 
429 
430 // Helper for creating a 'FRAMEWORK_REMOVED' event from a `FrameworkInfo`.
431 mesos::master::Event createFrameworkRemoved(const FrameworkInfo& frameworkInfo);
432 
433 
434 // Helper for creating an `Agent` response.
435 mesos::master::Response::GetAgents::Agent createAgentResponse(
436  const mesos::internal::master::Slave& slave,
437  const Option<process::Owned<ObjectApprovers>>& approvers = None());
438 
439 
440 // Helper for creating an `AGENT_ADDED` event from a `Slave`.
442  const mesos::internal::master::Slave& slave);
443 
444 
445 // Helper for creating an `AGENT_REMOVED` event from a `SlaveID`.
446 mesos::master::Event createAgentRemoved(const SlaveID& slaveId);
447 
448 } // namespace event {
449 } // namespace master {
450 
451 namespace framework {
452 
453 // TODO(bmahler): Store the repeated field within this so that we
454 // don't drop unknown capabilities.
456 {
457  Capabilities() = default;
458 
459  template <typename Iterable>
460  Capabilities(const Iterable& capabilities)
461  {
462  foreach (const FrameworkInfo::Capability& capability, capabilities) {
463  switch (capability.type()) {
465  break;
466  case FrameworkInfo::Capability::REVOCABLE_RESOURCES:
467  revocableResources = true;
468  break;
469  case FrameworkInfo::Capability::TASK_KILLING_STATE:
470  taskKillingState = true;
471  break;
472  case FrameworkInfo::Capability::GPU_RESOURCES:
473  gpuResources = true;
474  break;
475  case FrameworkInfo::Capability::SHARED_RESOURCES:
476  sharedResources = true;
477  break;
478  case FrameworkInfo::Capability::PARTITION_AWARE:
479  partitionAware = true;
480  break;
481  case FrameworkInfo::Capability::MULTI_ROLE:
482  multiRole = true;
483  break;
484  case FrameworkInfo::Capability::RESERVATION_REFINEMENT:
485  reservationRefinement = true;
486  break;
487  case FrameworkInfo::Capability::REGION_AWARE:
488  regionAware = true;
489  break;
490  }
491  }
492  }
493 
494  // See mesos.proto for the meaning of these capabilities.
495  bool revocableResources = false;
496  bool taskKillingState = false;
497  bool gpuResources = false;
498  bool sharedResources = false;
499  bool partitionAware = false;
500  bool multiRole = false;
501  bool reservationRefinement = false;
502  bool regionAware = false;
503 };
504 
505 
506 // Helper to get roles from FrameworkInfo based on the
507 // presence of the MULTI_ROLE capability.
508 std::set<std::string> getRoles(const FrameworkInfo& frameworkInfo);
509 
510 } // namespace framework {
511 
512 } // namespace protobuf {
513 } // namespace internal {
514 } // namespace mesos {
515 
516 #endif // __PROTOBUF_UTILS_HPP__
hashmap< Option< ResourceProviderID >, UUID > parseResourceVersions(const google::protobuf::RepeatedPtrField< ResourceVersionUUID > &resourceVersionUUIDs)
Definition: path.hpp:26
UpdateOperationStatusMessage createUpdateOperationStatusMessage(const UUID &operationUUID, const OperationStatus &status, const Option< OperationStatus > &latestStatus=None(), const Option< FrameworkID > &frameworkId=None(), const Option< SlaveID > &slaveId=None())
Capabilities(const Iterable &capabilities)
Definition: protobuf_utils.hpp:389
Try< Resources > getConsumedResources(const Offer::Operation &operation)
Option< bool > getTaskHealth(const Task &task)
mesos::master::Event createFrameworkAdded(const mesos::internal::master::Framework &framework)
std::ostream & operator<<(std::ostream &stream, const Future< T > &future)
Definition: future.hpp:1826
Definition: option.hpp:28
void stripAllocationInfo(Offer::Operation *operation)
bool operator==(const std::string &s, const UPID::ID &id)
Definition: pid.hpp:216
Definition: master.hpp:27
Definition: check.hpp:33
Labels convertStringMapToLabels(const google::protobuf::Map< std::string, std::string > &map)
google::protobuf::RepeatedPtrField< MachineID > createMachineList(std::initializer_list< MachineID > ids)
Helper for constructing a list of MachineID.
void injectAllocationInfo(Offer::Operation *operation, const Resource::AllocationInfo &allocationInfo)
Definition: protobuf_utils.hpp:261
Unavailability createUnavailability(const process::Time &start, const Option< Duration > &duration=None())
Helper for constructing an unavailability from a Time and Duration.
std::set< std::string > getRoles(const FrameworkInfo &frameworkInfo)
Capabilities(const Iterable &capabilities)
Definition: protobuf_utils.hpp:266
Result< ProcessStatus > status(pid_t pid)
Definition: proc.hpp:166
mesos::slave::ContainerState createContainerState(const Option< ExecutorInfo > &executorInfo, const ContainerID &id, pid_t pid, const std::string &directory)
Definition: resources.hpp:81
mesos::master::Event createFrameworkUpdated(const mesos::internal::master::Framework &framework)
ContainerID getRootContainerId(const ContainerID &containerId)
mesos::maintenance::Schedule createSchedule(std::initializer_list< mesos::maintenance::Window > windows)
Helper for constructing a maintenance Schedule.
Capability
Definition: capabilities.hpp:35
Try< google::protobuf::Map< std::string, std::string > > convertLabelsToStringMap(const Labels &labels)
Operation
Definition: cgroups.hpp:458
mesos::master::Event createAgentAdded(const mesos::internal::master::Slave &slave)
Try< Nothing > start(const std::string &name)
Starts the slice with the given name (via &#39;systemctl start <name>&#39;).
Definition: protobuf_utils.hpp:384
Capabilities(const Iterable &capabilities)
Definition: protobuf_utils.hpp:460
mesos::v1::scheduler::Event Event
Definition: mesos.hpp:2617
Definition: hashmap.hpp:38
mesos::master::Event createTaskAdded(const Task &task)
DWORD pid_t
Definition: windows.hpp:181
mesos::slave::ContainerLimitation createContainerLimitation(const Resources &resources, const std::string &message, const TaskStatus::Reason &reason)
FileInfo createFileInfo(const std::string &path, const struct stat &s)
mesos::master::Event createTaskUpdated(const Task &task, const TaskState &state, const TaskStatus &status)
mesos::master::Event createAgentRemoved(const SlaveID &slaveId)
Try< Nothing > unavailability(const Unavailability &unavailability)
An "untyped" PID, used to encapsulate the process ID for lower-layer abstractions (eg...
Definition: pid.hpp:39
StatusUpdate createStatusUpdate(const FrameworkID &frameworkId, const TaskStatus &status, const Option< SlaveID > &slaveId)
Option< ContainerStatus > getTaskContainerStatus(const Task &task)
Definition: uuid.hpp:35
Definition: protobuf_utils.hpp:455
Definition: spec.hpp:26
Task createTask(const TaskInfo &task, const TaskState &state, const FrameworkID &frameworkId)
const int UNKNOWN
Definition: diagnosis.hpp:39
google::protobuf::RepeatedPtrField< SlaveInfo::Capability > toRepeatedPtrField() const
Definition: protobuf_utils.hpp:301
Definition: protobuf.hpp:59
Try< hashmap< std::string, uint64_t > > stat(const std::string &hierarchy, const std::string &cgroup, const std::string &file)
Definition: time.hpp:23
Operation createOperation(const Offer::Operation &info, const OperationStatus &latestStatus, const Option< FrameworkID > &frameworkId, const Option< SlaveID > &slaveId, const Option< UUID > &operationUUID=None())
bool isTerminalState(const OperationState &state)
Iterable< V > map(F &&f, const Iterable< U, Us... > &input)
Definition: lambda.hpp:46
Definition: none.hpp:27
Definition: attributes.hpp:24
bool operator!=(const std::string &s, const UPID::ID &id)
Definition: pid.hpp:222
mesos::master::Event createFrameworkRemoved(const FrameworkInfo &frameworkInfo)
mesos::master::Response::GetAgents::Agent createAgentResponse(const mesos::internal::master::Slave &slave, const Option< process::Owned< ObjectApprovers >> &approvers=None())
Definition: executor.hpp:48
Type
Definition: capabilities.hpp:79
Definition: master.hpp:116
mesos::maintenance::Window createWindow(std::initializer_list< MachineID > ids, const Unavailability &unavailability)
Helper for constructing a maintenance Window.
TaskStatus createTaskStatus(TaskStatus status, const id::UUID &uuid, double timestamp, const Option< TaskState > &state=None(), const Option< std::string > &message=None(), const Option< TaskStatus::Source > &source=None(), const Option< TaskStatus::Reason > &reason=None(), const Option< std::string > &data=None(), const Option< bool > &healthy=None(), const Option< CheckStatusInfo > &checkStatus=None(), const Option< Labels > &labels=None(), const Option< ContainerStatus > &containerStatus=None(), const Option< TimeInfo > &unreachableTime=None())
bool frameworkHasCapability(const FrameworkInfo &framework, FrameworkInfo::Capability::Type capability)
bool isSpeculativeOperation(const Offer::Operation &operation)
Label createLabel(const std::string &key, const Option< std::string > &value=None())
Definition: owned.hpp:36
Definition: master.hpp:2361
MasterInfo createMasterInfo(const process::UPID &pid)
Option< CheckStatusInfo > getTaskCheckStatus(const Task &task)
OperationStatus createOperationStatus(const OperationState &state, const Option< OperationID > &operationId=None(), const Option< std::string > &message=None(), const Option< Resources > &convertedResources=None(), const Option< id::UUID > &statusUUID=None())
ContainerID parseContainerId(const std::string &value)
UUID createUUID(const Option< id::UUID > &uuid=None())
google::protobuf::RepeatedPtrField< ResourceVersionUUID > createResourceVersions(const hashmap< Option< ResourceProviderID >, UUID > &resourceVersions)