Apache Mesos
protobuf_utils.hpp
Go to the documentation of this file.
1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 
17 #ifndef __PROTOBUF_UTILS_HPP__
18 #define __PROTOBUF_UTILS_HPP__
19 
20 #include <initializer_list>
21 #include <ostream>
22 #include <set>
23 #include <string>
24 
25 #include <sys/stat.h>
26 
27 #include <mesos/mesos.hpp>
28 
30 
31 #include <mesos/master/master.hpp>
32 
33 #include <mesos/slave/isolator.hpp>
34 
35 #include <process/time.hpp>
36 
37 #include <stout/duration.hpp>
38 #include <stout/ip.hpp>
39 #include <stout/none.hpp>
40 #include <stout/option.hpp>
41 #include <stout/try.hpp>
42 #include <stout/uuid.hpp>
43 
44 #include "messages/messages.hpp"
45 
46 // Forward declaration (in lieu of an include).
47 namespace process {
48 struct UPID;
49 }
50 
51 namespace mesos {
52 
53 class AuthorizationAcceptor;
54 
55 namespace internal {
56 
57 namespace master {
58 // Forward declaration (in lieu of an include).
59 struct Framework;
60 struct Slave;
61 } // namespace master {
62 
63 namespace protobuf {
64 
66  const FrameworkInfo& framework,
68 
69 
70 bool isTerminalState(const TaskState& state);
71 
72 
73 // See TaskStatus for more information about these fields. Note
74 // that the 'uuid' must be provided for updates that need
75 // acknowledgement. Currently, all slave and executor generated
76 // updates require acknowledgement, whereas master generated
77 // and scheduler driver generated updates do not.
78 StatusUpdate createStatusUpdate(
79  const FrameworkID& frameworkId,
80  const Option<SlaveID>& slaveId,
81  const TaskID& taskId,
82  const TaskState& state,
83  const TaskStatus::Source& source,
84  const Option<id::UUID>& uuid,
85  const std::string& message = "",
86  const Option<TaskStatus::Reason>& reason = None(),
87  const Option<ExecutorID>& executorId = None(),
88  const Option<bool>& healthy = None(),
89  const Option<CheckStatusInfo>& checkStatus = None(),
90  const Option<Labels>& labels = None(),
91  const Option<ContainerStatus>& containerStatus = None(),
92  const Option<TimeInfo>& unreachableTime = None(),
93  const Option<Resources>& limitedResources = None());
94 
95 
96 StatusUpdate createStatusUpdate(
97  const FrameworkID& frameworkId,
98  const TaskStatus& status,
99  const Option<SlaveID>& slaveId);
100 
101 
102 // Helper function that creates a new task status from scratch with
103 // obligatory fields set.
104 TaskStatus createTaskStatus(
105  const TaskID& taskId,
106  const TaskState& state,
107  const id::UUID& uuid,
108  double timestamp);
109 
110 
111 // Helper function that creates a new task status from the given task
112 // status. Specific fields in `status` can be overridden in the new
113 // status by specifying the appropriate argument. Fields `task_id`,
114 // `slave_id`, `executor_id`, cannot be changed; while `timestamp`
115 // and `uuid` cannot be preserved.
116 //
117 // NOTE: A task status update may be used for guaranteed delivery of
118 // some task-related information, e.g., task's health update. In this
119 // case, it is often desirable to preserve specific fields from the
120 // previous status update to avoid shadowing information that was
121 // delivered previously.
122 TaskStatus createTaskStatus(
123  TaskStatus status,
124  const id::UUID& uuid,
125  double timestamp,
126  const Option<TaskState>& state = None(),
127  const Option<std::string>& message = None(),
128  const Option<TaskStatus::Source>& source = None(),
129  const Option<TaskStatus::Reason>& reason = None(),
130  const Option<std::string>& data = None(),
131  const Option<bool>& healthy = None(),
132  const Option<CheckStatusInfo>& checkStatus = None(),
133  const Option<Labels>& labels = None(),
134  const Option<ContainerStatus>& containerStatus = None(),
135  const Option<TimeInfo>& unreachableTime = None());
136 
137 
138 Task createTask(
139  const TaskInfo& task,
140  const TaskState& state,
141  const FrameworkID& frameworkId);
142 
143 
144 Option<bool> getTaskHealth(const Task& task);
145 
146 
148 
149 
151 
152 
153 bool isTerminalState(const OperationState& state);
154 
155 
156 OperationStatus createOperationStatus(
157  const OperationState& state,
158  const Option<OperationID>& operationId = None(),
159  const Option<std::string>& message = None(),
160  const Option<Resources>& convertedResources = None(),
161  const Option<id::UUID>& statusUUID = None());
162 
163 
165  const Offer::Operation& info,
166  const OperationStatus& latestStatus,
167  const Option<FrameworkID>& frameworkId,
168  const Option<SlaveID>& slaveId,
169  const Option<id::UUID>& operationUUID = None());
170 
171 
172 UpdateOperationStatusMessage createUpdateOperationStatusMessage(
173  const id::UUID& operationUUID,
174  const OperationStatus& status,
175  const Option<OperationStatus>& latestStatus = None(),
176  const Option<FrameworkID>& frameworkId = None(),
177  const Option<SlaveID>& slaveId = None());
178 
179 
180 // Helper function that creates a MasterInfo from UPID.
181 MasterInfo createMasterInfo(const process::UPID& pid);
182 
183 
184 Label createLabel(
185  const std::string& key,
186  const Option<std::string>& value = None());
187 
188 
189 // Helper function to convert a protobuf string map to `Labels`.
191  const google::protobuf::Map<std::string, std::string>& map);
192 
193 
194 // Helper function to convert a `Labels` to a protobuf string map.
196  const Labels& labels);
197 
198 
199 // Previously, `Resource` did not contain `AllocationInfo`.
200 // So for backwards compatibility with old schedulers and
201 // tooling, we must allow operations to contain `Resource`s
202 // without an `AllocationInfo`. This allows the master to
203 // inject the offer's `AllocationInfo` into the operation's
204 // resources.
206  Offer::Operation* operation,
207  const Resource::AllocationInfo& allocationInfo);
208 
209 
210 // This strips the Resource::AllocationInfo from all
211 // Resource objects contained within the operation.
212 void stripAllocationInfo(Offer::Operation* operation);
213 
214 
215 bool isSpeculativeOperation(const Offer::Operation& operation);
216 
217 
218 // Helper function to pack a protobuf list of resource versions.
219 google::protobuf::RepeatedPtrField<ResourceVersionUUID> createResourceVersions(
220  const hashmap<Option<ResourceProviderID>, id::UUID>& resourceVersions);
221 
222 
223 // Helper function to unpack a protobuf list of resource versions.
225  const google::protobuf::RepeatedPtrField<ResourceVersionUUID>&
226  resourceVersionUUIDs);
227 
228 
229 // Helper function that fills in a TimeInfo from the current time.
230 TimeInfo getCurrentTime();
231 
232 
233 // Helper function that creates a `FileInfo` from data returned by `stat()`.
234 FileInfo createFileInfo(const std::string& path, const struct stat& s);
235 
236 
237 ContainerID getRootContainerId(const ContainerID& containerId);
238 
239 
241 
242 namespace slave {
243 
244 // TODO(bmahler): Store the repeated field within this so that we
245 // don't drop unknown capabilities.
247 {
248  Capabilities() = default;
249 
250  template <typename Iterable>
251  Capabilities(const Iterable& capabilities)
252  {
253  foreach (const SlaveInfo::Capability& capability, capabilities) {
254  switch (capability.type()) {
256  break;
257  case SlaveInfo::Capability::MULTI_ROLE:
258  multiRole = true;
259  break;
260  case SlaveInfo::Capability::HIERARCHICAL_ROLE:
261  hierarchicalRole = true;
262  break;
263  case SlaveInfo::Capability::RESERVATION_REFINEMENT:
264  reservationRefinement = true;
265  break;
266  case SlaveInfo::Capability::RESOURCE_PROVIDER:
267  resourceProvider = true;
268  break;
269  // If adding another case here be sure to update the
270  // equality operator.
271  }
272  }
273  }
274 
275  // See mesos.proto for the meaning of agent capabilities.
276  bool multiRole = false;
277  bool hierarchicalRole = false;
278  bool reservationRefinement = false;
279  bool resourceProvider = false;
280 
281  google::protobuf::RepeatedPtrField<SlaveInfo::Capability>
283  {
284  google::protobuf::RepeatedPtrField<SlaveInfo::Capability> result;
285  if (multiRole) {
286  result.Add()->set_type(SlaveInfo::Capability::MULTI_ROLE);
287  }
288  if (hierarchicalRole) {
289  result.Add()->set_type(SlaveInfo::Capability::HIERARCHICAL_ROLE);
290  }
291  if (reservationRefinement) {
292  result.Add()->set_type(SlaveInfo::Capability::RESERVATION_REFINEMENT);
293  }
294  if (resourceProvider) {
295  result.Add()->set_type(SlaveInfo::Capability::RESOURCE_PROVIDER);
296  }
297 
298  return result;
299  }
300 };
301 
302 
303 bool operator==(const Capabilities& left, const Capabilities& right);
304 bool operator!=(const Capabilities& left, const Capabilities& right);
305 std::ostream& operator<<(std::ostream& stream, const Capabilities& c);
306 
307 
308 mesos::slave::ContainerLimitation createContainerLimitation(
309  const Resources& resources,
310  const std::string& message,
311  const TaskStatus::Reason& reason);
312 
313 
314 mesos::slave::ContainerState createContainerState(
315  const Option<ExecutorInfo>& executorInfo,
316  const ContainerID& id,
317  pid_t pid,
318  const std::string& directory);
319 
320 } // namespace slave {
321 
322 namespace maintenance {
323 
327 Unavailability createUnavailability(
328  const process::Time& start,
329  const Option<Duration>& duration = None());
330 
331 
335 google::protobuf::RepeatedPtrField<MachineID> createMachineList(
336  std::initializer_list<MachineID> ids);
337 
338 
343 mesos::maintenance::Window createWindow(
344  std::initializer_list<MachineID> ids,
345  const Unavailability& unavailability);
346 
347 
352 mesos::maintenance::Schedule createSchedule(
353  std::initializer_list<mesos::maintenance::Window> windows);
354 
355 } // namespace maintenance {
356 
357 
358 namespace master {
359 
360 // TODO(bmahler): Store the repeated field within this so that we
361 // don't drop unknown capabilities.
363 {
364  Capabilities() = default;
365 
366  template <typename Iterable>
367  Capabilities(const Iterable& capabilities)
368  {
369  foreach (const MasterInfo::Capability& capability, capabilities) {
370  switch (capability.type()) {
372  break;
373  case MasterInfo::Capability::AGENT_UPDATE:
374  agentUpdate = true;
375  break;
376  }
377  }
378  }
379 
380  bool agentUpdate = false;
381 };
382 
383 namespace event {
384 
385 // Helper for creating a `TASK_UPDATED` event from a `Task`, its
386 // latest state according to the agent, and its status corresponding
387 // to the last status update acknowledged from the scheduler.
389  const Task& task,
390  const TaskState& state,
391  const TaskStatus& status);
392 
393 
394 // Helper for creating a `TASK_ADDED` event from a `Task`.
395 mesos::master::Event createTaskAdded(const Task& task);
396 
397 
398 // Helper for creating a 'FRAMEWORK_ADDED' event from a `Framework`.
400  const mesos::internal::master::Framework& framework);
401 
402 
403 // Helper for creating a 'FRAMEWORK_UPDATED' event from a `Framework`.
405  const mesos::internal::master::Framework& framework);
406 
407 
408 // Helper for creating a 'FRAMEWORK_REMOVED' event from a `FrameworkInfo`.
409 mesos::master::Event createFrameworkRemoved(const FrameworkInfo& frameworkInfo);
410 
411 
412 // Helper for creating an `Agent` response.
413 mesos::master::Response::GetAgents::Agent createAgentResponse(
414  const mesos::internal::master::Slave& slave,
415  const Option<process::Owned<AuthorizationAcceptor>>& rolesAcceptor =
416  None());
417 
418 
419 // Helper for creating an `AGENT_ADDED` event from a `Slave`.
421  const mesos::internal::master::Slave& slave);
422 
423 
424 // Helper for creating an `AGENT_REMOVED` event from a `SlaveID`.
425 mesos::master::Event createAgentRemoved(const SlaveID& slaveId);
426 
427 } // namespace event {
428 } // namespace master {
429 
430 namespace framework {
431 
432 // TODO(bmahler): Store the repeated field within this so that we
433 // don't drop unknown capabilities.
435 {
436  Capabilities() = default;
437 
438  template <typename Iterable>
439  Capabilities(const Iterable& capabilities)
440  {
441  foreach (const FrameworkInfo::Capability& capability, capabilities) {
442  switch (capability.type()) {
444  break;
445  case FrameworkInfo::Capability::REVOCABLE_RESOURCES:
446  revocableResources = true;
447  break;
448  case FrameworkInfo::Capability::TASK_KILLING_STATE:
449  taskKillingState = true;
450  break;
451  case FrameworkInfo::Capability::GPU_RESOURCES:
452  gpuResources = true;
453  break;
454  case FrameworkInfo::Capability::SHARED_RESOURCES:
455  sharedResources = true;
456  break;
457  case FrameworkInfo::Capability::PARTITION_AWARE:
458  partitionAware = true;
459  break;
460  case FrameworkInfo::Capability::MULTI_ROLE:
461  multiRole = true;
462  break;
463  case FrameworkInfo::Capability::RESERVATION_REFINEMENT:
464  reservationRefinement = true;
465  break;
466  case FrameworkInfo::Capability::REGION_AWARE:
467  regionAware = true;
468  break;
469  }
470  }
471  }
472 
473  // See mesos.proto for the meaning of these capabilities.
474  bool revocableResources = false;
475  bool taskKillingState = false;
476  bool gpuResources = false;
477  bool sharedResources = false;
478  bool partitionAware = false;
479  bool multiRole = false;
480  bool reservationRefinement = false;
481  bool regionAware = false;
482 };
483 
484 
485 // Helper to get roles from FrameworkInfo based on the
486 // presence of the MULTI_ROLE capability.
487 std::set<std::string> getRoles(const FrameworkInfo& frameworkInfo);
488 
489 } // namespace framework {
490 
491 } // namespace protobuf {
492 } // namespace internal {
493 } // namespace mesos {
494 
495 #endif // __PROTOBUF_UTILS_HPP__
hashmap< Option< ResourceProviderID >, id::UUID > parseResourceVersions(const google::protobuf::RepeatedPtrField< ResourceVersionUUID > &resourceVersionUUIDs)
Capabilities(const Iterable &capabilities)
Definition: protobuf_utils.hpp:367
Try< Resources > getConsumedResources(const Offer::Operation &operation)
Option< bool > getTaskHealth(const Task &task)
bool multiRole
Definition: protobuf_utils.hpp:276
mesos::master::Event createFrameworkAdded(const mesos::internal::master::Framework &framework)
Definition: option.hpp:28
StatusUpdate createStatusUpdate(const FrameworkID &frameworkId, const Option< SlaveID > &slaveId, const TaskID &taskId, const TaskState &state, const TaskStatus::Source &source, const Option< id::UUID > &uuid, const std::string &message="", const Option< TaskStatus::Reason > &reason=None(), const Option< ExecutorID > &executorId=None(), const Option< bool > &healthy=None(), const Option< CheckStatusInfo > &checkStatus=None(), const Option< Labels > &labels=None(), const Option< ContainerStatus > &containerStatus=None(), const Option< TimeInfo > &unreachableTime=None(), const Option< Resources > &limitedResources=None())
void stripAllocationInfo(Offer::Operation *operation)
mesos::master::Response::GetAgents::Agent createAgentResponse(const mesos::internal::master::Slave &slave, const Option< process::Owned< AuthorizationAcceptor >> &rolesAcceptor=None())
bool resourceProvider
Definition: protobuf_utils.hpp:279
bool reservationRefinement
Definition: protobuf_utils.hpp:480
Definition: try.hpp:34
Labels convertStringMapToLabels(const google::protobuf::Map< std::string, std::string > &map)
google::protobuf::RepeatedPtrField< MachineID > createMachineList(std::initializer_list< MachineID > ids)
Helper for constructing a list of MachineID.
bool taskKillingState
Definition: protobuf_utils.hpp:475
void injectAllocationInfo(Offer::Operation *operation, const Resource::AllocationInfo &allocationInfo)
Definition: protobuf_utils.hpp:246
Unavailability createUnavailability(const process::Time &start, const Option< Duration > &duration=None())
Helper for constructing an unavailability from a Time and Duration.
std::set< std::string > getRoles(const FrameworkInfo &frameworkInfo)
Capabilities(const Iterable &capabilities)
Definition: protobuf_utils.hpp:251
Result< ProcessStatus > status(pid_t pid)
Definition: proc.hpp:166
mesos::slave::ContainerState createContainerState(const Option< ExecutorInfo > &executorInfo, const ContainerID &id, pid_t pid, const std::string &directory)
TaskStatus createTaskStatus(const TaskID &taskId, const TaskState &state, const id::UUID &uuid, double timestamp)
Definition: resources.hpp:79
mesos::master::Event createFrameworkUpdated(const mesos::internal::master::Framework &framework)
bool revocableResources
Definition: protobuf_utils.hpp:474
ContainerID getRootContainerId(const ContainerID &containerId)
mesos::maintenance::Schedule createSchedule(std::initializer_list< mesos::maintenance::Window > windows)
Helper for constructing a maintenance Schedule.
Capability
Definition: capabilities.hpp:35
Try< google::protobuf::Map< std::string, std::string > > convertLabelsToStringMap(const Labels &labels)
Operation
Definition: cgroups.hpp:441
mesos::master::Event createAgentAdded(const mesos::internal::master::Slave &slave)
Try< Nothing > start(const std::string &name)
Starts the slice with the given name (via &#39;systemctl start &lt;name&gt;&#39;).
Definition: protobuf_utils.hpp:362
Capabilities(const Iterable &capabilities)
Definition: protobuf_utils.hpp:439
mesos::v1::scheduler::Event Event
Definition: mesos.hpp:2584
bool partitionAware
Definition: protobuf_utils.hpp:478
Definition: hashmap.hpp:38
mesos::master::Event createTaskAdded(const Task &task)
DWORD pid_t
Definition: windows.hpp:187
mesos::slave::ContainerLimitation createContainerLimitation(const Resources &resources, const std::string &message, const TaskStatus::Reason &reason)
FileInfo createFileInfo(const std::string &path, const struct stat &s)
mesos::master::Event createTaskUpdated(const Task &task, const TaskState &state, const TaskStatus &status)
bool isTerminalState(const TaskState &state)
mesos::master::Event createAgentRemoved(const SlaveID &slaveId)
Try< Nothing > unavailability(const Unavailability &unavailability)
An &quot;untyped&quot; PID, used to encapsulate the process ID for lower-layer abstractions (eg...
Definition: pid.hpp:39
Iterable< V > map(F &&f, const Iterable< U, Us...> &input)
Definition: lambda.hpp:45
bool agentUpdate
Definition: protobuf_utils.hpp:380
Operation createOperation(const Offer::Operation &info, const OperationStatus &latestStatus, const Option< FrameworkID > &frameworkId, const Option< SlaveID > &slaveId, const Option< id::UUID > &operationUUID=None())
Option< ContainerStatus > getTaskContainerStatus(const Task &task)
Definition: uuid.hpp:35
Definition: protobuf_utils.hpp:434
Task createTask(const TaskInfo &task, const TaskState &state, const FrameworkID &frameworkId)
const int UNKNOWN
Definition: diagnosis.hpp:39
google::protobuf::RepeatedPtrField< SlaveInfo::Capability > toRepeatedPtrField() const
Definition: protobuf_utils.hpp:282
UpdateOperationStatusMessage createUpdateOperationStatusMessage(const id::UUID &operationUUID, const OperationStatus &status, const Option< OperationStatus > &latestStatus=None(), const Option< FrameworkID > &frameworkId=None(), const Option< SlaveID > &slaveId=None())
Try< hashmap< std::string, uint64_t > > stat(const std::string &hierarchy, const std::string &cgroup, const std::string &file)
Definition: time.hpp:23
bool sharedResources
Definition: protobuf_utils.hpp:477
bool multiRole
Definition: protobuf_utils.hpp:479
bool reservationRefinement
Definition: protobuf_utils.hpp:278
bool operator==(const Capabilities &left, const Capabilities &right)
bool gpuResources
Definition: protobuf_utils.hpp:476
Result< Process > process(pid_t pid)
Definition: freebsd.hpp:30
bool hierarchicalRole
Definition: protobuf_utils.hpp:277
Definition: none.hpp:27
mesos::master::Event createFrameworkRemoved(const FrameworkInfo &frameworkInfo)
Type
Definition: capabilities.hpp:79
Object protobuf(const google::protobuf::Message &message)
Definition: protobuf.hpp:836
Definition: master.hpp:117
mesos::maintenance::Window createWindow(std::initializer_list< MachineID > ids, const Unavailability &unavailability)
Helper for constructing a maintenance Window.
bool frameworkHasCapability(const FrameworkInfo &framework, FrameworkInfo::Capability::Type capability)
bool operator!=(const Capabilities &left, const Capabilities &right)
bool isSpeculativeOperation(const Offer::Operation &operation)
Label createLabel(const std::string &key, const Option< std::string > &value=None())
Definition: owned.hpp:35
Definition: master.hpp:2155
MasterInfo createMasterInfo(const process::UPID &pid)
Option< CheckStatusInfo > getTaskCheckStatus(const Task &task)
std::ostream & operator<<(std::ostream &stream, const Capabilities &c)
OperationStatus createOperationStatus(const OperationState &state, const Option< OperationID > &operationId=None(), const Option< std::string > &message=None(), const Option< Resources > &convertedResources=None(), const Option< id::UUID > &statusUUID=None())
bool regionAware
Definition: protobuf_utils.hpp:481
google::protobuf::RepeatedPtrField< ResourceVersionUUID > createResourceVersions(const hashmap< Option< ResourceProviderID >, id::UUID > &resourceVersions)