Apache Mesos
resources_utils.hpp
Go to the documentation of this file.
1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 
17 #ifndef __RESOURCES_UTILS_HPP__
18 #define __RESOURCES_UTILS_HPP__
19 
20 #include <vector>
21 
22 #include <google/protobuf/repeated_field.h>
23 
24 #include <mesos/mesos.hpp>
26 #include <mesos/resources.hpp>
27 
28 #include <mesos/v1/mesos.hpp>
29 #include <mesos/v1/resources.hpp>
30 
31 #include <stout/error.hpp>
32 #include <stout/nothing.hpp>
33 #include <stout/option.hpp>
34 #include <stout/try.hpp>
35 
36 namespace mesos {
37 
38 // Tests if the given Resource needs to be checkpointed on the slave.
39 // NOTE: We assume the given resource is validated.
40 bool needCheckpointing(const Resource& resource);
41 
42 // Returns the total resources by applying the given checkpointed
43 // resources to the given resources. This function is useful when we
44 // want to calculate the total resources of a slave from the resources
45 // specified from the command line and the checkpointed resources.
46 // Returns error if the given resources are not compatible with the
47 // given checkpointed resources.
49  const Resources& resources,
50  const Resources& checkpointedResources);
51 
52 
53 // Returns the resource provider ID associated with the given
54 // operation. Returns None if the operation is for agent default
55 // resources. We assume the given operation is validated. Therefore,
56 // the specified operation should not contain resources from more than
57 // one resource provider.
59  const Offer::Operation& operation);
60 
61 
62 // Returns the resource provider ID associated with the given resources, None if
63 // the given resources are agent default resources, or Error if the given
64 // resources are from more than one resource providers.
66  const Resources& resources);
67 
68 
69 // Returns the resource conversions from the given offer operation.
70 // This helper assumes that the given operation has already been
71 // validated.
73  const Offer::Operation& operation);
74 
75 
76 // Returns the resource conversions from the given offer operation.
77 // This helper assumes that the given operation has already been
78 // validated.
80  const v1::Offer::Operation& operation);
81 
82 
83 // Resource format options to be used with the `convertResourceFormat` function.
84 //
85 // The preconditions of the options are asymmetric, centered around the
86 // "post-reservation-refinement" format. This is mainly due to the fact that
87 // "post-reservation-refinement" format is our canonical representation.
88 // The transformations are generally applied to any of the 3 formats to be
89 // converted to the canonical format, then later converted back as necessary.
90 //
91 // See 'Resource Format' section in `mesos.proto` for more details.
93 {
94  // "post-reservation-refinement" -> "pre-reservation-refinement"
95  //
96  // The `Resource` objects must be in the "post-reservation-refinement" format,
97  // and must not have refined reservations.
98  //
99  // All resources end up with the `Resource.role` and `Resource.reservation`
100  // fields set, and the `Resource.reservations` field unset.
101  //
102  // We convert the resources to the "pre-reservation-refinement" format to
103  // checkpoint resources for example. This enables downgrading to an agent
104  // without a RESERVATION_REFINEMENT caapbility, since the resources will
105  // be checkpointed in a format that the downgraded agent can recover from.
107 
108  // "pre-reservation-refinement" -> "post-reservation-refinement"
109  // "post-reservation-refinement" -> "post-reservation-refinement"
110  // "endpoint" -> "post-reservation-refinement"
111  //
112  // The `Resource` objects can be in any of the valid resource formats:
113  // "pre-reservation-refinement", "post-reservation-refinement", "endpoint".
114  //
115  // All resources end up with the `Resource.reservations` field set,
116  // and the `Resource.role` and `Resource.reservation` fields unset.
117  //
118  // If the `Resource` objects are already in the "post-reservation-refinement"
119  // format, this is a no-op.
120  //
121  // We convert the resources to the "post-reservation-refinement" format,
122  // for example, when a master receives a message from an agent without
123  // the RESERVATION_REFINEMENT capability. This allows a component
124  // (e.g. master) code to deal with a canonical format to simplify the code.
126 
127  // "post-reservation-refinement" -> "endpoint"
128  //
129  // This is a special case for endpoints, which injects
130  // the "pre-reservation-refinement" format.
131  //
132  // The `Resource` objects must be in the "post-reservation-refinement" format.
133  //
134  // All resources continue to have the `Resource.reservations` field set.
135  // The `Resource` objects without refined reservations end up with the
136  // `Resource.role` and `Resource.reservation` fields set, and the objects
137  // with refined reservations have them unset.
138  //
139  // We inject the resources with the "pre-reservation-refinement" format to
140  // enable backward compatibility with external tooling. If the master has been
141  // upgraded to a version that supports reservation refinement but no refined
142  // reservations have been made, the endpoints will return the data in both new
143  // and old formats to maximize backward compatibility. However, once
144  // a reservation refinement is made to a resource, that resource is only
145  // returned in the new format.
147 };
148 
149 
150 // Converts the given `Resource` to the specified `ResourceFormat`.
151 //
152 // See the "Resource Format" section in `mesos.proto` for more details.
153 // See the `ResourceFormat` enum above for more details.
154 void convertResourceFormat(Resource* resource, ResourceFormat format);
155 
156 
157 // Converts the given `Resource`s to the specified `ResourceFormat`.
159  google::protobuf::RepeatedPtrField<Resource>* resources,
161 
162 
163 // Converts the given `Resource`s to the specified `ResourceFormat`.
165  std::vector<Resource>* resources,
167 
168 
169 // Convert any resources contained in the given message(s)
170 // to the "post-reservation-refinement" format.
171 void upgradeResource(Resource* resource);
172 
173 
174 void upgradeResources(google::protobuf::RepeatedPtrField<Resource>* resources);
175 
176 
177 void upgradeResources(std::vector<Resource>* resources);
178 
179 
180 void upgradeResources(google::protobuf::Message* message);
181 
182 
183 // Convert the resources in the given `Operation` to the
184 // "post-reservation-refinement" format from any format
185 // ("pre-", "post-" or "endpoint") if all of the resources are valid.
186 // Returns an `Error` if there are any invalid resources present;
187 // in this case, all resources are left unchanged.
188 // NOTE: The validate and upgrade steps are bundled because currently
189 // it would be an error to validate but not upgrade or to upgrade
190 // without validating.
192 
193 
194 // Convert any resources contained in the given message(s)
195 // to the "pre-reservation-refinement" format, if possible.
196 //
197 // These functions do not provide "all-or-nothing" semantics.
198 // The resources are downgraded to "pre-" format until either
199 // (1) there are no more resources, or
200 // (2) a non-downgradable resource is encountered.
201 //
202 // For (1), `Nothing` is returned.
203 // For (2), `Error` is returned, and the rest of the resources are untouched.
204 //
205 // This implies that components that have refined resources created cannot
206 // be downgraded to a version that does not support reservation refinement.
207 Try<Nothing> downgradeResource(Resource* resource);
208 
209 
211  google::protobuf::RepeatedPtrField<Resource>* resources);
212 
213 
214 Try<Nothing> downgradeResources(std::vector<Resource>* resources);
215 
216 
217 Try<Nothing> downgradeResources(google::protobuf::Message* message);
218 
219 
220 // These two functions shrink resources down to the target scalar resource
221 // quantities or limits respectively. Target can only be specified for
222 // scalar resources. Otherwise a `CHECK` error will occur.
223 //
224 // The primary difference between these two shrink functions is about resources
225 // that have no quantity/limit specified in the `target`:
226 //
227 // - If the target is `ResourceQuantities`, due to its absent-means-zero
228 // semantic, such resources will the dropped i.e. shrink down to zero.
229 //
230 // - If the target is `ResourceLimits`, due to its absent-means-infinite
231 // semantic, such resources will be kept as-is in the result since it
232 // is already below the (infinite) limit.
233 //
234 // Note that some resources are indivisible (e.g. MOUNT volume) and
235 // may be excluded in entirety in order to achieve the target size
236 // (this may lead to the result size being smaller than the target size).
237 //
238 // Note also that there may be more than one result that satisfies
239 // the target sizes (e.g. need to exclude 1 of 2 disks); this function
240 // will make a random choice in these cases.
242  const Resources& resources, ResourceQuantities target);
244  const Resources& resources, ResourceLimits target);
245 
246 
247 } // namespace mesos {
248 
249 #endif // __RESOURCES_UTILS_HPP__
Try< Nothing > downgradeResources(google::protobuf::RepeatedPtrField< Resource > *resources)
Definition: resources_utils.hpp:125
Definition: resource_quantities.hpp:192
Definition: check.hpp:33
Definition: resource_quantities.hpp:63
Definition: resources.hpp:83
void convertResourceFormat(Resource *resource, ResourceFormat format)
Resources shrinkResources(const Resources &resources, ResourceQuantities target)
Operation
Definition: cgroups.hpp:444
Definition: check.hpp:30
void upgradeResources(google::protobuf::RepeatedPtrField< Resource > *resources)
Definition: resources_utils.hpp:106
Definition: agent.hpp:25
Definition: resources_utils.hpp:146
Result< ResourceProviderID > getResourceProviderId(const Offer::Operation &operation)
bool needCheckpointing(const Resource &resource)
Try< Resources > applyCheckpointedResources(const Resources &resources, const Resources &checkpointedResources)
void upgradeResource(Resource *resource)
Try< std::string > format(const std::string &fmt, va_list args)
Definition: format.hpp:68
Try< std::vector< ResourceConversion > > getResourceConversions(const Offer::Operation &operation)
Option< Error > validateAndUpgradeResources(Offer::Operation *operation)
Try< Nothing > downgradeResource(Resource *resource)
ResourceFormat
Definition: resources_utils.hpp:92