Apache Mesos
hierarchical.hpp
Go to the documentation of this file.
1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 
17 #ifndef __MASTER_ALLOCATOR_MESOS_HIERARCHICAL_HPP__
18 #define __MASTER_ALLOCATOR_MESOS_HIERARCHICAL_HPP__
19 
20 #include <memory>
21 #include <set>
22 #include <string>
23 
24 #include <mesos/mesos.hpp>
25 
26 #include <process/future.hpp>
27 #include <process/id.hpp>
28 #include <process/owned.hpp>
29 
30 #include <stout/boundedhashmap.hpp>
31 #include <stout/duration.hpp>
32 #include <stout/hashmap.hpp>
33 #include <stout/hashset.hpp>
34 #include <stout/lambda.hpp>
35 #include <stout/option.hpp>
36 
38 
41 
44 
45 #include "master/constants.hpp"
46 
47 namespace mesos {
48 namespace internal {
49 namespace master {
50 namespace allocator {
51 
52 // We forward declare the hierarchical allocator process so that we
53 // can typedef an instantiation of it with DRF sorters.
54 template <
55  typename RoleSorter,
56  typename FrameworkSorter>
58 
61 
64 
67 
70 
71 
72 namespace internal {
73 
74 // Forward declarations.
75 class OfferFilter;
76 class InverseOfferFilter;
77 class RoleTree;
78 
79 
80 struct Framework
81 {
82  Framework(
83  const FrameworkInfo& frameworkInfo,
84  const std::set<std::string>& suppressedRoles,
85  bool active,
87 
88  const FrameworkID frameworkId;
89 
90  std::set<std::string> roles;
91 
92  std::set<std::string> suppressedRoles;
93 
95 
96  // Offer filters are tied to the role the filtered
97  // resources were offered to.
100 
103 
104  bool active;
105 
107 
109 
110  // TODO(bbannier): Consider documenting examples on how to use this setting.
112 };
113 
114 
115 class Role
116 {
117 public:
118  Role(const std::string& name, Role* parent);
119 
121  {
122  return reservationScalarQuantities_;
123  }
124 
126  {
127  return offeredOrAllocatedScalars_;
128  }
129 
130  const hashset<FrameworkID>& frameworks() const { return frameworks_; }
131 
132  const Quota& quota() const { return quota_; }
133 
135  {
137  allocatedScalars_.unreserved().nonRevocable()) +
138  reservationScalarQuantities_;
139  }
140 
141  double weight() const { return weight_; }
142 
143  bool isEmpty() const
144  {
145  return children_.empty() &&
146  frameworks_.empty() &&
147  reservationScalarQuantities_.empty() &&
148  quota_ == DEFAULT_QUOTA &&
149  weight_ == DEFAULT_WEIGHT;
150  }
151 
152  std::vector<Role*> children() const { return children_.values(); }
153 
154  const std::string role; // E.g. "a/b/c"
155  const std::string basename; // E.g. "c"
156 
157 private:
158  // We keep fields that are related to the tree structure as private
159  // and only allow mutations through the RoleTree structure.
160  friend class RoleTree;
161 
162  // Add a child to the role, the child must not already exist.
163  void addChild(Role* child);
164 
165  // Remove a child from the role, the child must be present.
166  void removeChild(Role* child);
167 
168  Role* parent;
169 
170  // Configured guaranteed resource quantities and resource limits for
171  // this role. By default, a role has no guarantee and no limit.
172  Quota quota_;
173 
174  // Configured weight for the role. This affects sorting precedence.
175  // By default, weights == DEFAULT_WEIGHT == 1.0.
176  double weight_;
177 
178  // IDs of the frameworks tracked under the role, if any.
179  // A framework is tracked under the role if the framework:
180  //
181  // (1) is subscribed to the role;
182  // *OR*
183  // (2) has resources allocated under the role.
184  //
185  // NOTE: (2) could be true without (1). This is because the allocator
186  // interface allows for a framework role to be removed without recovering
187  // resources offered or allocated to this role.
188  hashset<FrameworkID> frameworks_;
189 
190  // Total allocated or offered scalar resources to this role, including
191  // meta data. This field dose not affect role's lifecycle. However, since
192  // any offered or allocated resources should be tied to a framework,
193  // an empty role (that has no registered framework) must have
194  // empty offeredOrAllocated resources.
195  Resources offeredOrAllocatedScalars_;
196 
197  // Aggregated reserved scalar resource quantities on all agents tied to this
198  // role, if any. This includes both its own reservations as well as
199  // reservations of any of its subroles (i.e. it is hierarchical aware).
200  // Note that non-scalar resources, such as ports, are excluded.
201  ResourceQuantities reservationScalarQuantities_;
202 
203  // Scalar resources actually allocated (i.e. used for launching tasks) to this
204  // role and any of its subroles, both reserved and unreserved, on all agents.
205  Resources allocatedScalars_;
206 
207  hashmap<std::string, Role*> children_;
208 };
209 
210 
211 // A tree abstraction for organizing `class Role` hierarchically.
212 //
213 // We track a role when it has:
214 //
215 // * a non-default weight, or
216 // * a non-default quota, or
217 // * frameworks subscribed to it, or
218 // * reservations, or
219 // * descendent roles meeting any of the above conditions.
220 //
221 // Any roles that do not meet these conditions are not tracked in the role tree.
222 class RoleTree
223 {
224 public:
225  RoleTree(); // Only used in tests.
226 
228 
229  ~RoleTree();
230 
231  Option<const Role*> get(const std::string& role) const;
232 
233  // Return a hashmap of all known roles. Root is not included.
234  const hashmap<std::string, Role>& roles() const { return roles_; }
235 
236  const Role* root() const { return root_; }
237 
238  // We keep track of reservations to enforce role quota limit
239  // in the presence of unallocated reservations. See MESOS-4527.
240  void trackReservations(const Resources& resources);
241  void untrackReservations(const Resources& resources);
242 
243  // We keep track of allocated resources which are actially used by frameworks.
244  void trackAllocated(const Resources& resources);
245  void untrackAllocated(const Resources& resources);
246 
247  void trackFramework(
248  const FrameworkID& frameworkId, const std::string& role);
249  void untrackFramework(
250  const FrameworkID& frameworkId, const std::string& role);
251 
252  void updateQuota(const std::string& role, const Quota& quota);
253 
254  void updateWeight(const std::string& role, double weight);
255 
256  void trackOfferedOrAllocated(const Resources& resources);
257  void untrackOfferedOrAllocated(const Resources& resources);
258 
259  // Dump the role tree state in JSON format for debugging.
260  std::string toJSON() const;
261 
262 private:
263  // Private helper to get non-const pointers.
264  Option<Role*> get_(const std::string& role);
265 
266  // Lookup or add the role struct associated with the role. Ancestor roles
267  // along the tree path will be created if necessary.
268  Role& operator[](const std::string& role);
269 
270  // Helper for modifying a role and all its ancestors.
271  template<class UnaryFunction>
272  static void applyToRoleAndAncestors(Role* role, UnaryFunction f) {
273  for (; role != nullptr; role = role->parent) {
274  f(role);
275  }
276  }
277 
278  // Try to remove the role associated with the given role.
279  // The role must exist. The role and its ancestors will be removed
280  // if they become "empty". See "Role:isEmpty()".
281  // Return true if the role instance associated with the role is removed.
282  // This should be called whenever a role's state (that defines its emptiness)
283  // gets updated, such as quota, weight, reservation and tracked frameworks.
284  // Otherwise the "tracking only non-empty" tree invariant may break.
285  bool tryRemove(const std::string& role);
286 
287  void updateQuotaConsumedMetric(const Role* role);
288 
289  // Root node of the tree, its `basename` == `role` == "".
290  Role* root_;
291 
292  // Allocator's metrics handle for publishing role related metrics.
294 
295  // A map of role and `Role` pairs for quick lookup.
297 };
298 
299 
300 class Slave
301 {
302 public:
304  const SlaveInfo& _info,
305  const protobuf::slave::Capabilities& _capabilities,
306  bool _activated,
307  const Resources& _total,
308  const hashmap<FrameworkID, Resources>& _allocated)
309  : id(_info.id()),
310  info(_info),
311  capabilities(_capabilities),
312  activated(_activated),
313  totalAllocated(Resources::sum(_allocated)),
314  total(_total),
315  offeredOrAllocated(_allocated),
316  totalOfferedOrAllocated(Resources::sum(_allocated)),
317  shared(_total.shared()),
318  hasGpu_(_total.gpus().getOrElse(0) > 0)
319  {
320  CHECK(_info.has_id());
321  updateAvailable();
322  }
323 
324  const Resources& getTotal() const { return total; }
325 
327  {
328  return offeredOrAllocated;
329  }
330 
332  {
333  return totalOfferedOrAllocated;
334  }
335 
336  const Resources& getAvailable() const { return available; }
337 
338  bool hasGpu() const { return hasGpu_; }
339 
340  void updateTotal(const Resources& newTotal) {
341  total = newTotal;
342  shared = total.shared();
343  hasGpu_ = total.gpus().getOrElse(0) > 0;
344 
345  updateAvailable();
346  }
347 
349  const FrameworkID& frameworkId, const Resources& offeredOrAllocated_)
350  {
351  // Increasing available is to subtract offered or allocated.
352  if (offeredOrAllocated_.empty()) {
353  return;
354  }
355 
356  // It is possible that the reference of `offeredOrAllocated_`
357  // points to the same object as `resources` below. We must
358  // do subtraction here before any mutation on the object.
359  totalOfferedOrAllocated -= offeredOrAllocated_;
360 
361  Resources& resources = offeredOrAllocated.at(frameworkId);
362  CHECK_CONTAINS(resources, offeredOrAllocated_);
363  resources -= offeredOrAllocated_;
364  if (resources.empty()) {
365  offeredOrAllocated.erase(frameworkId);
366  }
367 
368  updateAvailable();
369  }
370 
372  const FrameworkID& frameworkId, const Resources& offeredOrAllocated_)
373  {
374  if (offeredOrAllocated_.empty()) {
375  return;
376  }
377 
378  // Decreasing available is to add offered or allocated.
379 
380  offeredOrAllocated[frameworkId] += offeredOrAllocated_;
381 
382  totalOfferedOrAllocated += offeredOrAllocated_;
383 
384  updateAvailable();
385  }
386 
387  const SlaveID id;
388 
389  // The `SlaveInfo` that was passed to the allocator when the slave was added
390  // or updated. Currently only two fields are used: `hostname` for host
391  // whitelisting and in log messages, and `domain` for region-aware
392  // scheduling.
393  SlaveInfo info;
394 
396 
397  bool activated; // Whether to offer resources.
398 
399  // Represents a scheduled unavailability due to maintenance for a specific
400  // slave, and the responses from frameworks as to whether they will be able
401  // to gracefully handle this unavailability.
402  //
403  // NOTE: We currently implement maintenance in the allocator to be able to
404  // leverage state and features such as the FrameworkSorter and OfferFilter.
405  struct Maintenance
406  {
407  Maintenance(const Unavailability& _unavailability)
408  : unavailability(_unavailability) {}
409 
410  // The start time and optional duration of the event.
411  Unavailability unavailability;
412 
413  // A mapping of frameworks to the inverse offer status associated with
414  // this unavailability.
415  //
416  // NOTE: We currently lose this information during a master fail over
417  // since it is not persisted or replicated. This is ok as the new master's
418  // allocator will send out new inverse offers and re-collect the
419  // information. This is similar to all the outstanding offers from an old
420  // master being invalidated, and new offers being sent out.
422 
423  // Represents the "unit of accounting" for maintenance. When a
424  // `FrameworkID` is present in the hashset it means an inverse offer has
425  // been sent out. When it is not present it means no offer is currently
426  // outstanding.
428  };
429 
430  // When the `maintenance` is set the slave is scheduled to be unavailable at
431  // a given point in time, for an optional duration. This information is used
432  // to send out `InverseOffers`.
434 
435  // Sum of all allocated (i.e. occupied by running tasks) resources on the
436  // agent. This information is needed to untrack allocated resources when the
437  // agent is removed, because the master is not obligated to separately inform
438  // allocator that resources of the removed agent are not offered/allocated
439  // anymore.
441 
442 private:
443  void updateAvailable()
444  {
445  // In order to subtract from the total,
446  // we strip the allocation information.
447  Resources totalOfferedOrAllocated_ = totalOfferedOrAllocated;
448  totalOfferedOrAllocated_.unallocate();
449 
450  // This is hot path. We avoid the unnecessary resource traversals
451  // in the common case where there are no shared resources.
452  if (shared.empty()) {
453  available = total - totalOfferedOrAllocated_;
454  } else {
455  // Since shared resources are offerable even when they are in use, we
456  // always include them as part of available resources.
457  available =
458  (total.nonShared() - totalOfferedOrAllocated_.nonShared()) + shared;
459  }
460  }
461 
462  // Total amount of regular *and* oversubscribed resources.
463  Resources total;
464 
465  // NOTE: We keep track of the slave's allocated resources despite
466  // having that information in sorters. This is because the
467  // information in sorters is not accurate if some framework
468  // hasn't reregistered. See MESOS-2919 for details.
469  //
470  // This includes both regular *and* oversubscribed resources.
471  //
472  // An entry is erased if a framework no longer has any
473  // offered or allocated on the agent.
474  hashmap<FrameworkID, Resources> offeredOrAllocated;
475 
476  // Sum of all offered or allocated resources on the agent. This should equal
477  // to sum of `offeredOrAllocated` (including all the meta-data).
478  Resources totalOfferedOrAllocated;
479 
480  // We track the total and allocated resources on the slave to
481  // avoid calculating it in place every time.
482  //
483  // Note that `available` always contains all the shared resources on the
484  // agent regardless whether they have ever been allocated or not.
485  // NOTE, however, we currently only offer a shared resource only if it has
486  // not been offered in an allocation cycle to a framework. We do this mainly
487  // to preserve the normal offer behavior. This may change in the future
488  // depending on use cases.
489  //
490  // Note that it's possible for the slave to be over-allocated!
491  // In this case, allocated > total.
492  Resources available;
493 
494  // We keep a copy of the shared resources to avoid unnecessary copying.
495  Resources shared;
496 
497  // We cache whether the agent has gpus as an optimization.
498  bool hasGpu_;
499 };
500 
501 
502 // Implements the basic allocator algorithm - first pick a role by
503 // some criteria, then pick one of their frameworks to allocate to.
505 {
506 public:
508  const std::function<Sorter*()>& roleSorterFactory,
509  const std::function<Sorter*()>& _frameworkSorterFactory)
510  : initialized(false),
511  paused(true),
512  metrics(*this),
513  completedFrameworkMetrics(0),
514  roleTree(&metrics),
515  roleSorter(roleSorterFactory()),
516  frameworkSorterFactory(_frameworkSorterFactory) {}
517 
519 
521  {
522  return process::PID<Self>(this);
523  }
524 
525  void initialize(
526  const mesos::allocator::Options& options,
527  const lambda::function<
528  void(const FrameworkID&,
529  const hashmap<std::string, hashmap<SlaveID, Resources>>&)>&
530  offerCallback,
531  const lambda::function<
532  void(const FrameworkID&,
534  inverseOfferCallback) override;
535 
536  void recover(
537  const int _expectedAgentCount,
538  const hashmap<std::string, Quota>& quotas) override;
539 
540  void addFramework(
541  const FrameworkID& frameworkId,
542  const FrameworkInfo& frameworkInfo,
544  bool active,
545  const std::set<std::string>& suppressedRoles) override;
546 
547  void removeFramework(
548  const FrameworkID& frameworkId) override;
549 
550  void activateFramework(
551  const FrameworkID& frameworkId) override;
552 
553  void deactivateFramework(
554  const FrameworkID& frameworkId) override;
555 
556  void updateFramework(
557  const FrameworkID& frameworkId,
558  const FrameworkInfo& frameworkInfo,
559  const std::set<std::string>& suppressedRoles) override;
560 
561  void addSlave(
562  const SlaveID& slaveId,
563  const SlaveInfo& slaveInfo,
564  const std::vector<SlaveInfo::Capability>& capabilities,
566  const Resources& total,
567  const hashmap<FrameworkID, Resources>& used) override;
568 
569  void removeSlave(
570  const SlaveID& slaveId) override;
571 
572  void updateSlave(
573  const SlaveID& slave,
574  const SlaveInfo& slaveInfo,
575  const Option<Resources>& total = None(),
576  const Option<std::vector<SlaveInfo::Capability>>& capabilities = None())
577  override;
578 
579  void addResourceProvider(
580  const SlaveID& slave,
581  const Resources& total,
582  const hashmap<FrameworkID, Resources>& used) override;
583 
584  void deactivateSlave(
585  const SlaveID& slaveId) override;
586 
587  void activateSlave(
588  const SlaveID& slaveId) override;
589 
590  void updateWhitelist(
591  const Option<hashset<std::string>>& whitelist) override;
592 
593  void requestResources(
594  const FrameworkID& frameworkId,
595  const std::vector<Request>& requests) override;
596 
597  void updateAllocation(
598  const FrameworkID& frameworkId,
599  const SlaveID& slaveId,
600  const Resources& offeredResources,
601  const std::vector<ResourceConversion>& conversions) override;
602 
603  process::Future<Nothing> updateAvailable(
604  const SlaveID& slaveId,
605  const std::vector<Offer::Operation>& operations) override;
606 
607  void updateUnavailability(
608  const SlaveID& slaveId,
609  const Option<Unavailability>& unavailability) override;
610 
611  void updateInverseOffer(
612  const SlaveID& slaveId,
613  const FrameworkID& frameworkId,
614  const Option<UnavailableResources>& unavailableResources,
616  const Option<Filters>& filters) override;
617 
619  hashmap<SlaveID,
621  getInverseOfferStatuses() override;
622 
623  void transitionOfferedToAllocated(
624  const SlaveID& slaveId, const Resources& resources) override;
625 
626  void recoverResources(
627  const FrameworkID& frameworkId,
628  const SlaveID& slaveId,
629  const Resources& resources,
630  const Option<Filters>& filters,
631  bool isAllocated) override;
632 
633  void suppressOffers(
634  const FrameworkID& frameworkId,
635  const std::set<std::string>& roles) override;
636 
637  void reviveOffers(
638  const FrameworkID& frameworkId,
639  const std::set<std::string>& roles) override;
640 
641  void updateQuota(
642  const std::string& role,
643  const Quota& quota) override;
644 
645  void updateWeights(
646  const std::vector<WeightInfo>& weightInfos) override;
647 
648  void pause() override;
649 
650  void resume() override;
651 
652 protected:
653  // Useful typedefs for dispatch/delay/defer to self()/this.
656 
657  // Generate offers from all known agents.
658  process::Future<Nothing> generateOffers();
659 
660  // Generate offers from the specified agent.
661  process::Future<Nothing> generateOffers(const SlaveID& slaveId);
662 
663  // Generate offers from the specified agents. The offer generation is
664  // deferred and batched with other offer generation requests.
665  process::Future<Nothing> generateOffers(const hashset<SlaveID>& slaveIds);
666 
667  Nothing _generateOffers();
668 
669  void __generateOffers();
670 
671  void generateInverseOffers();
672 
673  // Remove an offer filter for the specified role of the framework.
674  void expire(
675  const FrameworkID& frameworkId,
676  const std::string& role,
677  const SlaveID& slaveId,
678  const std::weak_ptr<OfferFilter>& offerFilter);
679 
680  void _expire(
681  const FrameworkID& frameworkId,
682  const std::string& role,
683  const SlaveID& slaveId,
684  const std::weak_ptr<OfferFilter>& offerFilter);
685 
686  // Remove an inverse offer filter for the specified framework.
687  void expire(
688  const FrameworkID& frameworkId,
689  const SlaveID& slaveId,
690  const std::weak_ptr<InverseOfferFilter>& inverseOfferFilter);
691 
692  // Checks whether the slave is whitelisted.
693  bool isWhitelisted(const SlaveID& slaveId) const;
694 
695  // Returns true if there is a resource offer filter for the
696  // specified role of this framework on this slave.
697  bool isFiltered(
698  const Framework& framework,
699  const std::string& role,
700  const Slave& slave,
701  const Resources& resources) const;
702 
703  // Returns true if there is an inverse offer filter for this framework
704  // on this slave.
705  bool isFiltered(
706  const Framework& framework,
707  const Slave& slave) const;
708 
709  bool allocatable(
710  const Resources& resources,
711  const std::string& role,
712  const Framework& framework) const;
713 
715  bool paused;
716 
718 
719  // Recovery data.
721 
722  lambda::function<
723  void(const FrameworkID&,
724  const hashmap<std::string, hashmap<SlaveID, Resources>>&)>
726 
727  lambda::function<
728  void(const FrameworkID&,
731 
732  friend Metrics;
734 
736  {
737  return static_cast<double>(eventCount<process::DispatchEvent>());
738  }
739 
740  double _resources_total(
741  const std::string& resource);
742 
743  double _resources_offered_or_allocated(
744  const std::string& resource);
745 
746  double _quota_offered_or_allocated(
747  const std::string& role,
748  const std::string& resource);
749 
750  double _offer_filters_active(
751  const std::string& role);
752 
754 
757 
759 
760  // Total scalar resource quantities on all agents.
762 
764 
765  // A set of agents that are kept as allocation candidates. Events
766  // may add or remove candidates to the set. When an offer generation is
767  // processed, the set of candidates is cleared.
769 
770  // Future for the dispatched offer generation that becomes
771  // ready after the offer generation run is complete.
773 
774  // Slaves to send offers for.
776 
777  // There are two stages of offer generation:
778  //
779  // Stage 1: Generate offers to satisfy quota guarantees.
780  //
781  // Stage 2: Generate offers above quota guarantees up to quota limits.
782  // Note that we need to hold back enough "headroom"
783  // to ensure that any unsatisfied quota can be
784  // satisfied later.
785  //
786  // Each stage comprises two levels of sorting, hence "hierarchical".
787  // Level 1 sorts across roles:
788  // Currently, only the offered or allocated portion of the reserved
789  // resources are accounted for fairness calculation.
790  //
791  // TODO(mpark): Reserved resources should be accounted for fairness
792  // calculation whether they are offered/allocated or not, since they model
793  // a long or forever running task. That is, the effect of reserving resources
794  // is equivalent to launching a task in that the resources that make up the
795  // reservation are not available to other roles as non-revocable.
796  //
797  // Level 2 sorts across frameworks within a particular role:
798  // Reserved resources at this level are, and should be accounted for
799  // fairness calculation only if they are allocated. This is because
800  // reserved resources are fairly shared across the frameworks in the role.
801  //
802  // The allocator relies on `Sorter`s to employ a particular sorting
803  // algorithm. Each level has its own sorter and hence may have different
804  // fairness calculations.
805  //
806  // NOTE: The hierarchical allocator considers revocable resources as
807  // regular resources when doing fairness calculations.
808  //
809  // TODO(vinod): Consider using a different fairness algorithm for
810  // revocable resources.
811 
812  // A sorter for active roles. This sorter determines the order in which
813  // roles are offered resources during Level 1 of the second stage.
814  // The total cluster resources are used as the resource pool.
816 
817  // A collection of sorters, one per active role. Each sorter determines
818  // the order in which frameworks that belong to the same role are offered
819  // resources inside the role's share. These sorters are used during Level 2
820  // for both the first and the second stages. Since frameworks are sharing
821  // resources of a role, resources offered or allocated to the role are used as
822  // the resource pool for each role specific framework sorter.
824 
825  // Factory function for framework sorters.
826  const std::function<Sorter*()> frameworkSorterFactory;
827 
828 private:
829  bool isFrameworkTrackedUnderRole(
830  const FrameworkID& frameworkId,
831  const std::string& role) const;
832 
833  Option<Slave*> getSlave(const SlaveID& slaveId) const;
834  Option<Framework*> getFramework(const FrameworkID& frameworkId) const;
835 
836  Option<Sorter*> getFrameworkSorter(const std::string& role) const;
837 
838  const Quota& getQuota(const std::string& role) const;
839 
840  // Helpers to track and untrack a framework under a role.
841  // Frameworks should be tracked under a role either if it subscribes to the
842  // role *OR* it has resources allocated/offered to that role. when neither
843  // conditions are met, it should be untracked.
844  //
845  // `tryUntrackFrameworkUnderRole` returns true if the framework is untracked
846  // under the role.
847  void trackFrameworkUnderRole(
848  const Framework& framework, const std::string& role);
849  bool tryUntrackFrameworkUnderRole(
850  const Framework& framework, const std::string& role);
851 
852  void suppressRoles(Framework& framework, const std::set<std::string>& roles);
853  void reviveRoles(Framework& framework, const std::set<std::string>& roles);
854 
855  // Helper to update the agent's total resources maintained in the allocator
856  // and the role and quota sorters (whose total resources match the agent's
857  // total resources). Returns true iff the stored agent total was changed.
858  bool updateSlaveTotal(const SlaveID& slaveId, const Resources& total);
859 
860  // Helper that returns true if the given agent is located in a
861  // different region than the master. This can only be the case if
862  // the agent and the master are both configured with a fault domain.
863  bool isRemoteSlave(const Slave& slave) const;
864 
865  // Helper function that checks if a framework is capable of
866  // receiving resources on the agent based on the framework capability.
867  //
868  // TODO(mzhu): Make this a `Framework` member function once we pull
869  // `struct Framework` out from being nested.
870  bool isCapableOfReceivingAgent(
871  const protobuf::framework::Capabilities& frameworkCapabilities,
872  const Slave& slave) const;
873 
874  // Helper function that removes any resources that the framework is not
875  // capable of receiving based on the given framework capability.
876  //
877  // TODO(mzhu): Make this a `Framework` member function once we pull
878  // `struct Framework` out from being nested.
879  Resources stripIncapableResources(
880  const Resources& resources,
881  const protobuf::framework::Capabilities& frameworkCapabilities) const;
882 
883  // Helper to track offered or allocated resources on an agent.
884  //
885  // TODO(asekretenko): rename `(un)trackAllocatedResources()` to reflect the
886  // fact that these methods do not distinguish between offered and allocated.
887  //
888  // TODO(mzhu): replace this with `RoleTree::trackOfferedOrAllocated`.
889  void trackAllocatedResources(
890  const SlaveID& slaveId,
891  const FrameworkID& frameworkId,
892  const Resources& offeredOrAllocated);
893 
894  // Helper to untrack resources that are no longer offered or allocated
895  // on an agent.
896  //
897  // TODO(mzhu): replace this with `RoleTree::untrackOfferedOrAllocated`.
898  void untrackAllocatedResources(
899  const SlaveID& slaveId,
900  const FrameworkID& frameworkId,
901  const Resources& offeredOrallocated);
902 
903  // Helper that removes all existing offer filters for the given slave
904  // id.
905  void removeFilters(const SlaveID& slaveId);
906 };
907 
908 
909 } // namespace internal {
910 
911 
912 // We map the templatized version of the `HierarchicalAllocatorProcess` to one
913 // that relies on sorter factories in the internal namespace. This allows us
914 // to keep the implementation of the allocator in the implementation file.
915 template <
916  typename RoleSorter,
917  typename FrameworkSorter>
920 {
921 public:
923  : ProcessBase(process::ID::generate("hierarchical-allocator")),
925  [this]() -> Sorter* {
926  return new RoleSorter(this->self(), "allocator/mesos/roles/");
927  },
928  []() -> Sorter* { return new FrameworkSorter(); }) {}
929 };
930 
931 } // namespace allocator {
932 } // namespace master {
933 } // namespace internal {
934 } // namespace mesos {
935 
936 #endif // __MASTER_ALLOCATOR_MESOS_HIERARCHICAL_HPP__
Protocol< RecoverRequest, RecoverResponse > recover
std::string generate(const std::string &prefix="")
Returns &#39;prefix(N)&#39; where N represents the number of instances where the same prefix (wrt...
std::set< std::string > roles
Definition: hierarchical.hpp:90
#define CHECK_CONTAINS(container, key)
Definition: check.hpp:298
HierarchicalAllocatorProcess(const std::function< Sorter *()> &roleSorterFactory, const std::function< Sorter *()> &_frameworkSorterFactory)
Definition: hierarchical.hpp:507
mesos::allocator::Options options
Definition: hierarchical.hpp:717
Definition: nothing.hpp:16
Definition: option.hpp:29
T getOrElse(U &&u) const &
Definition: option.hpp:133
const hashmap< FrameworkID, Resources > & getOfferedOrAllocated() const
Definition: hierarchical.hpp:326
const Resources & getTotal() const
Definition: hierarchical.hpp:324
Maintenance(const Unavailability &_unavailability)
Definition: hierarchical.hpp:407
F && f
Definition: defer.hpp:270
protobuf::framework::Capabilities capabilities
Definition: hierarchical.hpp:94
hashmap< std::string, hashmap< SlaveID, hashset< std::shared_ptr< OfferFilter > > > > offerFilters
Definition: hierarchical.hpp:99
Definition: master.hpp:27
HierarchicalAllocatorProcess< RandomSorter, RandomSorter > HierarchicalRandomAllocatorProcess
Definition: hierarchical.hpp:66
hashmap< std::string, process::Owned< Sorter > > frameworkSorters
Definition: hierarchical.hpp:823
Definition: resource_quantities.hpp:63
BoundedHashMap< FrameworkID, process::Owned< FrameworkMetrics > > completedFrameworkMetrics
Definition: hierarchical.hpp:756
Definition: protobuf_utils.hpp:332
Framework(const FrameworkInfo &frameworkInfo, const std::set< std::string > &suppressedRoles, bool active, bool publishPerFrameworkMetrics)
Result< ProcessStatus > status(pid_t pid)
Definition: proc.hpp:166
Definition: uuid.hpp:33
Option< int > expectedAgentCount
Definition: hierarchical.hpp:720
Definition: resources.hpp:83
Slave * getSlave(Master *master, const SlaveID &slaveId)
Slave(const SlaveInfo &_info, const protobuf::slave::Capabilities &_capabilities, bool _activated, const Resources &_total, const hashmap< FrameworkID, Resources > &_allocated)
Definition: hierarchical.hpp:303
Option< double > gpus() const
~HierarchicalAllocatorProcess() override
Definition: hierarchical.hpp:518
Pass in configuration to the allocator.
Definition: allocator.hpp:49
const hashmap< std::string, Role > & roles() const
Definition: hierarchical.hpp:234
hashmap< SlaveID, Slave > slaves
Definition: hierarchical.hpp:758
constexpr double DEFAULT_WEIGHT
Definition: constants.hpp:180
MesosAllocator< HierarchicalRandomAllocatorProcess > HierarchicalRandomAllocator
Definition: hierarchical.hpp:69
const Resources & offeredOrAllocatedScalars() const
Definition: hierarchical.hpp:125
const std::string basename
Definition: hierarchical.hpp:155
SlaveInfo info
Definition: hierarchical.hpp:393
bool hasGpu() const
Definition: hierarchical.hpp:338
MesosAllocator< HierarchicalDRFAllocatorProcess > HierarchicalDRFAllocator
Definition: hierarchical.hpp:63
std::vector< Role * > children() const
Definition: hierarchical.hpp:152
const std::function< Sorter *()> frameworkSorterFactory
Definition: hierarchical.hpp:826
bool activated
Definition: hierarchical.hpp:397
Definition: hashmap.hpp:38
const SlaveID id
Definition: hierarchical.hpp:387
lambda::function< void(const FrameworkID &, const hashmap< std::string, hashmap< SlaveID, Resources >> &)> offerCallback
Definition: hierarchical.hpp:725
bool active
Definition: hierarchical.hpp:104
Try< Nothing > unavailability(const Unavailability &unavailability)
static ResourceQuantities fromScalarResources(const Resources &resources)
process::Owned< Sorter > roleSorter
Definition: hierarchical.hpp:815
std::set< std::string > suppressedRoles
Definition: hierarchical.hpp:92
Try< Bytes > used(const std::string &path="/")
Definition: fs.hpp:43
Try< Nothing > initialize(const Flags &flags)
Initialized state for support of systemd functions in this file.
hashmap< FrameworkID, mesos::allocator::InverseOfferStatus > statuses
Definition: hierarchical.hpp:421
Result< std::vector< Filter< Classifier > > > filters(const std::string &_link, const Handle &parent)
Definition: internal.hpp:769
ResourceQuantities totalScalarQuantities
Definition: hierarchical.hpp:761
const FrameworkID frameworkId
Definition: hierarchical.hpp:88
double weight() const
Definition: hierarchical.hpp:141
Definition: protobuf_utils.hpp:631
Definition: agent.hpp:25
hashmap< FrameworkID, Framework > frameworks
Definition: hierarchical.hpp:753
HierarchicalAllocatorProcess< DRFSorter, DRFSorter > HierarchicalDRFAllocatorProcess
Definition: hierarchical.hpp:57
bool publishPerFrameworkMetrics
Definition: hierarchical.hpp:106
const Quota & quota() const
Definition: hierarchical.hpp:132
const ResourceQuantities & reservationScalarQuantities() const
Definition: hierarchical.hpp:120
hashmap< SlaveID, hashset< std::shared_ptr< InverseOfferFilter > > > inverseOfferFilters
Definition: hierarchical.hpp:102
const Role * root() const
Definition: hierarchical.hpp:236
process::Owned< FrameworkMetrics > metrics
Definition: hierarchical.hpp:108
Option< hashset< std::string > > whitelist
Definition: hierarchical.hpp:775
hashset< SlaveID > allocationCandidates
Definition: hierarchical.hpp:768
Definition: boundedhashmap.hpp:27
const Resources & getAvailable() const
Definition: hierarchical.hpp:336
A "process identifier" used to uniquely identify a process when dispatching messages.
Definition: pid.hpp:289
Resources shared() const
bool empty() const
Definition: resources.hpp:451
Definition: none.hpp:27
Definition: attributes.hpp:24
bool isEmpty() const
Definition: hierarchical.hpp:143
hashset< FrameworkID > offersOutstanding
Definition: hierarchical.hpp:427
Resources nonShared() const
Option< process::Future< Nothing > > offerGeneration
Definition: hierarchical.hpp:772
Definition: executor.hpp:48
Unavailability unavailability
Definition: hierarchical.hpp:411
HierarchicalAllocatorProcess This
Definition: hierarchical.hpp:655
hashmap< std::string, std::vector< ResourceQuantities > > minAllocatableResources
Definition: hierarchical.hpp:111
ResourceQuantities quotaConsumed() const
Definition: hierarchical.hpp:134
const std::string role
Definition: hierarchical.hpp:154
void increaseAvailable(const FrameworkID &frameworkId, const Resources &offeredOrAllocated_)
Definition: hierarchical.hpp:348
HierarchicalAllocatorProcess Self
Definition: hierarchical.hpp:654
Option< Maintenance > maintenance
Definition: hierarchical.hpp:433
const Resources & getTotalOfferedOrAllocated() const
Definition: hierarchical.hpp:331
void decreaseAvailable(const FrameworkID &frameworkId, const Resources &offeredOrAllocated_)
Definition: hierarchical.hpp:371
const Quota DEFAULT_QUOTA
Definition: constants.hpp:177
Definition: owned.hpp:36
Resources totalAllocated
Definition: hierarchical.hpp:440
const hashset< FrameworkID > & frameworks() const
Definition: hierarchical.hpp:130
Definition: quota.hpp:27
constexpr const char * name
Definition: shell.hpp:43
protobuf::slave::Capabilities capabilities
Definition: hierarchical.hpp:395
void updateTotal(const Resources &newTotal)
Definition: hierarchical.hpp:340
lambda::function< void(const FrameworkID &, const hashmap< SlaveID, UnavailableResources > &)> inverseOfferCallback
Definition: hierarchical.hpp:730