Apache Mesos
hierarchical.hpp
Go to the documentation of this file.
1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 
17 #ifndef __MASTER_ALLOCATOR_MESOS_HIERARCHICAL_HPP__
18 #define __MASTER_ALLOCATOR_MESOS_HIERARCHICAL_HPP__
19 
20 #include <memory>
21 #include <set>
22 #include <string>
23 
25 #include <mesos/mesos.hpp>
26 
27 #include <process/future.hpp>
28 #include <process/id.hpp>
29 #include <process/http.hpp>
30 #include <process/owned.hpp>
31 
32 #include <stout/boundedhashmap.hpp>
33 #include <stout/duration.hpp>
34 #include <stout/hashmap.hpp>
35 #include <stout/hashset.hpp>
36 #include <stout/lambda.hpp>
37 #include <stout/option.hpp>
38 #include <stout/strings.hpp>
39 
41 
44 
47 
48 #include "master/constants.hpp"
49 
50 namespace mesos {
51 namespace internal {
52 namespace master {
53 namespace allocator {
54 
55 // We forward declare the hierarchical allocator process so that we
56 // can typedef an instantiation of it with DRF sorters.
57 template <
58  typename RoleSorter,
59  typename FrameworkSorter>
61 
64 
67 
70 
73 
74 
75 namespace internal {
76 
77 // Forward declarations.
78 class OfferFilter;
79 class InverseOfferFilter;
80 class RoleTree;
81 
82 
83 struct Framework
84 {
85  Framework(
86  const FrameworkInfo& frameworkInfo,
88  bool active,
90 
91  const FrameworkID frameworkId;
92 
93  FrameworkInfo info;
94 
95  std::set<std::string> roles;
96 
97  std::set<std::string> suppressedRoles;
98 
100 
101  // Offer filters are tied to the role the filtered
102  // resources were offered to.
105 
108 
109  bool active;
110 
112 
114 
115  // TODO(bbannier): Consider documenting examples on how to use this setting.
116  // TODO(asekretenko): reimplement minAllocatableResources in terms of
117  // offer constraints.
119 
121 };
122 
123 
124 // Helper for tracking cross-agent scalar resource totals.
125 // Needed because directly summing Resources across agents has
126 // prohibitively expensive time complexity: O(N^2) vs the number of agents,
127 // and also violates the convention that Resources belonging to different agents
128 // should not be added.
130 {
131 public:
132  // These methods implicitly filter out non-scalars from the inputs, thus
133  // the caller is not obliged to ensure that `resources` contains only scalars.
134  void add(const SlaveID& slaveID, const Resources& resources);
135  void subtract(const SlaveID& slaveID, const Resources& resources);
136 
137  bool empty() const { return scalars.empty(); }
138  ResourceQuantities quantities() const { return scalarsTotal; }
139 
140 private:
142  ResourceQuantities scalarsTotal;
143 };
144 
145 
146 class Role
147 {
148 public:
149  Role(const std::string& name, Role* parent);
150 
152  {
153  return reservationScalarQuantities_;
154  }
155 
157  {
158  return offeredOrAllocatedReserved.quantities();
159  }
160 
161  const hashset<FrameworkID>& frameworks() const { return frameworks_; }
162 
163  const Quota& quota() const { return quota_; }
164 
166  {
167  return offeredOrAllocatedUnreservedNonRevocable.quantities() +
168  reservationScalarQuantities_;
169  }
170 
172  {
173  return allocatedUnreservedNonRevocable.quantities() +
174  reservationScalarQuantities_;
175  }
176 
177  double weight() const { return weight_; }
178 
179  bool isEmpty() const
180  {
181  return children_.empty() &&
182  frameworks_.empty() &&
183  reservationScalarQuantities_.empty() &&
184  quota_ == DEFAULT_QUOTA &&
185  weight_ == DEFAULT_WEIGHT;
186  }
187 
188  std::vector<Role*> children() const { return children_.values(); }
189 
190  const std::string role; // E.g. "a/b/c"
191  const std::string basename; // E.g. "c"
192 
193 private:
194  // We keep fields that are related to the tree structure as private
195  // and only allow mutations through the RoleTree structure.
196  friend class RoleTree;
197 
198  // Add a child to the role, the child must not already exist.
199  void addChild(Role* child);
200 
201  // Remove a child from the role, the child must be present.
202  void removeChild(Role* child);
203 
204  Role* parent;
205 
206  // Configured guaranteed resource quantities and resource limits for
207  // this role. By default, a role has no guarantee and no limit.
208  Quota quota_;
209 
210  // Configured weight for the role. This affects sorting precedence.
211  // By default, weights == DEFAULT_WEIGHT == 1.0.
212  double weight_;
213 
214  // IDs of the frameworks tracked under the role, if any.
215  // A framework is tracked under the role if the framework:
216  //
217  // (1) is subscribed to the role;
218  // *OR*
219  // (2) has resources allocated under the role.
220  //
221  // NOTE: (2) could be true without (1). This is because the allocator
222  // interface allows for a framework role to be removed without recovering
223  // resources offered or allocated to this role.
224  hashset<FrameworkID> frameworks_;
225 
226  // Totals tracker for unreserved non-revocable offered/allocated resources.
227  // Note that since any offered or allocated resources should be tied to
228  // a framework, an empty role (that has no registered framework) must have
229  // this total empty.
230  ScalarResourceTotals offeredOrAllocatedUnreservedNonRevocable;
231 
232  ScalarResourceTotals offeredOrAllocatedReserved;
233 
234  // Aggregated reserved scalar resource quantities on all agents tied to this
235  // role, if any. This includes both its own reservations as well as
236  // reservations of any of its subroles (i.e. it is hierarchical aware).
237  // Note that non-scalar resources, such as ports, are excluded.
238  ResourceQuantities reservationScalarQuantities_;
239 
240  // Totals tracker for unreserved non-revocable resources actually allocated
241  // (i.e. used for launching tasks) to this role and any of its subroles.
242  ScalarResourceTotals allocatedUnreservedNonRevocable;
243 
244  hashmap<std::string, Role*> children_;
245 };
246 
247 
248 // A tree abstraction for organizing `class Role` hierarchically.
249 //
250 // We track a role when it has:
251 //
252 // * a non-default weight, or
253 // * a non-default quota, or
254 // * frameworks subscribed to it, or
255 // * reservations, or
256 // * descendent roles meeting any of the above conditions.
257 //
258 // Any roles that do not meet these conditions are not tracked in the role tree.
259 class RoleTree
260 {
261 public:
262  RoleTree(); // Only used in tests.
263 
265 
266  ~RoleTree();
267 
268  Option<const Role*> get(const std::string& role) const;
269 
270  // Return a hashmap of all known roles. Root is not included.
271  const hashmap<std::string, Role>& roles() const { return roles_; }
272 
273  const Role* root() const { return root_; }
274 
275  // We keep track of reservations to enforce role quota limit
276  // in the presence of unallocated reservations. See MESOS-4527.
277  void trackReservations(const Resources& resources);
278  void untrackReservations(const Resources& resources);
279 
280  // We keep track of allocated resources which are actually used by frameworks.
281  void trackAllocated(const SlaveID& slaveId, const Resources& resources);
282  void untrackAllocated(const SlaveID& slaveId, const Resources& resources);
283 
284  void trackFramework(
285  const FrameworkID& frameworkId, const std::string& role);
286  void untrackFramework(
287  const FrameworkID& frameworkId, const std::string& role);
288 
289  void updateQuota(const std::string& role, const Quota& quota);
290 
291  void updateWeight(const std::string& role, double weight);
292 
293  void trackOfferedOrAllocated(
294  const SlaveID& slaveId,
295  const Resources& resources);
296 
297  void untrackOfferedOrAllocated(
298  const SlaveID& slaveId,
299  const Resources& resources);
300 
301  // Dump the role tree state in JSON format for debugging.
302  std::string toJSON() const;
303 
304 private:
305  // Private helper to get non-const pointers.
306  Option<Role*> get_(const std::string& role);
307 
308  // Lookup or add the role struct associated with the role. Ancestor roles
309  // along the tree path will be created if necessary.
310  Role& operator[](const std::string& role);
311 
312  // Helper for modifying a role and all its ancestors.
313  template<class UnaryFunction>
314  static void applyToRoleAndAncestors(Role* role, UnaryFunction f) {
315  for (; role != nullptr; role = role->parent) {
316  f(role);
317  }
318  }
319 
320  // Try to remove the role associated with the given role.
321  // The role must exist. The role and its ancestors will be removed
322  // if they become "empty". See "Role:isEmpty()".
323  // Return true if the role instance associated with the role is removed.
324  // This should be called whenever a role's state (that defines its emptiness)
325  // gets updated, such as quota, weight, reservation and tracked frameworks.
326  // Otherwise the "tracking only non-empty" tree invariant may break.
327  bool tryRemove(const std::string& role);
328 
329  void updateQuotaConsumedMetric(const Role* role);
330 
331  // Root node of the tree, its `basename` == `role` == "".
332  Role* root_;
333 
334  // Allocator's metrics handle for publishing role related metrics.
336 
337  // A map of role and `Role` pairs for quick lookup.
339 };
340 
341 
342 class Slave
343 {
344 public:
346  const SlaveInfo& _info,
347  const protobuf::slave::Capabilities& _capabilities,
348  bool _activated,
349  const Resources& _total,
350  const hashmap<FrameworkID, Resources>& _allocated)
351  : id(_info.id()),
352  info(_info),
353  capabilities(_capabilities),
354  activated(_activated),
355  totalAllocated(Resources::sum(_allocated)),
356  total(_total),
357  offeredOrAllocated(_allocated),
358  totalOfferedOrAllocated(Resources::sum(_allocated)),
359  shared(_total.shared()),
360  hasGpu_(_total.gpus().getOrElse(0) > 0)
361  {
362  CHECK(_info.has_id());
363  updateAvailable();
364  }
365 
366  const Resources& getTotal() const { return total; }
367 
369  {
370  return offeredOrAllocated;
371  }
372 
374  {
375  return totalOfferedOrAllocated;
376  }
377 
378  const Resources& getAvailable() const { return available; }
379 
380  bool hasGpu() const { return hasGpu_; }
381 
382  void updateTotal(const Resources& newTotal) {
383  total = newTotal;
384  shared = total.shared();
385  hasGpu_ = total.gpus().getOrElse(0) > 0;
386 
387  updateAvailable();
388  }
389 
391  const FrameworkID& frameworkId, const Resources& offeredOrAllocated_)
392  {
393  // Increasing available is to subtract offered or allocated.
394  if (offeredOrAllocated_.empty()) {
395  return;
396  }
397 
398  // It is possible that the reference of `offeredOrAllocated_`
399  // points to the same object as `resources` below. We must
400  // do subtraction here before any mutation on the object.
401  totalOfferedOrAllocated -= offeredOrAllocated_;
402 
403  Resources& resources = offeredOrAllocated.at(frameworkId);
404  CHECK_CONTAINS(resources, offeredOrAllocated_);
405  resources -= offeredOrAllocated_;
406  if (resources.empty()) {
407  offeredOrAllocated.erase(frameworkId);
408  }
409 
410  updateAvailable();
411  }
412 
414  const FrameworkID& frameworkId, const Resources& offeredOrAllocated_)
415  {
416  if (offeredOrAllocated_.empty()) {
417  return;
418  }
419 
420  // Decreasing available is to add offered or allocated.
421 
422  offeredOrAllocated[frameworkId] += offeredOrAllocated_;
423 
424  totalOfferedOrAllocated += offeredOrAllocated_;
425 
426  updateAvailable();
427  }
428 
429  const SlaveID id;
430 
431  // The `SlaveInfo` that was passed to the allocator when the slave was added
432  // or updated. Currently only two fields are used: `hostname` for host
433  // whitelisting and in log messages, and `domain` for region-aware
434  // scheduling.
435  SlaveInfo info;
436 
438 
439  bool activated; // Whether to offer resources.
440 
441  // Represents a scheduled unavailability due to maintenance for a specific
442  // slave, and the responses from frameworks as to whether they will be able
443  // to gracefully handle this unavailability.
444  //
445  // NOTE: We currently implement maintenance in the allocator to be able to
446  // leverage state and features such as the FrameworkSorter and OfferFilter.
447  struct Maintenance
448  {
449  Maintenance(const Unavailability& _unavailability)
450  : unavailability(_unavailability) {}
451 
452  // The start time and optional duration of the event.
453  Unavailability unavailability;
454 
455  // A mapping of frameworks to the inverse offer status associated with
456  // this unavailability.
457  //
458  // NOTE: We currently lose this information during a master fail over
459  // since it is not persisted or replicated. This is ok as the new master's
460  // allocator will send out new inverse offers and re-collect the
461  // information. This is similar to all the outstanding offers from an old
462  // master being invalidated, and new offers being sent out.
464 
465  // Represents the "unit of accounting" for maintenance. When a
466  // `FrameworkID` is present in the hashset it means an inverse offer has
467  // been sent out. When it is not present it means no offer is currently
468  // outstanding.
470  };
471 
472  // When the `maintenance` is set the slave is scheduled to be unavailable at
473  // a given point in time, for an optional duration. This information is used
474  // to send out `InverseOffers`.
476 
477  // Sum of all allocated (i.e. occupied by running tasks) resources on the
478  // agent. This information is needed to untrack allocated resources when the
479  // agent is removed, because the master is not obligated to separately inform
480  // allocator that resources of the removed agent are not offered/allocated
481  // anymore.
483 
484 private:
485  void updateAvailable()
486  {
487  // In order to subtract from the total,
488  // we strip the allocation information.
489  Resources totalOfferedOrAllocated_ = totalOfferedOrAllocated;
490  totalOfferedOrAllocated_.unallocate();
491 
492  // This is hot path. We avoid the unnecessary resource traversals
493  // in the common case where there are no shared resources.
494  if (shared.empty()) {
495  available = total - totalOfferedOrAllocated_;
496  } else {
497  // Since shared resources are offerable even when they are in use, we
498  // always include them as part of available resources.
499  available =
500  (total.nonShared() - totalOfferedOrAllocated_.nonShared()) + shared;
501  }
502  }
503 
504  // Total amount of regular *and* oversubscribed resources.
505  Resources total;
506 
507  // NOTE: We keep track of the slave's allocated resources despite
508  // having that information in sorters. This is because the
509  // information in sorters is not accurate if some framework
510  // hasn't reregistered. See MESOS-2919 for details.
511  //
512  // This includes both regular *and* oversubscribed resources.
513  //
514  // An entry is erased if a framework no longer has any
515  // offered or allocated on the agent.
516  hashmap<FrameworkID, Resources> offeredOrAllocated;
517 
518  // Sum of all offered or allocated resources on the agent. This should equal
519  // to sum of `offeredOrAllocated` (including all the meta-data).
520  Resources totalOfferedOrAllocated;
521 
522  // We track the total and allocated resources on the slave to
523  // avoid calculating it in place every time.
524  //
525  // Note that `available` always contains all the shared resources on the
526  // agent regardless whether they have ever been allocated or not.
527  // NOTE, however, we currently only offer a shared resource only if it has
528  // not been offered in an allocation cycle to a framework. We do this mainly
529  // to preserve the normal offer behavior. This may change in the future
530  // depending on use cases.
531  //
532  // Note that it's possible for the slave to be over-allocated!
533  // In this case, allocated > total.
534  Resources available;
535 
536  // We keep a copy of the shared resources to avoid unnecessary copying.
537  Resources shared;
538 
539  // We cache whether the agent has gpus as an optimization.
540  bool hasGpu_;
541 };
542 
543 
544 // Implements the basic allocator algorithm - first pick a role by
545 // some criteria, then pick one of their frameworks to allocate to.
547 {
548 public:
550  const std::function<Sorter*()>& roleSorterFactory,
551  const std::function<Sorter*()>& _frameworkSorterFactory)
552  : initialized(false),
553  paused(true),
554  metrics(*this),
555  completedFrameworkMetrics(0),
556  roleTree(&metrics),
557  roleSorter(roleSorterFactory()),
558  frameworkSorterFactory(_frameworkSorterFactory) {}
559 
561 
563  {
564  return process::PID<Self>(this);
565  }
566 
567  void initialize(
568  const mesos::allocator::Options& options,
569  const lambda::function<
570  void(const FrameworkID&,
571  const hashmap<std::string, hashmap<SlaveID, Resources>>&)>&
572  offerCallback,
573  const lambda::function<
574  void(const FrameworkID&,
576  inverseOfferCallback) override;
577 
578  void recover(
579  const int _expectedAgentCount,
580  const hashmap<std::string, Quota>& quotas) override;
581 
582  void addFramework(
583  const FrameworkID& frameworkId,
584  const FrameworkInfo& frameworkInfo,
586  bool active,
587  ::mesos::allocator::FrameworkOptions&& frameworkOptions) override;
588 
589  void removeFramework(
590  const FrameworkID& frameworkId) override;
591 
592  void activateFramework(
593  const FrameworkID& frameworkId) override;
594 
595  void deactivateFramework(
596  const FrameworkID& frameworkId) override;
597 
598  void updateFramework(
599  const FrameworkID& frameworkId,
600  const FrameworkInfo& frameworkInfo,
601  ::mesos::allocator::FrameworkOptions&& frameworkOptions) override;
602 
603  void addSlave(
604  const SlaveID& slaveId,
605  const SlaveInfo& slaveInfo,
606  const std::vector<SlaveInfo::Capability>& capabilities,
608  const Resources& total,
609  const hashmap<FrameworkID, Resources>& used) override;
610 
611  void removeSlave(
612  const SlaveID& slaveId) override;
613 
614  void updateSlave(
615  const SlaveID& slave,
616  const SlaveInfo& slaveInfo,
617  const Option<Resources>& total = None(),
618  const Option<std::vector<SlaveInfo::Capability>>& capabilities = None())
619  override;
620 
621  void addResourceProvider(
622  const SlaveID& slave,
623  const Resources& total,
624  const hashmap<FrameworkID, Resources>& used) override;
625 
626  void deactivateSlave(
627  const SlaveID& slaveId) override;
628 
629  void activateSlave(
630  const SlaveID& slaveId) override;
631 
632  void updateWhitelist(
633  const Option<hashset<std::string>>& whitelist) override;
634 
635  void requestResources(
636  const FrameworkID& frameworkId,
637  const std::vector<Request>& requests) override;
638 
639  void updateAllocation(
640  const FrameworkID& frameworkId,
641  const SlaveID& slaveId,
642  const Resources& offeredResources,
643  const std::vector<ResourceConversion>& conversions) override;
644 
645  process::Future<Nothing> updateAvailable(
646  const SlaveID& slaveId,
647  const std::vector<Offer::Operation>& operations) override;
648 
649  void updateUnavailability(
650  const SlaveID& slaveId,
651  const Option<Unavailability>& unavailability) override;
652 
653  void updateInverseOffer(
654  const SlaveID& slaveId,
655  const FrameworkID& frameworkId,
656  const Option<UnavailableResources>& unavailableResources,
658  const Option<Filters>& filters) override;
659 
661  hashmap<SlaveID,
663  getInverseOfferStatuses() override;
664 
665  void transitionOfferedToAllocated(
666  const SlaveID& slaveId, const Resources& resources) override;
667 
668  void recoverResources(
669  const FrameworkID& frameworkId,
670  const SlaveID& slaveId,
671  const Resources& resources,
672  const Option<Filters>& filters,
673  bool isAllocated) override;
674 
675  void suppressOffers(
676  const FrameworkID& frameworkId,
677  const std::set<std::string>& roles) override;
678 
679  void reviveOffers(
680  const FrameworkID& frameworkId,
681  const std::set<std::string>& roles) override;
682 
683  void updateQuota(
684  const std::string& role,
685  const Quota& quota) override;
686 
687  void updateWeights(
688  const std::vector<WeightInfo>& weightInfos) override;
689 
690  void pause() override;
691 
692  void resume() override;
693 
694 protected:
695  // Useful typedefs for dispatch/delay/defer to self()/this.
698 
699  // Generate offers from all known agents.
700  process::Future<Nothing> generateOffers();
701 
702  // Generate offers from the specified agent.
703  process::Future<Nothing> generateOffers(const SlaveID& slaveId);
704 
705  // Generate offers from the specified agents. The offer generation is
706  // deferred and batched with other offer generation requests.
707  process::Future<Nothing> generateOffers(const hashset<SlaveID>& slaveIds);
708 
709  Nothing _generateOffers();
710 
711  void __generateOffers();
712 
713  void generateInverseOffers();
714 
715  // Remove an offer filter for the specified role of the framework.
716  void expire(
717  const FrameworkID& frameworkId,
718  const std::string& role,
719  const SlaveID& slaveId,
720  const std::weak_ptr<OfferFilter>& offerFilter);
721 
722  void _expire(
723  const FrameworkID& frameworkId,
724  const std::string& role,
725  const SlaveID& slaveId,
726  const std::weak_ptr<OfferFilter>& offerFilter);
727 
728  // Remove an inverse offer filter for the specified framework.
729  void expire(
730  const FrameworkID& frameworkId,
731  const SlaveID& slaveId,
732  const std::weak_ptr<InverseOfferFilter>& inverseOfferFilter);
733 
734  // Checks whether the slave is whitelisted.
735  bool isWhitelisted(const SlaveID& slaveId) const;
736 
737  // Returns true if there is a resource offer filter for the
738  // specified role of this framework on this slave.
739  bool isFiltered(
740  const Framework& framework,
741  const std::string& role,
742  const Slave& slave,
743  const Resources& resources) const;
744 
745  // Returns true if there is an inverse offer filter for this framework
746  // on this slave.
747  bool isFiltered(
748  const Framework& framework,
749  const Slave& slave) const;
750 
751  bool allocatable(
752  const Resources& resources,
753  const std::string& role,
754  const Framework& framework) const;
755 
757  bool paused;
758 
760 
761  // Recovery data.
763 
764  lambda::function<
765  void(const FrameworkID&,
766  const hashmap<std::string, hashmap<SlaveID, Resources>>&)>
768 
769  lambda::function<
770  void(const FrameworkID&,
773 
774  friend Metrics;
776 
778  {
779  return static_cast<double>(eventCount<process::DispatchEvent>());
780  }
781 
782  double _resources_total(
783  const std::string& resource);
784 
785  double _resources_offered_or_allocated(
786  const std::string& resource);
787 
788  double _quota_offered_or_allocated(
789  const std::string& role,
790  const std::string& resource);
791 
792  double _offer_filters_active(
793  const std::string& role);
794 
796 
799 
801 
802  // Total scalar resource quantities on all agents.
804 
806 
807  // A set of agents that are kept as allocation candidates. Events
808  // may add or remove candidates to the set. When an offer generation is
809  // processed, the set of candidates is cleared.
811 
812  // Future for the dispatched offer generation that becomes
813  // ready after the offer generation run is complete.
815 
816  // Slaves to send offers for.
818 
819  // There are two stages of offer generation:
820  //
821  // Stage 1: Generate offers to satisfy quota guarantees.
822  //
823  // Stage 2: Generate offers above quota guarantees up to quota limits.
824  // Note that we need to hold back enough "headroom"
825  // to ensure that any unsatisfied quota can be
826  // satisfied later.
827  //
828  // Each stage comprises two levels of sorting, hence "hierarchical".
829  // Level 1 sorts across roles:
830  // Currently, only the offered or allocated portion of the reserved
831  // resources are accounted for fairness calculation.
832  //
833  // TODO(mpark): Reserved resources should be accounted for fairness
834  // calculation whether they are offered/allocated or not, since they model
835  // a long or forever running task. That is, the effect of reserving resources
836  // is equivalent to launching a task in that the resources that make up the
837  // reservation are not available to other roles as non-revocable.
838  //
839  // Level 2 sorts across frameworks within a particular role:
840  // Reserved resources at this level are, and should be accounted for
841  // fairness calculation only if they are allocated. This is because
842  // reserved resources are fairly shared across the frameworks in the role.
843  //
844  // The allocator relies on `Sorter`s to employ a particular sorting
845  // algorithm. Each level has its own sorter and hence may have different
846  // fairness calculations.
847  //
848  // NOTE: The hierarchical allocator considers revocable resources as
849  // regular resources when doing fairness calculations.
850  //
851  // TODO(vinod): Consider using a different fairness algorithm for
852  // revocable resources.
853 
854  // A sorter for active roles. This sorter determines the order in which
855  // roles are offered resources during Level 1 of the second stage.
856  // The total cluster resources are used as the resource pool.
858 
859  // A collection of sorters, one per active role. Each sorter determines
860  // the order in which frameworks that belong to the same role are offered
861  // resources inside the role's share. These sorters are used during Level 2
862  // for both the first and the second stages. Since frameworks are sharing
863  // resources of a role, resources offered or allocated to the role are used as
864  // the resource pool for each role specific framework sorter.
866 
867  // Factory function for framework sorters.
868  const std::function<Sorter*()> frameworkSorterFactory;
869 
870 private:
871  process::Future<process::http::Response> offerConstraintsDebug(
872  const process::http::Request&,
874 
875  process::http::Response offerConstraintsDebug_(
876  std::shared_ptr<const ObjectApprover> frameworksApprover);
877 
878  bool isFrameworkTrackedUnderRole(
879  const FrameworkID& frameworkId,
880  const std::string& role) const;
881 
882  Option<Slave*> getSlave(const SlaveID& slaveId) const;
883  Option<Framework*> getFramework(const FrameworkID& frameworkId) const;
884 
885  Option<Sorter*> getFrameworkSorter(const std::string& role) const;
886 
887  const Quota& getQuota(const std::string& role) const;
888 
889  // Helpers to track and untrack a framework under a role.
890  // Frameworks should be tracked under a role either if it subscribes to the
891  // role *OR* it has resources allocated/offered to that role. when neither
892  // conditions are met, it should be untracked.
893  //
894  // `tryUntrackFrameworkUnderRole` returns true if the framework is untracked
895  // under the role.
896  void trackFrameworkUnderRole(
897  const Framework& framework, const std::string& role);
898  bool tryUntrackFrameworkUnderRole(
899  const Framework& framework, const std::string& role);
900 
901  void suppressRoles(Framework& framework, const std::set<std::string>& roles);
902  void reviveRoles(Framework& framework, const std::set<std::string>& roles);
903 
904  // Helper to update the agent's total resources maintained in the allocator
905  // and the role and quota sorters (whose total resources match the agent's
906  // total resources). Returns true iff the stored agent total was changed.
907  bool updateSlaveTotal(const SlaveID& slaveId, const Resources& total);
908 
909  // Helper that returns true if the given agent is located in a
910  // different region than the master. This can only be the case if
911  // the agent and the master are both configured with a fault domain.
912  bool isRemoteSlave(const Slave& slave) const;
913 
914  // Helper function that checks if a framework is capable of
915  // receiving resources on the agent based on the framework capability.
916  //
917  // TODO(mzhu): Make this a `Framework` member function once we pull
918  // `struct Framework` out from being nested.
919  bool isCapableOfReceivingAgent(
920  const protobuf::framework::Capabilities& frameworkCapabilities,
921  const Slave& slave) const;
922 
923  // Helper function that removes any resources that the framework is not
924  // capable of receiving based on the given framework capability.
925  //
926  // TODO(mzhu): Make this a `Framework` member function once we pull
927  // `struct Framework` out from being nested.
928  Resources stripIncapableResources(
929  const Resources& resources,
930  const protobuf::framework::Capabilities& frameworkCapabilities) const;
931 
932  // Helper to track offered or allocated resources on an agent.
933  //
934  // TODO(asekretenko): rename `(un)trackAllocatedResources()` to reflect the
935  // fact that these methods do not distinguish between offered and allocated.
936  //
937  // TODO(mzhu): replace this with `RoleTree::trackOfferedOrAllocated`.
938  void trackAllocatedResources(
939  const SlaveID& slaveId,
940  const FrameworkID& frameworkId,
941  const Resources& offeredOrAllocated);
942 
943  // Helper to untrack resources that are no longer offered or allocated
944  // on an agent.
945  //
946  // TODO(mzhu): replace this with `RoleTree::untrackOfferedOrAllocated`.
947  void untrackAllocatedResources(
948  const SlaveID& slaveId,
949  const FrameworkID& frameworkId,
950  const Resources& offeredOrallocated);
951 
952  // Helper that removes all existing offer filters for the given slave
953  // id.
954  void removeFilters(const SlaveID& slaveId);
955 };
956 
957 
958 } // namespace internal {
959 
960 
961 // We map the templatized version of the `HierarchicalAllocatorProcess` to one
962 // that relies on sorter factories in the internal namespace. This allows us
963 // to keep the implementation of the allocator in the implementation file.
964 template <
965  typename RoleSorter,
966  typename FrameworkSorter>
969 {
970 public:
972  : ProcessBase(strings::remove(
973  process::ID::generate("allocator"), "(1)", strings::Mode::SUFFIX)),
975  [this]() -> Sorter* {
976  return new RoleSorter(this->self(), "allocator/mesos/roles/");
977  },
978  []() -> Sorter* { return new FrameworkSorter(); }) {}
979 };
980 
981 } // namespace allocator {
982 } // namespace master {
983 } // namespace internal {
984 } // namespace mesos {
985 
986 #endif // __MASTER_ALLOCATOR_MESOS_HIERARCHICAL_HPP__
Protocol< RecoverRequest, RecoverResponse > recover
std::string generate(const std::string &prefix="")
Returns &#39;prefix(N)&#39; where N represents the number of instances where the same prefix (wrt...
std::set< std::string > roles
Definition: hierarchical.hpp:95
#define CHECK_CONTAINS(container, key)
Definition: check.hpp:298
HierarchicalAllocatorProcess(const std::function< Sorter *()> &roleSorterFactory, const std::function< Sorter *()> &_frameworkSorterFactory)
Definition: hierarchical.hpp:549
Per-framework allocator-specific options that are not part of FrameworkInfo.
Definition: allocator.hpp:147
mesos::allocator::Options options
Definition: hierarchical.hpp:759
Definition: nothing.hpp:16
Definition: option.hpp:29
T getOrElse(U &&u) const &
Definition: option.hpp:133
const hashmap< FrameworkID, Resources > & getOfferedOrAllocated() const
Definition: hierarchical.hpp:368
const Resources & getTotal() const
Definition: hierarchical.hpp:366
Maintenance(const Unavailability &_unavailability)
Definition: hierarchical.hpp:449
F && f
Definition: defer.hpp:270
protobuf::framework::Capabilities capabilities
Definition: hierarchical.hpp:99
hashmap< std::string, hashmap< SlaveID, hashset< std::shared_ptr< OfferFilter > > > > offerFilters
Definition: hierarchical.hpp:104
Definition: master.hpp:27
HierarchicalAllocatorProcess< RandomSorter, RandomSorter > HierarchicalRandomAllocatorProcess
Definition: hierarchical.hpp:69
hashmap< std::string, process::Owned< Sorter > > frameworkSorters
Definition: hierarchical.hpp:865
Definition: resource_quantities.hpp:63
BoundedHashMap< FrameworkID, process::Owned< FrameworkMetrics > > completedFrameworkMetrics
Definition: hierarchical.hpp:798
Definition: protobuf_utils.hpp:332
Definition: format.hpp:45
Result< ProcessStatus > status(pid_t pid)
Definition: proc.hpp:166
Definition: uuid.hpp:33
Option< int > expectedAgentCount
Definition: hierarchical.hpp:762
Definition: resources.hpp:83
Slave * getSlave(Master *master, const SlaveID &slaveId)
Slave(const SlaveInfo &_info, const protobuf::slave::Capabilities &_capabilities, bool _activated, const Resources &_total, const hashmap< FrameworkID, Resources > &_allocated)
Definition: hierarchical.hpp:345
Option< double > gpus() const
~HierarchicalAllocatorProcess() override
Definition: hierarchical.hpp:560
Pass in configuration to the allocator.
Definition: allocator.hpp:53
ResourceQuantities quantities() const
Definition: hierarchical.hpp:138
const hashmap< std::string, Role > & roles() const
Definition: hierarchical.hpp:271
hashmap< SlaveID, Slave > slaves
Definition: hierarchical.hpp:800
constexpr double DEFAULT_WEIGHT
Definition: constants.hpp:180
MesosAllocator< HierarchicalRandomAllocatorProcess > HierarchicalRandomAllocator
Definition: hierarchical.hpp:72
Future< Nothing > add(const T &metric)
Definition: metrics.hpp:95
const std::string basename
Definition: hierarchical.hpp:191
SlaveInfo info
Definition: hierarchical.hpp:435
bool hasGpu() const
Definition: hierarchical.hpp:380
::mesos::allocator::OfferConstraintsFilter offerConstraintsFilter
Definition: hierarchical.hpp:120
ResourceQuantities quotaOfferedOrConsumed() const
Definition: hierarchical.hpp:165
MesosAllocator< HierarchicalDRFAllocatorProcess > HierarchicalDRFAllocator
Definition: hierarchical.hpp:66
Definition: http.hpp:533
std::vector< Role * > children() const
Definition: hierarchical.hpp:188
const std::function< Sorter *()> frameworkSorterFactory
Definition: hierarchical.hpp:868
bool activated
Definition: hierarchical.hpp:439
Definition: hashmap.hpp:38
const SlaveID id
Definition: hierarchical.hpp:429
lambda::function< void(const FrameworkID &, const hashmap< std::string, hashmap< SlaveID, Resources >> &)> offerCallback
Definition: hierarchical.hpp:767
bool active
Definition: hierarchical.hpp:109
Try< Nothing > unavailability(const Unavailability &unavailability)
process::Owned< Sorter > roleSorter
Definition: hierarchical.hpp:857
std::set< std::string > suppressedRoles
Definition: hierarchical.hpp:97
Try< Bytes > used(const std::string &path="/")
Definition: fs.hpp:43
Try< Nothing > initialize(const Flags &flags)
Initialized state for support of systemd functions in this file.
hashmap< FrameworkID, mesos::allocator::InverseOfferStatus > statuses
Definition: hierarchical.hpp:463
Result< std::vector< Filter< Classifier > > > filters(const std::string &_link, const Handle &parent)
Definition: internal.hpp:769
ResourceQuantities totalScalarQuantities
Definition: hierarchical.hpp:803
const FrameworkID frameworkId
Definition: hierarchical.hpp:91
double weight() const
Definition: hierarchical.hpp:177
Definition: protobuf_utils.hpp:631
Definition: agent.hpp:25
hashmap< FrameworkID, Framework > frameworks
Definition: hierarchical.hpp:795
HierarchicalAllocatorProcess< DRFSorter, DRFSorter > HierarchicalDRFAllocatorProcess
Definition: hierarchical.hpp:60
bool publishPerFrameworkMetrics
Definition: hierarchical.hpp:111
const Quota & quota() const
Definition: hierarchical.hpp:163
Definition: allocator.hpp:91
ResourceQuantities offeredOrAllocatedReservedScalarQuantities() const
Definition: hierarchical.hpp:156
const ResourceQuantities & reservationScalarQuantities() const
Definition: hierarchical.hpp:151
hashmap< SlaveID, hashset< std::shared_ptr< InverseOfferFilter > > > inverseOfferFilters
Definition: hierarchical.hpp:107
const Role * root() const
Definition: hierarchical.hpp:273
process::Owned< FrameworkMetrics > metrics
Definition: hierarchical.hpp:113
Framework(const FrameworkInfo &frameworkInfo,::mesos::allocator::FrameworkOptions &&options, bool active, bool publishPerFrameworkMetrics)
Option< hashset< std::string > > whitelist
Definition: hierarchical.hpp:817
hashset< SlaveID > allocationCandidates
Definition: hierarchical.hpp:810
Definition: boundedhashmap.hpp:27
Try< bool > remove(const std::string &link, const Handle &parent, uint16_t protocol)
FrameworkInfo info
Definition: hierarchical.hpp:93
const Resources & getAvailable() const
Definition: hierarchical.hpp:378
A "process identifier" used to uniquely identify a process when dispatching messages.
Definition: pid.hpp:289
Resources shared() const
bool empty() const
Definition: resources.hpp:451
Definition: none.hpp:27
Definition: attributes.hpp:24
bool isEmpty() const
Definition: hierarchical.hpp:179
hashset< FrameworkID > offersOutstanding
Definition: hierarchical.hpp:469
Resources nonShared() const
Option< process::Future< Nothing > > offerGeneration
Definition: hierarchical.hpp:814
Definition: executor.hpp:48
Definition: http.hpp:612
Unavailability unavailability
Definition: hierarchical.hpp:453
HierarchicalAllocatorProcess This
Definition: hierarchical.hpp:697
hashmap< std::string, std::vector< ResourceQuantities > > minAllocatableResources
Definition: hierarchical.hpp:118
ResourceQuantities quotaConsumed() const
Definition: hierarchical.hpp:171
bool empty() const
Definition: hierarchical.hpp:137
const std::string role
Definition: hierarchical.hpp:190
void increaseAvailable(const FrameworkID &frameworkId, const Resources &offeredOrAllocated_)
Definition: hierarchical.hpp:390
HierarchicalAllocatorProcess Self
Definition: hierarchical.hpp:696
Option< Maintenance > maintenance
Definition: hierarchical.hpp:475
const Resources & getTotalOfferedOrAllocated() const
Definition: hierarchical.hpp:373
void decreaseAvailable(const FrameworkID &frameworkId, const Resources &offeredOrAllocated_)
Definition: hierarchical.hpp:413
const Quota DEFAULT_QUOTA
Definition: constants.hpp:177
Definition: owned.hpp:36
Resources totalAllocated
Definition: hierarchical.hpp:482
const hashset< FrameworkID > & frameworks() const
Definition: hierarchical.hpp:161
Definition: quota.hpp:27
constexpr const char * name
Definition: shell.hpp:41
protobuf::slave::Capabilities capabilities
Definition: hierarchical.hpp:437
void updateTotal(const Resources &newTotal)
Definition: hierarchical.hpp:382
lambda::function< void(const FrameworkID &, const hashmap< SlaveID, UnavailableResources > &)> inverseOfferCallback
Definition: hierarchical.hpp:772