Apache Mesos
containerizer.hpp
Go to the documentation of this file.
1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 
17 #ifndef __MESOS_CONTAINERIZER_HPP__
18 #define __MESOS_CONTAINERIZER_HPP__
19 
20 #include <vector>
21 
23 
24 #include <mesos/slave/isolator.hpp>
25 
26 #include <process/id.hpp>
27 #include <process/http.hpp>
28 #include <process/sequence.hpp>
29 #include <process/shared.hpp>
30 
32 
33 #include <stout/hashmap.hpp>
34 #include <stout/multihashmap.hpp>
35 #include <stout/os/int_fd.hpp>
36 
37 #include "slave/gc.hpp"
38 #include "slave/state.hpp"
39 
41 
43 
45 
47 
49 
50 namespace mesos {
51 namespace internal {
52 namespace slave {
53 
54 // If the container class is not of type `DEBUG` (i.e., it is not set or
55 // `DEFAULT`), we log the line at the INFO level. Otherwise, we use VLOG(1).
56 // The purpose of this macro is to avoid polluting agent logs with information
57 // related to `DEBUG` containers as this type of container can run periodically.
58 #define LOG_BASED_ON_CLASS(containerClass) \
59  LOG_IF(INFO, (containerClass != ContainerClass::DEBUG) || VLOG_IS_ON(1))
60 
61 // Forward declaration.
62 class MesosContainerizerProcess;
63 
64 
66 {
67 public:
69  const Flags& flags,
70  bool local,
71  Fetcher* fetcher,
72  GarbageCollector* gc = nullptr,
73  SecretResolver* secretResolver = nullptr,
74  const Option<NvidiaComponents>& nvidia = None());
75 
77  const Flags& flags,
78  bool local,
79  Fetcher* fetcher,
80  GarbageCollector* gc,
81  const process::Owned<Launcher>& launcher,
82  const process::Shared<Provisioner>& provisioner,
83  const std::vector<process::Owned<mesos::slave::Isolator>>& isolators);
84 
85  ~MesosContainerizer() override;
86 
88  const Option<state::SlaveState>& state) override;
89 
91  const ContainerID& containerId,
92  const mesos::slave::ContainerConfig& containerConfig,
93  const std::map<std::string, std::string>& environment,
94  const Option<std::string>& pidCheckpointPath) override;
95 
97  const ContainerID& containerId) override;
98 
100  const ContainerID& containerId,
101  const Resources& resources) override;
102 
104  const ContainerID& containerId) override;
105 
107  const ContainerID& containerId) override;
108 
110  const ContainerID& containerId) override;
111 
113  const ContainerID& containerId) override;
114 
116  const ContainerID& containerId,
117  int signal) override;
118 
120 
121  process::Future<Nothing> remove(const ContainerID& containerId) override;
122 
124  const std::vector<Image>& excludedImages) override;
125 
126 private:
127  explicit MesosContainerizer(
129 
131 };
132 
133 
135  : public process::Process<MesosContainerizerProcess>
136 {
137 public:
139  const Flags& _flags,
140  Fetcher* _fetcher,
141  GarbageCollector* _gc,
142  IOSwitchboard* _ioSwitchboard,
143  const process::Owned<Launcher>& _launcher,
144  const process::Shared<Provisioner>& _provisioner,
145  const std::vector<process::Owned<mesos::slave::Isolator>>& _isolators)
146  : ProcessBase(process::ID::generate("mesos-containerizer")),
147  flags(_flags),
148  fetcher(_fetcher),
149  gc(_gc),
150  ioSwitchboard(_ioSwitchboard),
151  launcher(_launcher),
152  provisioner(_provisioner),
153  isolators(_isolators) {}
154 
156 
158  const Option<state::SlaveState>& state);
159 
161  const ContainerID& containerId,
162  const mesos::slave::ContainerConfig& containerConfig,
163  const std::map<std::string, std::string>& environment,
164  const Option<std::string>& pidCheckpointPath);
165 
167  const ContainerID& containerId);
168 
170  const ContainerID& containerId,
171  const Resources& resources);
172 
174  const ContainerID& containerId);
175 
177  const ContainerID& containerId);
178 
180  const ContainerID& containerId);
181 
183  const ContainerID& containerId,
184  int_fd pipeWrite);
185 
187  const ContainerID& containerId,
188  const Option<mesos::slave::ContainerTermination>& termination);
189 
190  virtual process::Future<bool> kill(
191  const ContainerID& containerId,
192  int signal);
193 
194  virtual process::Future<Nothing> remove(const ContainerID& containerId);
195 
197 
199  const std::vector<Image>& excludedImages);
200 
201 private:
202  enum State
203  {
204  PROVISIONING,
205  PREPARING,
206  ISOLATING,
207  FETCHING,
208  RUNNING,
209  DESTROYING
210  };
211 
212  friend std::ostream& operator<<(std::ostream& stream, const State& state);
213 
214  process::Future<Nothing> _recover(
215  const std::vector<mesos::slave::ContainerState>& recoverable,
216  const hashset<ContainerID>& orphans);
217 
218  process::Future<std::vector<Nothing>> recoverIsolators(
219  const std::vector<mesos::slave::ContainerState>& recoverable,
220  const hashset<ContainerID>& orphans);
221 
222  process::Future<Nothing> recoverProvisioner(
223  const std::vector<mesos::slave::ContainerState>& recoverable,
224  const hashset<ContainerID>& orphans);
225 
226  process::Future<Nothing> __recover(
227  const std::vector<mesos::slave::ContainerState>& recovered,
228  const hashset<ContainerID>& orphans);
229 
231  const ContainerID& containerId,
232  const Option<ProvisionInfo>& provisionInfo);
233 
235  const ContainerID& containerId);
236 
238  const ContainerID& containerId,
239  const Option<mesos::slave::ContainerIO>& containerIO,
240  const std::map<std::string, std::string>& environment,
241  const Option<std::string>& pidCheckpointPath);
242 
244  const ContainerID& containerId,
245  pid_t _pid);
246 
247  // Continues 'destroy()' once nested containers are handled.
248  void _destroy(
249  const ContainerID& containerId,
251  const State& previousState,
252  const std::vector<
254 
255  // Continues '_destroy()' once isolators has completed.
256  void __destroy(
257  const ContainerID& containerId,
258  const Option<mesos::slave::ContainerTermination>& termination);
259 
260  // Continues '__destroy()' once all processes have been killed
261  // by the launcher.
262  void ___destroy(
263  const ContainerID& containerId,
265  const process::Future<Nothing>& future);
266 
267  // Continues '___destroy()' once we get the exit status of the container.
268  void ____destroy(
269  const ContainerID& containerId,
270  const Option<mesos::slave::ContainerTermination>& termination);
271 
272  // Continues '____destroy()' once all isolators have completed
273  // cleanup.
274  void _____destroy(
275  const ContainerID& containerId,
277  const process::Future<std::vector<process::Future<Nothing>>>& cleanups);
278 
279  // Continues '_____destroy()' once provisioner have completed destroy.
280  void ______destroy(
281  const ContainerID& containerId,
283  const process::Future<bool>& destroy);
284 
285  // Schedules a path for garbage collection based on its modification time.
286  // Equivalent to the `Slave::garbageCollect` method.
287  process::Future<Nothing> garbageCollect(const std::string& path);
288 
289  // Call back for when an isolator limits a container and impacts the
290  // processes. This will trigger container destruction.
291  void limited(
292  const ContainerID& containerId,
294 
295  // Helper for reaping the 'init' process of a container.
297  const ContainerID& containerId,
298  pid_t pid);
299 
300  // Call back for when the executor exits. This will trigger container
301  // destroy.
302  void reaped(const ContainerID& containerId);
303 
304  // TODO(jieyu): Consider introducing an Isolators struct and moving
305  // all isolator related operations to that struct.
307  const ContainerID& containerId);
308 
309  const Flags flags;
310  Fetcher* fetcher;
311 
312  // NOTE: This actor may be nullptr in tests, as not all tests need to
313  // share this actor with the agent.
314  GarbageCollector* gc;
315 
316  IOSwitchboard* ioSwitchboard;
317  const process::Owned<Launcher> launcher;
318  const process::Shared<Provisioner> provisioner;
319  const std::vector<process::Owned<mesos::slave::Isolator>> isolators;
320 
321  struct Container
322  {
323  Container() : sequence("mesos-container-status-updates") {}
324 
325  // Promise for futures returned from wait().
327 
328  // NOTE: this represents 'PID 1', i.e., the "init" of the
329  // container that we created (it may be for an executor, or any
330  // arbitrary process that has been launched in the event of nested
331  // containers).
332  Option<pid_t> pid;
333 
334  // Sandbox directory for the container. It is optional here because
335  // we don't keep track of sandbox directory for orphan containers.
336  // It is not checkpointed explicitly; on recovery, it is reconstructed
337  // from executor's directory and hierarchy of containers.
338  //
339  // NOTE: This holds the sandbox path in the host mount namespace,
340  // while MESOS_SANDBOX is the path in the container mount namespace.
341  Option<std::string> directory;
342 
343  // We keep track of the future exit status for the container if it
344  // has been launched. If the container has not been launched yet,
345  // 'status' will be set to None().
346  //
347  // NOTE: A container has an exit status does not mean that it has
348  // been properly destroyed. We need to perform cleanup on
349  // isolators and provisioner after that.
351 
352  // We keep track of the future for 'provisioner->provision' so
353  // that destroy will only start calling 'provisioner->destroy'
354  // after 'provisioner->provision' has finished.
355  process::Future<ProvisionInfo> provisioning;
356 
357  // We keep track of the future that is waiting for all the
358  // 'isolator->prepare' to finish so that destroy will only start
359  // calling cleanup after all isolators have finished preparing.
361  launchInfos;
362 
363  // We keep track of the future that is waiting for all the
364  // 'isolator->isolate' futures so that destroy will only start
365  // calling cleanup after all isolators have finished isolating.
367 
368  // We keep track of the resources for each container so we can set
369  // the ResourceStatistics limits in usage().
371 
372  // The configuration for the container to be launched.
373  // This can only be None if the underlying container is launched
374  // before we checkpoint `ContainerConfig` in MESOS-6894.
375  // TODO(zhitao): Drop the `Option` part at the end of deprecation
376  // cycle.
378 
379  // The container class that can be `DEFAULT` or `DEBUG`.
380  // Returns `DEFAULT` even if the container class is not defined.
381  mesos::slave::ContainerClass containerClass();
382 
383  // Container's information at the moment it was launched. For example,
384  // used to bootstrap the launch information of future child DEBUG
385  // containers. Checkpointed and restored on recovery. Optional because
386  // it is not set for orphan containers.
387  //
388  // NOTE: Some of these data, may change during the container lifetime,
389  // e.g., the working directory. Such changes are not be captured here,
390  // which might be problematic, e.g., for DEBUG containers relying on
391  // some data in parent working directory.
393 
394  State state;
395 
396  // Used when `status` needs to be collected from isolators
397  // associated with this container. `Sequence` allows us to
398  // maintain the order of `status` requests for a given container.
399  process::Sequence sequence;
400 
401  // Child containers nested under this container.
403  };
404 
406 
407  // Helper to transition container state.
408  void transition(const ContainerID& containerId, const State& state);
409 
410  // Helper to determine if a container is supported by an isolator.
411  bool isSupportedByIsolator(
412  const ContainerID& containerId,
413  bool isolatorSupportsNesting,
414  bool isolatorSupportsStandalone);
415 
416  struct Metrics
417  {
418  Metrics();
419  ~Metrics();
420 
421  process::metrics::Counter container_destroy_errors;
422  } metrics;
423 };
424 
425 
426 std::ostream& operator<<(
427  std::ostream& stream,
428  const MesosContainerizerProcess::State& state);
429 
430 } // namespace slave {
431 } // namespace internal {
432 } // namespace mesos {
433 
434 #endif // __MESOS_CONTAINERIZER_HPP__
process::Future< hashset< ContainerID > > containers() override
Definition: path.hpp:26
Try< Nothing > isolate(const std::string &hierarchy, const std::string &cgroup, pid_t pid)
std::string generate(const std::string &prefix="")
Returns &#39;prefix(N)&#39; where N represents the number of instances where the same prefix (wrt...
process::Future< Containerizer::LaunchResult > launch(const ContainerID &containerId, const mesos::slave::ContainerConfig &containerConfig, const std::map< std::string, std::string > &environment, const Option< std::string > &pidCheckpointPath) override
~MesosContainerizerProcess() override
Definition: containerizer.hpp:155
Definition: option.hpp:28
std::ostream & operator<<(std::ostream &stream, const MesosContainerizerProcess::State &state)
Definition: fetcher.hpp:49
process::Future< Nothing > update(const ContainerID &containerId, const Resources &resources) override
Definition: check.hpp:33
process::Future< Nothing > pruneImages(const std::vector< Image > &excludedImages) override
process::Future< bool > kill(const ContainerID &containerId, int signal) override
Definition: resources.hpp:81
Try< T > fetch(const std::string &value)
Definition: fetch.hpp:38
Future< Option< int > > reap(pid_t pid)
Definition: flags.hpp:39
process::Future< Option< mesos::slave::ContainerTermination > > wait(const ContainerID &containerId) override
Definition: counter.hpp:26
Definition: sequence.hpp:33
Definition: hashmap.hpp:38
Environment * environment
DWORD pid_t
Definition: windows.hpp:181
Definition: containerizer.hpp:59
Definition: owned.hpp:26
process::Future< Nothing > recover(const Option< state::SlaveState > &state) override
process::Future< ResourceStatistics > usage(const ContainerID &containerId) override
Definition: spec.hpp:26
Definition: switchboard.hpp:53
Definition: containerizer.hpp:134
process::Future< process::http::Connection > attach(const ContainerID &containerId) override
#define flags
Definition: decoder.hpp:18
Definition: none.hpp:27
Definition: attributes.hpp:24
std::set< pid_t > children(pid_t, const std::list< Process > &, bool)
Definition: os.hpp:215
Definition: executor.hpp:48
static Try< Resources > resources(const Flags &flags)
Definition: containerizer.hpp:65
Try< std::string > prepare(const std::string &baseHierarchy, const std::string &subsystem, const std::string &cgroup)
JSON::Object Metrics()
Definition: resolver.hpp:34
int int_fd
Definition: int_fd.hpp:35
Definition: owned.hpp:36
Definition: process.hpp:501
static Try< MesosContainerizer * > create(const Flags &flags, bool local, Fetcher *fetcher, GarbageCollector *gc=nullptr, SecretResolver *secretResolver=nullptr, const Option< NvidiaComponents > &nvidia=None())
Definition: parse.hpp:33
PID< MetricsProcess > metrics
MesosContainerizerProcess(const Flags &_flags, Fetcher *_fetcher, GarbageCollector *_gc, IOSwitchboard *_ioSwitchboard, const process::Owned< Launcher > &_launcher, const process::Shared< Provisioner > &_provisioner, const std::vector< process::Owned< mesos::slave::Isolator >> &_isolators)
Definition: containerizer.hpp:138
process::Future< Option< mesos::slave::ContainerTermination > > destroy(const ContainerID &containerId) override
process::Future< ContainerStatus > status(const ContainerID &containerId) override