Apache Mesos
containerizer.hpp
Go to the documentation of this file.
1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 
17 #ifndef __MESOS_CONTAINERIZER_HPP__
18 #define __MESOS_CONTAINERIZER_HPP__
19 
20 #include <vector>
21 
23 
24 #include <mesos/slave/isolator.hpp>
25 
26 #include <process/id.hpp>
27 #include <process/http.hpp>
28 #include <process/sequence.hpp>
29 #include <process/shared.hpp>
30 
32 
33 #include <stout/hashmap.hpp>
34 #include <stout/multihashmap.hpp>
35 #include <stout/os/int_fd.hpp>
36 
37 #include "slave/state.hpp"
38 
40 
42 
44 
46 
48 
49 namespace mesos {
50 namespace internal {
51 namespace slave {
52 
53 // If the container class is not of type `DEBUG` (i.e., it is not set or
54 // `DEFAULT`), we log the line at the INFO level. Otherwise, we use VLOG(1).
55 // The purpose of this macro is to avoid polluting agent logs with information
56 // related to `DEBUG` containers as this type of container can run periodically.
57 #define LOG_BASED_ON_CLASS(containerClass) \
58  LOG_IF(INFO, (containerClass != ContainerClass::DEBUG) || VLOG_IS_ON(1))
59 
60 // Forward declaration.
61 class MesosContainerizerProcess;
62 
63 
65 {
66 public:
68  const Flags& flags,
69  bool local,
70  Fetcher* fetcher,
71  SecretResolver* secretResolver = nullptr,
72  const Option<NvidiaComponents>& nvidia = None());
73 
75  const Flags& flags,
76  bool local,
77  Fetcher* fetcher,
78  const process::Owned<Launcher>& launcher,
79  const process::Shared<Provisioner>& provisioner,
80  const std::vector<process::Owned<mesos::slave::Isolator>>& isolators);
81 
82  ~MesosContainerizer() override;
83 
85  const Option<state::SlaveState>& state) override;
86 
88  const ContainerID& containerId,
89  const mesos::slave::ContainerConfig& containerConfig,
90  const std::map<std::string, std::string>& environment,
91  const Option<std::string>& pidCheckpointPath) override;
92 
94  const ContainerID& containerId) override;
95 
97  const ContainerID& containerId,
98  const Resources& resources) override;
99 
101  const ContainerID& containerId) override;
102 
104  const ContainerID& containerId) override;
105 
107  const ContainerID& containerId) override;
108 
110  const ContainerID& containerId) override;
111 
113  const ContainerID& containerId,
114  int signal) override;
115 
117 
118  process::Future<Nothing> remove(const ContainerID& containerId) override;
119 
121  const std::vector<Image>& excludedImages) override;
122 
123 private:
124  explicit MesosContainerizer(
126 
128 };
129 
130 
132  : public process::Process<MesosContainerizerProcess>
133 {
134 public:
136  const Flags& _flags,
137  Fetcher* _fetcher,
138  IOSwitchboard* _ioSwitchboard,
139  const process::Owned<Launcher>& _launcher,
140  const process::Shared<Provisioner>& _provisioner,
141  const std::vector<process::Owned<mesos::slave::Isolator>>& _isolators)
142  : ProcessBase(process::ID::generate("mesos-containerizer")),
143  flags(_flags),
144  fetcher(_fetcher),
145  ioSwitchboard(_ioSwitchboard),
146  launcher(_launcher),
147  provisioner(_provisioner),
148  isolators(_isolators) {}
149 
151 
153  const Option<state::SlaveState>& state);
154 
156  const ContainerID& containerId,
157  const mesos::slave::ContainerConfig& containerConfig,
158  const std::map<std::string, std::string>& environment,
159  const Option<std::string>& pidCheckpointPath);
160 
162  const ContainerID& containerId);
163 
165  const ContainerID& containerId,
166  const Resources& resources);
167 
169  const ContainerID& containerId);
170 
172  const ContainerID& containerId);
173 
175  const ContainerID& containerId);
176 
178  const ContainerID& containerId,
179  int_fd pipeWrite);
180 
182  const ContainerID& containerId,
183  const Option<mesos::slave::ContainerTermination>& termination);
184 
185  virtual process::Future<bool> kill(
186  const ContainerID& containerId,
187  int signal);
188 
189  virtual process::Future<Nothing> remove(const ContainerID& containerId);
190 
192 
194  const std::vector<Image>& excludedImages);
195 
196 private:
197  enum State
198  {
199  PROVISIONING,
200  PREPARING,
201  ISOLATING,
202  FETCHING,
203  RUNNING,
204  DESTROYING
205  };
206 
207  friend std::ostream& operator<<(std::ostream& stream, const State& state);
208 
209  process::Future<Nothing> _recover(
210  const std::vector<mesos::slave::ContainerState>& recoverable,
211  const hashset<ContainerID>& orphans);
212 
213  process::Future<std::vector<Nothing>> recoverIsolators(
214  const std::vector<mesos::slave::ContainerState>& recoverable,
215  const hashset<ContainerID>& orphans);
216 
217  process::Future<Nothing> recoverProvisioner(
218  const std::vector<mesos::slave::ContainerState>& recoverable,
219  const hashset<ContainerID>& orphans);
220 
221  process::Future<Nothing> __recover(
222  const std::vector<mesos::slave::ContainerState>& recovered,
223  const hashset<ContainerID>& orphans);
224 
226  const ContainerID& containerId,
227  const Option<ProvisionInfo>& provisionInfo);
228 
230  const ContainerID& containerId);
231 
233  const ContainerID& containerId,
234  const Option<mesos::slave::ContainerIO>& containerIO,
235  const std::map<std::string, std::string>& environment,
236  const Option<std::string>& pidCheckpointPath);
237 
239  const ContainerID& containerId,
240  pid_t _pid);
241 
242  // Continues 'destroy()' once nested containers are handled.
243  void _destroy(
244  const ContainerID& containerId,
246  const State& previousState,
247  const std::vector<
249 
250  // Continues '_destroy()' once isolators has completed.
251  void __destroy(
252  const ContainerID& containerId,
253  const Option<mesos::slave::ContainerTermination>& termination);
254 
255  // Continues '__destroy()' once all processes have been killed
256  // by the launcher.
257  void ___destroy(
258  const ContainerID& containerId,
260  const process::Future<Nothing>& future);
261 
262  // Continues '___destroy()' once we get the exit status of the container.
263  void ____destroy(
264  const ContainerID& containerId,
265  const Option<mesos::slave::ContainerTermination>& termination);
266 
267  // Continues '____destroy()' once all isolators have completed
268  // cleanup.
269  void _____destroy(
270  const ContainerID& containerId,
272  const process::Future<std::vector<process::Future<Nothing>>>& cleanups);
273 
274  // Continues '_____destroy()' once provisioner have completed destroy.
275  void ______destroy(
276  const ContainerID& containerId,
278  const process::Future<bool>& destroy);
279 
280  // Call back for when an isolator limits a container and impacts the
281  // processes. This will trigger container destruction.
282  void limited(
283  const ContainerID& containerId,
285 
286  // Helper for reaping the 'init' process of a container.
288  const ContainerID& containerId,
289  pid_t pid);
290 
291  // Call back for when the executor exits. This will trigger container
292  // destroy.
293  void reaped(const ContainerID& containerId);
294 
295  // TODO(jieyu): Consider introducing an Isolators struct and moving
296  // all isolator related operations to that struct.
298  const ContainerID& containerId);
299 
300  const Flags flags;
301  Fetcher* fetcher;
302  IOSwitchboard* ioSwitchboard;
303  const process::Owned<Launcher> launcher;
304  const process::Shared<Provisioner> provisioner;
305  const std::vector<process::Owned<mesos::slave::Isolator>> isolators;
306 
307  struct Container
308  {
309  Container() : sequence("mesos-container-status-updates") {}
310 
311  // Promise for futures returned from wait().
313 
314  // NOTE: this represents 'PID 1', i.e., the "init" of the
315  // container that we created (it may be for an executor, or any
316  // arbitrary process that has been launched in the event of nested
317  // containers).
318  Option<pid_t> pid;
319 
320  // Sandbox directory for the container. It is optional here because
321  // we don't keep track of sandbox directory for orphan containers.
322  // It is not checkpointed explicitly; on recovery, it is reconstructed
323  // from executor's directory and hierarchy of containers.
324  //
325  // NOTE: This holds the sandbox path in the host mount namespace,
326  // while MESOS_SANDBOX is the path in the container mount namespace.
327  Option<std::string> directory;
328 
329  // We keep track of the future exit status for the container if it
330  // has been launched. If the container has not been launched yet,
331  // 'status' will be set to None().
332  //
333  // NOTE: A container has an exit status does not mean that it has
334  // been properly destroyed. We need to perform cleanup on
335  // isolators and provisioner after that.
337 
338  // We keep track of the future for 'provisioner->provision' so
339  // that destroy will only start calling 'provisioner->destroy'
340  // after 'provisioner->provision' has finished.
341  process::Future<ProvisionInfo> provisioning;
342 
343  // We keep track of the future that is waiting for all the
344  // 'isolator->prepare' to finish so that destroy will only start
345  // calling cleanup after all isolators have finished preparing.
347  launchInfos;
348 
349  // We keep track of the future that is waiting for all the
350  // 'isolator->isolate' futures so that destroy will only start
351  // calling cleanup after all isolators have finished isolating.
353 
354  // We keep track of the resources for each container so we can set
355  // the ResourceStatistics limits in usage().
357 
358  // The configuration for the container to be launched.
359  // This can only be None if the underlying container is launched
360  // before we checkpoint `ContainerConfig` in MESOS-6894.
361  // TODO(zhitao): Drop the `Option` part at the end of deprecation
362  // cycle.
364 
365  // The container class that can be `DEFAULT` or `DEBUG`.
366  // Returns `DEFAULT` even if the container class is not defined.
367  mesos::slave::ContainerClass containerClass();
368 
369  // Container's information at the moment it was launched. For example,
370  // used to bootstrap the launch information of future child DEBUG
371  // containers. Checkpointed and restored on recovery. Optional because
372  // it is not set for orphan containers.
373  //
374  // NOTE: Some of these data, may change during the container lifetime,
375  // e.g., the working directory. Such changes are not be captured here,
376  // which might be problematic, e.g., for DEBUG containers relying on
377  // some data in parent working directory.
379 
380  State state;
381 
382  // Used when `status` needs to be collected from isolators
383  // associated with this container. `Sequence` allows us to
384  // maintain the order of `status` requests for a given container.
385  process::Sequence sequence;
386 
387  // Child containers nested under this container.
389  };
390 
392 
393  // Helper to transition container state.
394  void transition(const ContainerID& containerId, const State& state);
395 
396  // Helper to determine if a container is supported by an isolator.
397  bool isSupportedByIsolator(
398  const ContainerID& containerId,
399  bool isolatorSupportsNesting,
400  bool isolatorSupportsStandalone);
401 
402  struct Metrics
403  {
404  Metrics();
405  ~Metrics();
406 
407  process::metrics::Counter container_destroy_errors;
408  } metrics;
409 };
410 
411 
412 std::ostream& operator<<(
413  std::ostream& stream,
414  const MesosContainerizerProcess::State& state);
415 
416 } // namespace slave {
417 } // namespace internal {
418 } // namespace mesos {
419 
420 #endif // __MESOS_CONTAINERIZER_HPP__
process::Future< hashset< ContainerID > > containers() override
Try< Nothing > isolate(const std::string &hierarchy, const std::string &cgroup, pid_t pid)
std::string generate(const std::string &prefix="")
Returns &#39;prefix(N)&#39; where N represents the number of instances where the same prefix (wrt...
process::Future< Containerizer::LaunchResult > launch(const ContainerID &containerId, const mesos::slave::ContainerConfig &containerConfig, const std::map< std::string, std::string > &environment, const Option< std::string > &pidCheckpointPath) override
~MesosContainerizerProcess() override
Definition: containerizer.hpp:150
Definition: option.hpp:28
MesosContainerizerProcess(const Flags &_flags, Fetcher *_fetcher, IOSwitchboard *_ioSwitchboard, const process::Owned< Launcher > &_launcher, const process::Shared< Provisioner > &_provisioner, const std::vector< process::Owned< mesos::slave::Isolator >> &_isolators)
Definition: containerizer.hpp:135
std::ostream & operator<<(std::ostream &stream, const MesosContainerizerProcess::State &state)
Definition: fetcher.hpp:49
process::Future< Nothing > update(const ContainerID &containerId, const Resources &resources) override
Definition: check.hpp:33
process::Future< Nothing > pruneImages(const std::vector< Image > &excludedImages) override
process::Future< bool > kill(const ContainerID &containerId, int signal) override
Definition: resources.hpp:79
Try< T > fetch(const std::string &value)
Definition: fetch.hpp:38
Future< Option< int > > reap(pid_t pid)
Definition: flags.hpp:39
process::Future< Option< mesos::slave::ContainerTermination > > wait(const ContainerID &containerId) override
Definition: counter.hpp:26
Definition: sequence.hpp:33
Definition: hashmap.hpp:38
Environment * environment
DWORD pid_t
Definition: windows.hpp:181
Definition: containerizer.hpp:57
Definition: owned.hpp:26
process::Future< Nothing > recover(const Option< state::SlaveState > &state) override
process::Future< ResourceStatistics > usage(const ContainerID &containerId) override
static Try< MesosContainerizer * > create(const Flags &flags, bool local, Fetcher *fetcher, SecretResolver *secretResolver=nullptr, const Option< NvidiaComponents > &nvidia=None())
Definition: spec.hpp:30
Definition: switchboard.hpp:53
Definition: containerizer.hpp:131
process::Future< process::http::Connection > attach(const ContainerID &containerId) override
#define flags
Definition: decoder.hpp:18
Definition: none.hpp:27
Definition: attributes.hpp:24
std::set< pid_t > children(pid_t, const std::list< Process > &, bool)
Definition: os.hpp:215
Definition: executor.hpp:48
static Try< Resources > resources(const Flags &flags)
Definition: containerizer.hpp:64
Try< std::string > prepare(const std::string &baseHierarchy, const std::string &subsystem, const std::string &cgroup)
JSON::Object Metrics()
Definition: resolver.hpp:34
int int_fd
Definition: int_fd.hpp:35
Definition: owned.hpp:36
Definition: process.hpp:501
Definition: parse.hpp:33
PID< MetricsProcess > metrics
process::Future< Option< mesos::slave::ContainerTermination > > destroy(const ContainerID &containerId) override
process::Future< ContainerStatus > status(const ContainerID &containerId) override