Apache Mesos
containerizer.hpp
Go to the documentation of this file.
1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 
17 #ifndef __MESOS_CONTAINERIZER_HPP__
18 #define __MESOS_CONTAINERIZER_HPP__
19 
20 #include <list>
21 #include <vector>
22 
24 
25 #include <mesos/slave/isolator.hpp>
26 
27 #include <process/id.hpp>
28 #include <process/http.hpp>
29 #include <process/sequence.hpp>
30 #include <process/shared.hpp>
31 
33 
34 #include <stout/hashmap.hpp>
35 #include <stout/multihashmap.hpp>
36 #include <stout/os/int_fd.hpp>
37 
38 #include "slave/state.hpp"
39 
41 
43 
45 
47 
49 
50 namespace mesos {
51 namespace internal {
52 namespace slave {
53 
54 // If the container class is not of type `DEBUG` (i.e., it is not set or
55 // `DEFAULT`), we log the line at the INFO level. Otherwise, we use VLOG(1).
56 // The purpose of this macro is to avoid polluting agent logs with information
57 // related to `DEBUG` containers as this type of container can run periodically.
58 #define LOG_BASED_ON_CLASS(containerClass) \
59  LOG_IF(INFO, (containerClass != ContainerClass::DEBUG) || VLOG_IS_ON(1))
60 
61 // Forward declaration.
62 class MesosContainerizerProcess;
63 
64 
66 {
67 public:
69  const Flags& flags,
70  bool local,
71  Fetcher* fetcher,
72  SecretResolver* secretResolver = nullptr,
73  const Option<NvidiaComponents>& nvidia = None());
74 
76  const Flags& flags,
77  bool local,
78  Fetcher* fetcher,
79  const process::Owned<Launcher>& launcher,
80  const process::Shared<Provisioner>& provisioner,
81  const std::vector<process::Owned<mesos::slave::Isolator>>& isolators);
82 
83  virtual ~MesosContainerizer();
84 
86  const Option<state::SlaveState>& state);
87 
89  const ContainerID& containerId,
90  const mesos::slave::ContainerConfig& containerConfig,
91  const std::map<std::string, std::string>& environment,
92  const Option<std::string>& pidCheckpointPath);
93 
95  const ContainerID& containerId);
96 
98  const ContainerID& containerId,
99  const Resources& resources);
100 
102  const ContainerID& containerId);
103 
105  const ContainerID& containerId);
106 
108  const ContainerID& containerId);
109 
111  const ContainerID& containerId);
112 
113  virtual process::Future<bool> kill(
114  const ContainerID& containerId,
115  int signal);
116 
118 
119  virtual process::Future<Nothing> remove(const ContainerID& containerId);
120 
122  const std::vector<Image>& excludedImages);
123 
124 private:
125  explicit MesosContainerizer(
127 
129 };
130 
131 
133  : public process::Process<MesosContainerizerProcess>
134 {
135 public:
137  const Flags& _flags,
138  Fetcher* _fetcher,
139  IOSwitchboard* _ioSwitchboard,
140  const process::Owned<Launcher>& _launcher,
141  const process::Shared<Provisioner>& _provisioner,
142  const std::vector<process::Owned<mesos::slave::Isolator>>& _isolators)
143  : ProcessBase(process::ID::generate("mesos-containerizer")),
144  flags(_flags),
145  fetcher(_fetcher),
146  ioSwitchboard(_ioSwitchboard),
147  launcher(_launcher),
148  provisioner(_provisioner),
149  isolators(_isolators) {}
150 
152 
154  const Option<state::SlaveState>& state);
155 
157  const ContainerID& containerId,
158  const mesos::slave::ContainerConfig& containerConfig,
159  const std::map<std::string, std::string>& environment,
160  const Option<std::string>& pidCheckpointPath);
161 
163  const ContainerID& containerId);
164 
166  const ContainerID& containerId,
167  const Resources& resources);
168 
170  const ContainerID& containerId);
171 
173  const ContainerID& containerId);
174 
176  const ContainerID& containerId);
177 
179  const ContainerID& containerId,
180  int_fd pipeWrite);
181 
183  const ContainerID& containerId,
184  const Option<mesos::slave::ContainerTermination>& termination);
185 
186  virtual process::Future<bool> kill(
187  const ContainerID& containerId,
188  int signal);
189 
190  virtual process::Future<Nothing> remove(const ContainerID& containerId);
191 
193 
195  const std::vector<Image>& excludedImages);
196 
197 private:
198  enum State
199  {
200  PROVISIONING,
201  PREPARING,
202  ISOLATING,
203  FETCHING,
204  RUNNING,
205  DESTROYING
206  };
207 
208  friend std::ostream& operator<<(std::ostream& stream, const State& state);
209 
210  process::Future<Nothing> _recover(
211  const std::list<mesos::slave::ContainerState>& recoverable,
212  const hashset<ContainerID>& orphans);
213 
214  process::Future<std::list<Nothing>> recoverIsolators(
215  const std::list<mesos::slave::ContainerState>& recoverable,
216  const hashset<ContainerID>& orphans);
217 
218  process::Future<Nothing> recoverProvisioner(
219  const std::list<mesos::slave::ContainerState>& recoverable,
220  const hashset<ContainerID>& orphans);
221 
222  process::Future<Nothing> __recover(
223  const std::list<mesos::slave::ContainerState>& recovered,
224  const hashset<ContainerID>& orphans);
225 
226  process::Future<Nothing> prepare(
227  const ContainerID& containerId,
228  const Option<ProvisionInfo>& provisionInfo);
229 
231  const ContainerID& containerId);
232 
234  const ContainerID& containerId,
235  const Option<mesos::slave::ContainerIO>& containerIO,
236  const std::map<std::string, std::string>& environment,
237  const Option<std::string>& pidCheckpointPath);
238 
239  process::Future<Nothing> isolate(
240  const ContainerID& containerId,
241  pid_t _pid);
242 
243  // Continues 'destroy()' once nested containers are handled.
244  void _destroy(
245  const ContainerID& containerId,
247  const State& previousState,
248  const std::list<process::Future<bool>>& destroys);
249 
250  // Continues '_destroy()' once isolators has completed.
251  void __destroy(
252  const ContainerID& containerId,
253  const Option<mesos::slave::ContainerTermination>& termination);
254 
255  // Continues '__destroy()' once all processes have been killed
256  // by the launcher.
257  void ___destroy(
258  const ContainerID& containerId,
260  const process::Future<Nothing>& future);
261 
262  // Continues '___destroy()' once we get the exit status of the container.
263  void ____destroy(
264  const ContainerID& containerId,
265  const Option<mesos::slave::ContainerTermination>& termination);
266 
267  // Continues '____destroy()' once all isolators have completed
268  // cleanup.
269  void _____destroy(
270  const ContainerID& containerId,
273 
274  // Continues '_____destroy()' once provisioner have completed destroy.
275  void ______destroy(
276  const ContainerID& containerId,
279 
280  // Call back for when an isolator limits a container and impacts the
281  // processes. This will trigger container destruction.
282  void limited(
283  const ContainerID& containerId,
285 
286  // Helper for reaping the 'init' process of a container.
288  const ContainerID& containerId,
289  pid_t pid);
290 
291  // Call back for when the executor exits. This will trigger container
292  // destroy.
293  void reaped(const ContainerID& containerId);
294 
295  // TODO(jieyu): Consider introducing an Isolators struct and moving
296  // all isolator related operations to that struct.
298  const ContainerID& containerId);
299 
300  const Flags flags;
301  Fetcher* fetcher;
302  IOSwitchboard* ioSwitchboard;
303  const process::Owned<Launcher> launcher;
304  const process::Shared<Provisioner> provisioner;
305  const std::vector<process::Owned<mesos::slave::Isolator>> isolators;
306 
307  struct Container
308  {
309  Container() : sequence("mesos-container-status-updates") {}
310 
311  // Promise for futures returned from wait().
313 
314  // NOTE: this represents 'PID 1', i.e., the "init" of the
315  // container that we created (it may be for an executor, or any
316  // arbitrary process that has been launched in the event of nested
317  // containers).
318  Option<pid_t> pid;
319 
320  // Sandbox directory for the container. It is optional here because
321  // we don't keep track of sandbox directory for orphan containers.
322  // It is not checkpointed explicitly; on recovery, it is reconstructed
323  // from executor's directory and hierarchy of containers.
324  //
325  // NOTE: This holds the sandbox path in the host mount namespace,
326  // while MESOS_SANDBOX is the path in the container mount namespace.
327  Option<std::string> directory;
328 
329  // We keep track of the future exit status for the container if it
330  // has been launched. If the container has not been launched yet,
331  // 'status' will be set to None().
332  //
333  // NOTE: A container has an exit status does not mean that it has
334  // been properly destroyed. We need to perform cleanup on
335  // isolators and provisioner after that.
337 
338  // We keep track of the future for 'provisioner->provision' so
339  // that destroy will only start calling 'provisioner->destroy'
340  // after 'provisioner->provision' has finished.
341  process::Future<ProvisionInfo> provisioning;
342 
343  // We keep track of the future that is waiting for all the
344  // 'isolator->prepare' to finish so that destroy will only start
345  // calling cleanup after all isolators have finished preparing.
347  launchInfos;
348 
349  // We keep track of the future that is waiting for all the
350  // 'isolator->isolate' futures so that destroy will only start
351  // calling cleanup after all isolators have finished isolating.
353 
354  // We keep track of the resources for each container so we can set
355  // the ResourceStatistics limits in usage().
356  Resources resources;
357 
358  // The configuration for the container to be launched.
359  // This can only be None if the underlying container is launched
360  // before we checkpoint `ContainerConfig` in MESOS-6894.
361  // TODO(zhitao): Drop the `Option` part at the end of deprecation
362  // cycle.
364 
365  // The container class that can be `DEFAULT` or `DEBUG`.
366  // Returns `DEFAULT` even if the container class is not defined.
367  mesos::slave::ContainerClass containerClass();
368 
369  // Container's information at the moment it was launched. For example,
370  // used to bootstrap the launch information of future child DEBUG
371  // containers. Checkpointed and restored on recovery. Optional because
372  // it is not set for orphan containers.
373  //
374  // NOTE: Some of these data, may change during the container lifetime,
375  // e.g., the working directory. Such changes are not be captured here,
376  // which might be problematic, e.g., for DEBUG containers relying on
377  // some data in parent working directory.
379 
380  State state;
381 
382  // Used when `status` needs to be collected from isolators
383  // associated with this container. `Sequence` allows us to
384  // maintain the order of `status` requests for a given container.
385  process::Sequence sequence;
386 
387  // Child containers nested under this container.
389  };
390 
392 
393  // Helper to transition container state.
394  void transition(const ContainerID& containerId, const State& state);
395 
396  // Helper to determine if a container is supported by an isolator.
397  bool isSupportedByIsolator(
398  const ContainerID& containerId,
399  bool isolatorSupportsNesting,
400  bool isolatorSupportsStandalone);
401 
402  struct Metrics
403  {
404  Metrics();
405  ~Metrics();
406 
407  process::metrics::Counter container_destroy_errors;
408  } metrics;
409 };
410 
411 
412 std::ostream& operator<<(
413  std::ostream& stream,
414  const MesosContainerizerProcess::State& state);
415 
416 } // namespace slave {
417 } // namespace internal {
418 } // namespace mesos {
419 
420 #endif // __MESOS_CONTAINERIZER_HPP__
virtual process::Future< Containerizer::LaunchResult > launch(const ContainerID &containerId, const mesos::slave::ContainerConfig &containerConfig, const std::map< std::string, std::string > &environment, const Option< std::string > &pidCheckpointPath)
std::string generate(const std::string &prefix="")
Returns &#39;prefix(N)&#39; where N represents the number of instances where the same prefix (wrt...
Definition: option.hpp:28
virtual process::Future< process::http::Connection > attach(const ContainerID &containerId)
MesosContainerizerProcess(const Flags &_flags, Fetcher *_fetcher, IOSwitchboard *_ioSwitchboard, const process::Owned< Launcher > &_launcher, const process::Shared< Provisioner > &_provisioner, const std::vector< process::Owned< mesos::slave::Isolator >> &_isolators)
Definition: containerizer.hpp:136
virtual process::Future< bool > kill(const ContainerID &containerId, int signal)
virtual process::Future< ResourceStatistics > usage(const ContainerID &containerId)
ProcessBase(const std::string &id="")
virtual ~MesosContainerizerProcess()
Definition: containerizer.hpp:151
std::ostream & operator<<(std::ostream &stream, const MesosContainerizerProcess::State &state)
Definition: fetcher.hpp:49
Definition: try.hpp:34
virtual process::Future< Nothing > pruneImages(const std::vector< Image > &excludedImages)
virtual process::Future< Option< mesos::slave::ContainerTermination > > wait(const ContainerID &containerId)
virtual process::Future< hashset< ContainerID > > containers()
Definition: resources.hpp:79
virtual process::Future< Containerizer::LaunchResult > exec(const ContainerID &containerId, int_fd pipeWrite)
Definition: flags.hpp:39
Definition: counter.hpp:26
Definition: sequence.hpp:33
virtual process::Future< hashset< ContainerID > > containers()
Definition: hashmap.hpp:38
Environment * environment
DWORD pid_t
Definition: windows.hpp:187
virtual process::Future< Nothing > pruneImages(const std::vector< Image > &excludedImages)
Definition: containerizer.hpp:57
Definition: owned.hpp:25
virtual process::Future< Option< mesos::slave::ContainerTermination > > wait(const ContainerID &containerId)
virtual process::Future< Nothing > update(const ContainerID &containerId, const Resources &resources)
virtual process::Future< Containerizer::LaunchResult > launch(const ContainerID &containerId, const mesos::slave::ContainerConfig &containerConfig, const std::map< std::string, std::string > &environment, const Option< std::string > &pidCheckpointPath)
virtual process::Future< ResourceStatistics > usage(const ContainerID &containerId)
virtual process::Future< Nothing > recover(const Option< state::SlaveState > &state)
static Try< MesosContainerizer * > create(const Flags &flags, bool local, Fetcher *fetcher, SecretResolver *secretResolver=nullptr, const Option< NvidiaComponents > &nvidia=None())
Definition: switchboard.hpp:53
virtual process::Future< Nothing > update(const ContainerID &containerId, const Resources &resources)
virtual process::Future< bool > destroy(const ContainerID &containerId)
Try< std::vector< Entry > > list(const std::string &hierarchy, const std::string &cgroup)
virtual process::Future< bool > kill(const ContainerID &containerId, int signal)
Definition: containerizer.hpp:132
Result< Process > process(pid_t pid)
Definition: freebsd.hpp:30
#define flags
Definition: decoder.hpp:18
virtual process::Future< ContainerStatus > status(const ContainerID &containerId)
Definition: none.hpp:27
std::set< pid_t > children(pid_t, const std::list< Process > &, bool)
Definition: os.hpp:215
static Try< Resources > resources(const Flags &flags)
virtual process::Future< Nothing > recover(const Option< state::SlaveState > &state)
Definition: containerizer.hpp:65
JSON::Object Metrics()
Definition: resolver.hpp:34
virtual process::Future< process::http::Connection > attach(const ContainerID &containerId)
int int_fd
Definition: int_fd.hpp:35
Definition: owned.hpp:35
Definition: process.hpp:493
virtual process::Future< ContainerStatus > status(const ContainerID &containerId)
friend std::ostream & operator<<(std::ostream &stream, const State &state)
virtual process::Future< bool > destroy(const ContainerID &containerId, const Option< mesos::slave::ContainerTermination > &termination)