Apache Mesos
containerizer.hpp
Go to the documentation of this file.
1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 
17 #ifndef __MESOS_CONTAINERIZER_HPP__
18 #define __MESOS_CONTAINERIZER_HPP__
19 
20 #include <list>
21 #include <vector>
22 
24 
25 #include <mesos/slave/isolator.hpp>
26 
27 #include <process/id.hpp>
28 #include <process/http.hpp>
29 #include <process/sequence.hpp>
30 #include <process/shared.hpp>
31 
33 
34 #include <stout/hashmap.hpp>
35 #include <stout/multihashmap.hpp>
36 #include <stout/os/int_fd.hpp>
37 
38 #include "slave/state.hpp"
39 
41 
43 
45 
47 
49 
50 namespace mesos {
51 namespace internal {
52 namespace slave {
53 
54 // If the container class is not of type `DEBUG` (i.e., it is not set or
55 // `DEFAULT`), we log the line at the INFO level. Otherwise, we use VLOG(1).
56 // The purpose of this macro is to avoid polluting agent logs with information
57 // related to `DEBUG` containers as this type of container can run periodically.
58 #define LOG_BASED_ON_CLASS(containerClass) \
59  LOG_IF(INFO, (containerClass != ContainerClass::DEBUG) || VLOG_IS_ON(1))
60 
61 // Forward declaration.
62 class MesosContainerizerProcess;
63 
64 
66 {
67 public:
69  const Flags& flags,
70  bool local,
71  Fetcher* fetcher,
72  SecretResolver* secretResolver = nullptr,
73  const Option<NvidiaComponents>& nvidia = None());
74 
76  const Flags& flags,
77  bool local,
78  Fetcher* fetcher,
79  const process::Owned<Launcher>& launcher,
80  const process::Shared<Provisioner>& provisioner,
81  const std::vector<process::Owned<mesos::slave::Isolator>>& isolators);
82 
83  virtual ~MesosContainerizer();
84 
86  const Option<state::SlaveState>& state);
87 
89  const ContainerID& containerId,
90  const mesos::slave::ContainerConfig& containerConfig,
91  const std::map<std::string, std::string>& environment,
92  const Option<std::string>& pidCheckpointPath);
93 
95  const ContainerID& containerId);
96 
98  const ContainerID& containerId,
99  const Resources& resources);
100 
102  const ContainerID& containerId);
103 
105  const ContainerID& containerId);
106 
108  const ContainerID& containerId);
109 
111  const ContainerID& containerId);
112 
113  virtual process::Future<bool> kill(
114  const ContainerID& containerId,
115  int signal);
116 
118 
119  virtual process::Future<Nothing> remove(const ContainerID& containerId);
120 
122  const std::vector<Image>& excludedImages);
123 
124 private:
125  explicit MesosContainerizer(
127 
129 };
130 
131 
133  : public process::Process<MesosContainerizerProcess>
134 {
135 public:
137  const Flags& _flags,
138  Fetcher* _fetcher,
139  IOSwitchboard* _ioSwitchboard,
140  const process::Owned<Launcher>& _launcher,
141  const process::Shared<Provisioner>& _provisioner,
142  const std::vector<process::Owned<mesos::slave::Isolator>>& _isolators)
143  : ProcessBase(process::ID::generate("mesos-containerizer")),
144  flags(_flags),
145  fetcher(_fetcher),
146  ioSwitchboard(_ioSwitchboard),
147  launcher(_launcher),
148  provisioner(_provisioner),
149  isolators(_isolators) {}
150 
152 
154  const Option<state::SlaveState>& state);
155 
157  const ContainerID& containerId,
158  const mesos::slave::ContainerConfig& containerConfig,
159  const std::map<std::string, std::string>& environment,
160  const Option<std::string>& pidCheckpointPath);
161 
163  const ContainerID& containerId);
164 
166  const ContainerID& containerId,
167  const Resources& resources);
168 
170  const ContainerID& containerId);
171 
173  const ContainerID& containerId);
174 
176  const ContainerID& containerId);
177 
179  const ContainerID& containerId,
180  int_fd pipeWrite);
181 
183  const ContainerID& containerId,
184  const Option<mesos::slave::ContainerTermination>& termination);
185 
186  virtual process::Future<bool> kill(
187  const ContainerID& containerId,
188  int signal);
189 
190  virtual process::Future<Nothing> remove(const ContainerID& containerId);
191 
193 
195  const std::vector<Image>& excludedImages);
196 
197 private:
198  enum State
199  {
200  PROVISIONING,
201  PREPARING,
202  ISOLATING,
203  FETCHING,
204  RUNNING,
205  DESTROYING
206  };
207 
208  friend std::ostream& operator<<(std::ostream& stream, const State& state);
209 
210  process::Future<Nothing> _recover(
211  const std::list<mesos::slave::ContainerState>& recoverable,
212  const hashset<ContainerID>& orphans);
213 
214  process::Future<std::list<Nothing>> recoverIsolators(
215  const std::list<mesos::slave::ContainerState>& recoverable,
216  const hashset<ContainerID>& orphans);
217 
218  process::Future<Nothing> recoverProvisioner(
219  const std::list<mesos::slave::ContainerState>& recoverable,
220  const hashset<ContainerID>& orphans);
221 
222  process::Future<Nothing> __recover(
223  const std::list<mesos::slave::ContainerState>& recovered,
224  const hashset<ContainerID>& orphans);
225 
227  const ContainerID& containerId,
228  const Option<ProvisionInfo>& provisionInfo);
229 
231  const ContainerID& containerId);
232 
234  const ContainerID& containerId,
235  const Option<mesos::slave::ContainerIO>& containerIO,
236  const std::map<std::string, std::string>& environment,
237  const Option<std::string>& pidCheckpointPath);
238 
240  const ContainerID& containerId,
241  pid_t _pid);
242 
243  // Continues 'destroy()' once nested containers are handled.
244  void _destroy(
245  const ContainerID& containerId,
247  const State& previousState,
248  const std::list<
250 
251  // Continues '_destroy()' once isolators has completed.
252  void __destroy(
253  const ContainerID& containerId,
254  const Option<mesos::slave::ContainerTermination>& termination);
255 
256  // Continues '__destroy()' once all processes have been killed
257  // by the launcher.
258  void ___destroy(
259  const ContainerID& containerId,
261  const process::Future<Nothing>& future);
262 
263  // Continues '___destroy()' once we get the exit status of the container.
264  void ____destroy(
265  const ContainerID& containerId,
266  const Option<mesos::slave::ContainerTermination>& termination);
267 
268  // Continues '____destroy()' once all isolators have completed
269  // cleanup.
270  void _____destroy(
271  const ContainerID& containerId,
274 
275  // Continues '_____destroy()' once provisioner have completed destroy.
276  void ______destroy(
277  const ContainerID& containerId,
279  const process::Future<bool>& destroy);
280 
281  // Call back for when an isolator limits a container and impacts the
282  // processes. This will trigger container destruction.
283  void limited(
284  const ContainerID& containerId,
286 
287  // Helper for reaping the 'init' process of a container.
289  const ContainerID& containerId,
290  pid_t pid);
291 
292  // Call back for when the executor exits. This will trigger container
293  // destroy.
294  void reaped(const ContainerID& containerId);
295 
296  // TODO(jieyu): Consider introducing an Isolators struct and moving
297  // all isolator related operations to that struct.
299  const ContainerID& containerId);
300 
301  const Flags flags;
302  Fetcher* fetcher;
303  IOSwitchboard* ioSwitchboard;
304  const process::Owned<Launcher> launcher;
305  const process::Shared<Provisioner> provisioner;
306  const std::vector<process::Owned<mesos::slave::Isolator>> isolators;
307 
308  struct Container
309  {
310  Container() : sequence("mesos-container-status-updates") {}
311 
312  // Promise for futures returned from wait().
314 
315  // NOTE: this represents 'PID 1', i.e., the "init" of the
316  // container that we created (it may be for an executor, or any
317  // arbitrary process that has been launched in the event of nested
318  // containers).
319  Option<pid_t> pid;
320 
321  // Sandbox directory for the container. It is optional here because
322  // we don't keep track of sandbox directory for orphan containers.
323  // It is not checkpointed explicitly; on recovery, it is reconstructed
324  // from executor's directory and hierarchy of containers.
325  //
326  // NOTE: This holds the sandbox path in the host mount namespace,
327  // while MESOS_SANDBOX is the path in the container mount namespace.
328  Option<std::string> directory;
329 
330  // We keep track of the future exit status for the container if it
331  // has been launched. If the container has not been launched yet,
332  // 'status' will be set to None().
333  //
334  // NOTE: A container has an exit status does not mean that it has
335  // been properly destroyed. We need to perform cleanup on
336  // isolators and provisioner after that.
338 
339  // We keep track of the future for 'provisioner->provision' so
340  // that destroy will only start calling 'provisioner->destroy'
341  // after 'provisioner->provision' has finished.
342  process::Future<ProvisionInfo> provisioning;
343 
344  // We keep track of the future that is waiting for all the
345  // 'isolator->prepare' to finish so that destroy will only start
346  // calling cleanup after all isolators have finished preparing.
348  launchInfos;
349 
350  // We keep track of the future that is waiting for all the
351  // 'isolator->isolate' futures so that destroy will only start
352  // calling cleanup after all isolators have finished isolating.
354 
355  // We keep track of the resources for each container so we can set
356  // the ResourceStatistics limits in usage().
358 
359  // The configuration for the container to be launched.
360  // This can only be None if the underlying container is launched
361  // before we checkpoint `ContainerConfig` in MESOS-6894.
362  // TODO(zhitao): Drop the `Option` part at the end of deprecation
363  // cycle.
365 
366  // The container class that can be `DEFAULT` or `DEBUG`.
367  // Returns `DEFAULT` even if the container class is not defined.
368  mesos::slave::ContainerClass containerClass();
369 
370  // Container's information at the moment it was launched. For example,
371  // used to bootstrap the launch information of future child DEBUG
372  // containers. Checkpointed and restored on recovery. Optional because
373  // it is not set for orphan containers.
374  //
375  // NOTE: Some of these data, may change during the container lifetime,
376  // e.g., the working directory. Such changes are not be captured here,
377  // which might be problematic, e.g., for DEBUG containers relying on
378  // some data in parent working directory.
380 
381  State state;
382 
383  // Used when `status` needs to be collected from isolators
384  // associated with this container. `Sequence` allows us to
385  // maintain the order of `status` requests for a given container.
386  process::Sequence sequence;
387 
388  // Child containers nested under this container.
390  };
391 
393 
394  // Helper to transition container state.
395  void transition(const ContainerID& containerId, const State& state);
396 
397  // Helper to determine if a container is supported by an isolator.
398  bool isSupportedByIsolator(
399  const ContainerID& containerId,
400  bool isolatorSupportsNesting,
401  bool isolatorSupportsStandalone);
402 
403  struct Metrics
404  {
405  Metrics();
406  ~Metrics();
407 
408  process::metrics::Counter container_destroy_errors;
409  } metrics;
410 };
411 
412 
413 std::ostream& operator<<(
414  std::ostream& stream,
415  const MesosContainerizerProcess::State& state);
416 
417 } // namespace slave {
418 } // namespace internal {
419 } // namespace mesos {
420 
421 #endif // __MESOS_CONTAINERIZER_HPP__
virtual process::Future< Containerizer::LaunchResult > launch(const ContainerID &containerId, const mesos::slave::ContainerConfig &containerConfig, const std::map< std::string, std::string > &environment, const Option< std::string > &pidCheckpointPath)
Try< Nothing > isolate(const std::string &hierarchy, const std::string &cgroup, pid_t pid)
std::string generate(const std::string &prefix="")
Returns &#39;prefix(N)&#39; where N represents the number of instances where the same prefix (wrt...
Definition: option.hpp:28
MesosContainerizerProcess(const Flags &_flags, Fetcher *_fetcher, IOSwitchboard *_ioSwitchboard, const process::Owned< Launcher > &_launcher, const process::Shared< Provisioner > &_provisioner, const std::vector< process::Owned< mesos::slave::Isolator >> &_isolators)
Definition: containerizer.hpp:136
virtual process::Future< bool > kill(const ContainerID &containerId, int signal)
virtual ~MesosContainerizerProcess()
Definition: containerizer.hpp:151
std::ostream & operator<<(std::ostream &stream, const MesosContainerizerProcess::State &state)
Definition: fetcher.hpp:49
Definition: check.hpp:33
virtual process::Future< Nothing > pruneImages(const std::vector< Image > &excludedImages)
Definition: resources.hpp:79
Try< T > fetch(const std::string &value)
Definition: fetch.hpp:38
Future< Option< int > > reap(pid_t pid)
Definition: flags.hpp:39
Definition: counter.hpp:26
Definition: sequence.hpp:33
virtual process::Future< hashset< ContainerID > > containers()
Definition: hashmap.hpp:38
Environment * environment
DWORD pid_t
Definition: windows.hpp:187
Definition: containerizer.hpp:57
Definition: owned.hpp:26
virtual process::Future< Option< mesos::slave::ContainerTermination > > wait(const ContainerID &containerId)
virtual process::Future< Nothing > update(const ContainerID &containerId, const Resources &resources)
virtual process::Future< ResourceStatistics > usage(const ContainerID &containerId)
static Try< MesosContainerizer * > create(const Flags &flags, bool local, Fetcher *fetcher, SecretResolver *secretResolver=nullptr, const Option< NvidiaComponents > &nvidia=None())
Definition: spec.hpp:30
Definition: switchboard.hpp:53
virtual process::Future< Option< mesos::slave::ContainerTermination > > destroy(const ContainerID &containerId)
Try< std::vector< Entry > > list(const std::string &hierarchy, const std::string &cgroup)
Definition: containerizer.hpp:132
#define flags
Definition: decoder.hpp:18
Definition: none.hpp:27
Definition: attributes.hpp:24
std::set< pid_t > children(pid_t, const std::list< Process > &, bool)
Definition: os.hpp:215
Definition: executor.hpp:47
static Try< Resources > resources(const Flags &flags)
virtual process::Future< Nothing > recover(const Option< state::SlaveState > &state)
Definition: containerizer.hpp:65
Try< std::string > prepare(const std::string &baseHierarchy, const std::string &subsystem, const std::string &cgroup)
JSON::Object Metrics()
Definition: resolver.hpp:34
virtual process::Future< process::http::Connection > attach(const ContainerID &containerId)
int int_fd
Definition: int_fd.hpp:35
Definition: owned.hpp:36
Definition: process.hpp:501
Definition: parse.hpp:33
PID< MetricsProcess > metrics
virtual process::Future< ContainerStatus > status(const ContainerID &containerId)