Apache Mesos
shell.hpp
Go to the documentation of this file.
1 // Licensed under the Apache License, Version 2.0 (the "License");
2 // you may not use this file except in compliance with the License.
3 // You may obtain a copy of the License at
4 //
5 // http://www.apache.org/licenses/LICENSE-2.0
6 //
7 // Unless required by applicable law or agreed to in writing, software
8 // distributed under the License is distributed on an "AS IS" BASIS,
9 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 // See the License for the specific language governing permissions and
11 // limitations under the License.
12 
13 #ifndef __STOUT_OS_WINDOWS_SHELL_HPP__
14 #define __STOUT_OS_WINDOWS_SHELL_HPP__
15 
16 #include <process.h>
17 #include <stdarg.h> // For va_list, va_start, etc.
18 
19 #include <algorithm>
20 #include <array>
21 #include <map>
22 #include <string>
23 #include <vector>
24 
25 #include <stout/error.hpp>
26 #include <stout/foreach.hpp>
27 #include <stout/none.hpp>
28 #include <stout/option.hpp>
29 #include <stout/os.hpp>
30 #include <stout/try.hpp>
31 #include <stout/windows.hpp>
32 
33 #include <stout/os/windows/fd.hpp>
34 
36 
37 namespace internal {
38 namespace windows {
39 
40 // Retrieves system environment in a `std::map`, ignoring
41 // the current process's environment variables.
43 {
44  std::map<std::wstring, std::wstring> system_env;
45  wchar_t* env_entry = nullptr;
46 
47  // Get the system environment.
48  // The third parameter (bool) tells the function *not* to inherit
49  // variables from the current process.
50  if (!::CreateEnvironmentBlock((LPVOID*)&env_entry, nullptr, FALSE)) {
51  return None();
52  }
53 
54  // Save the environment block in order to destroy it later.
55  wchar_t* env_block = env_entry;
56 
57  while (*env_entry != L'\0') {
58  // Each environment block contains the environment variables as follows:
59  // Var1=Value1\0
60  // Var2=Value2\0
61  // Var3=Value3\0
62  // ...
63  // VarN=ValueN\0\0
64  // The name of an environment variable cannot include an equal sign (=).
65 
66  // Construct a string from the pointer up to the first '\0',
67  // e.g. "Var1=Value1\0", then split into name and value.
68  std::wstring entry(env_entry);
69  std::wstring::size_type separator = entry.find(L"=");
70  std::wstring var_name(entry.substr(0, separator));
71  std::wstring varVal(entry.substr(separator + 1));
72 
73  // Mesos variables are upper case. Convert system variables to
74  // match the name provided by the scheduler in case of a collision.
75  // This is safe because Windows environment variables are case insensitive.
77  var_name.begin(), var_name.end(), var_name.begin(), ::towupper);
78 
79  // The system environment has priority.
80  system_env.insert_or_assign(var_name.data(), varVal.data());
81 
82  // Advance the pointer the length of the entry string plus the '\0'.
83  env_entry += entry.length() + 1;
84  }
85 
86  ::DestroyEnvironmentBlock(env_block);
87 
88  return system_env;
89 }
90 
91 
92 // Creates a null-terminated array of null-terminated strings that will be
93 // passed to `CreateProcessW` as the `lpEnvironment` argument, as described by
94 // MSDN[1]. This array needs to be sorted in alphabetical order, but the `map`
95 // already takes care of that. Note that this function explicitly handles
96 // UTF-16 environments, so it must be used in conjunction with the
97 // `CREATE_UNICODE_ENVIRONMENT` flag.
98 //
99 // NOTE: This function will add the system's environment variables into
100 // the returned string. These variables take precedence over the provided
101 // `env` and are generally necessary in order to launch things on Windows.
102 //
103 // [1] https://msdn.microsoft.com/en-us/library/windows/desktop/ms682425(v=vs.85).aspx
105  const Option<std::map<std::string, std::string>>& env)
106 {
107  if (env.isNone() || (env.isSome() && env.get().size() == 0)) {
108  return None();
109  }
110 
112 
113  // The system environment must be non-empty.
114  // No subprocesses will be able to launch if the system environment is blank.
115  CHECK(system_env.isSome() && system_env.get().size() > 0);
116 
117  std::map<std::wstring, std::wstring> combined_env;
118 
119  // Populate the combined environment first with the system environment.
120  foreachpair (const std::wstring& key,
121  const std::wstring& value,
122  system_env.get()) {
123  combined_env[key] = value;
124  }
125 
126  // Now override with the supplied environment.
127  foreachpair (const std::string& key,
128  const std::string& value,
129  env.get()) {
130  combined_env[wide_stringify(key)] = wide_stringify(value);
131  }
132 
133  std::wstring env_string;
134  foreachpair (const std::wstring& key,
135  const std::wstring& value,
136  combined_env) {
137  env_string += key + L'=' + value + L'\0';
138  }
139 
140  // Append final null terminating character.
141  env_string.push_back(L'\0');
142  return env_string;
143 }
144 
145 
146 // Concatenates multiple command-line arguments and escapes the values.
147 // NOTE: This is necessary even when using Windows APIs that "appear"
148 // to take arguments as a list, because those APIs will themselves
149 // concatenate command-line arguments *without* escaping them.
150 //
151 // This function escapes arguments with the following rules:
152 // 1) Any argument with a space, tab, newline, vertical tab,
153 // or double-quote must be surrounded in double-quotes.
154 // 2) Backslashes at the very end of an argument must be escaped.
155 // 3) Backslashes that precede a double-quote must be escaped.
156 // The double-quote must also be escaped.
157 //
158 // NOTE: The below algorithm is adapted from Daniel Colascione's public domain
159 // algorithm for quoting command line arguments on Windows for `CreateProcess`.
160 //
161 // https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/
162 // NOLINT(whitespace/line_length)
163 inline std::wstring stringify_args(const std::vector<std::string>& argv)
164 {
165  std::wstring command;
166  for (auto argit = argv.cbegin(); argit != argv.cend(); ++argit) {
167  std::wstring arg = wide_stringify(*argit);
168  // Don't quote empty arguments or those without troublesome characters.
169  if (!arg.empty() && arg.find_first_of(L" \t\n\v\"") == arg.npos) {
170  command.append(arg);
171  } else {
172  // Beginning double quotation mark.
173  command.push_back(L'"');
174  for (auto it = arg.cbegin(); it != arg.cend(); ++it) {
175  // Count existent backslashes in argument.
176  unsigned int backslashes = 0;
177  while (it != arg.cend() && *it == L'\\') {
178  ++it;
179  ++backslashes;
180  }
181 
182  if (it == arg.cend()) {
183  // Escape all backslashes, but let the terminating double quotation
184  // mark we add below be interpreted as a metacharacter.
185  command.append(backslashes * 2, L'\\');
186  break;
187  } else if (*it == L'"') {
188  // Escape all backslashes and the following double quotation mark.
189  command.append(backslashes * 2 + 1, L'\\');
190  command.push_back(*it);
191  } else {
192  // Backslashes aren't special here.
193  command.append(backslashes, L'\\');
194  command.push_back(*it);
195  }
196  }
197 
198  // Terminating double quotation mark.
199  command.push_back(L'"');
200  }
201  // Space separate arguments (but don't append at end).
202  if (argit != argv.cend() - 1) {
203  command.push_back(L' ');
204  }
205  }
206  // Append final null terminating character.
207  command.push_back(L'\0');
208  return command;
209 }
210 
211 
212 struct ProcessData {
216 };
217 
218 
219 // Provides an interface for creating a child process on Windows.
220 //
221 // The `command` argument is given for compatibility, and is ignored. This is
222 // because the `CreateProcess` will use `argv[0]` as the command to be executed,
223 // and will perform a `PATH` lookup. If `command` were to be used instead,
224 // `CreateProcess` would require an absolute path.
225 //
226 // If `create_suspended` is `true`, the process will not be started, and the
227 // caller must use `ResumeThread` to start the process.
228 //
229 // The caller can specify explicit `stdin`, `stdout`, and `stderr` handles,
230 // in that order, for the process via the `pipes` argument.
231 //
232 // NOTE: If `pipes` are specified, they will be temporarily set to
233 // inheritable, and then set to uninheritable. This is a side effect
234 // on each `HANDLE`.
235 //
236 // The return value is a `ProcessData` struct, with the process and thread
237 // handles each saved in a `SharedHandle`, ensuring they are closed when struct
238 // goes out of scope.
240  const std::string& command,
241  const std::vector<std::string>& argv,
242  const Option<std::map<std::string, std::string>>& environment,
243  const bool create_suspended = false,
244  const Option<std::array<os::WindowsFD, 3>> pipes = None())
245 {
246  // TODO(andschwa): Assert that `command` and `argv[0]` are the same.
247  const std::wstring arg_string = stringify_args(argv);
248  std::vector<wchar_t> arg_buffer(arg_string.begin(), arg_string.end());
249  arg_buffer.push_back(L'\0');
250 
251  // Create the process with a Unicode environment.
252  DWORD creation_flags = CREATE_UNICODE_ENVIRONMENT;
253  if (create_suspended) {
254  creation_flags |= CREATE_SUSPENDED;
255  }
256 
257  // Construct the environment that will be passed to `::CreateProcessW`.
259  std::vector<wchar_t> env_buffer;
260  if (env_string.isSome()) {
261  // This string contains the necessary null characters.
262  env_buffer.assign(env_string.get().begin(), env_string.get().end());
263  }
264 
265  wchar_t* process_env = env_buffer.empty() ? nullptr : env_buffer.data();
266 
267  PROCESS_INFORMATION process_info = {};
268 
269  STARTUPINFOW startup_info = {};
270  startup_info.cb = sizeof(STARTUPINFOW);
271 
272  // Hook up the stdin/out/err pipes and use the `STARTF_USESTDHANDLES`
273  // flag to instruct the child to use them [1].
274  // A more user-friendly example can be found in [2].
275  //
276  // [1] https://msdn.microsoft.com/en-us/library/windows/desktop/ms686331(v=vs.85).aspx
277  // [2] https://msdn.microsoft.com/en-us/library/windows/desktop/ms682499(v=vs.85).aspx
278  if (pipes.isSome()) {
279  // Each of these handles must be inheritable.
280  foreach (const os::WindowsFD& fd, pipes.get()) {
281  const Try<Nothing> inherit = set_inherit(fd, true);
282  if (inherit.isError()) {
283  return Error(inherit.error());
284  }
285  }
286 
287  startup_info.dwFlags |= STARTF_USESTDHANDLES;
288  startup_info.hStdInput = std::get<0>(pipes.get());
289  startup_info.hStdOutput = std::get<1>(pipes.get());
290  startup_info.hStdError = std::get<2>(pipes.get());
291  }
292 
293  const BOOL result = ::CreateProcessW(
294  // This is replaced by the first token of `arg_buffer` string.
295  static_cast<LPCWSTR>(nullptr),
296  static_cast<LPWSTR>(arg_buffer.data()),
297  static_cast<LPSECURITY_ATTRIBUTES>(nullptr),
298  static_cast<LPSECURITY_ATTRIBUTES>(nullptr),
299  TRUE, // Inherit parent process handles (such as those in `pipes`).
300  creation_flags,
301  static_cast<LPVOID>(process_env),
302  static_cast<LPCWSTR>(nullptr), // Inherit working directory.
303  &startup_info,
304  &process_info);
305 
306  // NOTE: The MSDN documentation for `CreateProcess` states that it
307  // returns before the process has "finished initialization," but is
308  // not clear on precisely what initialization entails. It would seem
309  // that this does not affect inherited handles, as it stands to
310  // reason that the system call to `CreateProcess` causes inheritable
311  // handles to become inherited, and not some "initialization" of the
312  // child process. However, if an inheritance race condition
313  // manifests, this assumption should be re-evaluated.
314 
315  if (pipes.isSome()) {
316  // These handles should no longer be inheritable. This prevents other child
317  // processes from accidentally inheriting the wrong handles.
318  //
319  // NOTE: This is explicit, and does not take into account the
320  // previous inheritance semantics of each `HANDLE`. It is assumed
321  // that users of this function send non-inheritable handles.
322  foreach (const os::WindowsFD& fd, pipes.get()) {
323  const Try<Nothing> inherit = set_inherit(fd, false);
324  if (inherit.isError()) {
325  return Error(inherit.error());
326  }
327  }
328  }
329 
330  if (result == FALSE) {
331  return WindowsError(
332  "Failed to call `CreateProcess`: " + stringify(arg_string));
333  }
334 
335  return ProcessData{SharedHandle{process_info.hProcess, ::CloseHandle},
336  SharedHandle{process_info.hThread, ::CloseHandle},
337  static_cast<pid_t>(process_info.dwProcessId)};
338 }
339 
340 } // namespace windows {
341 } // namespace internal {
342 
343 namespace os {
344 namespace Shell {
345 
346 // Canonical constants used as platform-dependent args to `exec` calls.
347 // `name` is the command name, `arg0` is the first argument received
348 // by the callee, usually the command name and `arg1` is the second
349 // command argument received by the callee.
350 constexpr const char* name = "cmd.exe";
351 constexpr const char* arg0 = "cmd.exe";
352 constexpr const char* arg1 = "/c";
353 
354 } // namespace Shell {
355 
356 template <typename... T>
357 Try<std::string> shell(const std::string& fmt, const T&... t) = delete;
358 
359 
360 template<typename... T>
361 inline int execlp(const char* file, T... t) = delete;
362 
363 
364 // Executes a command by calling "<command> <arguments...>", and
365 // returns after the command has been completed. Returns the process exit
366 // code on success and `None` on error.
368  const std::string& command,
369  const std::vector<std::string>& arguments,
370  const Option<std::map<std::string, std::string>>& environment = None())
371 {
374 
375  if (process_data.isError()) {
376  LOG(WARNING) << process_data.error();
377  return None();
378  }
379 
380  // Wait for the process synchronously.
381  ::WaitForSingleObject(
382  process_data->process_handle.get_handle(), INFINITE);
383 
384  DWORD status;
385  if (!::GetExitCodeProcess(
386  process_data->process_handle.get_handle(),
387  &status)) {
388  LOG(WARNING) << "Failed to `GetExitCodeProcess`: " << command;
389  return None();
390  }
391 
392  // Return the child exit code.
393  return static_cast<int>(status);
394 }
395 
396 
397 // Executes a command by calling "cmd /c <command>", and returns
398 // after the command has been completed. Returns the process exit
399 // code on success and `None` on error.
400 //
401 // Note: Be cautious about shell injection
402 // (https://en.wikipedia.org/wiki/Code_injection#Shell_injection)
403 // when using this method and use proper validation and sanitization
404 // on the `command`. For this reason in general `os::spawn` is
405 // preferred if a shell is not required.
406 inline Option<int> system(const std::string& command)
407 {
408  return os::spawn(Shell::name, {Shell::arg0, Shell::arg1, command});
409 }
410 
411 
412 // In order to emulate the semantics of `execvp`, `os::spawn` waits for the new
413 // process to exit, and returns its error code, which is propogated back to the
414 // parent via `exit` here.
415 inline int execvp(
416  const std::string& command,
417  const std::vector<std::string>& argv)
418 {
419  exit(os::spawn(command, argv).getOrElse(-1));
420  return -1;
421 }
422 
423 
424 // NOTE: This function can accept `Argv` and `Envp` constructs through their
425 // explicit type conversions, but unlike the POSIX implementations, it cannot
426 // accept the raw forms.
427 inline int execvpe(
428  const std::string& command,
429  const std::vector<std::string>& argv,
430  const std::map<std::string, std::string>& envp)
431 {
432  exit(os::spawn(command, argv, envp).getOrElse(-1));
433  return -1;
434 }
435 
436 } // namespace os {
437 
438 #endif // __STOUT_OS_WINDOWS_SHELL_HPP__
std::wstring stringify_args(const std::vector< std::string > &argv)
Definition: shell.hpp:163
constexpr const char * arg1
Definition: shell.hpp:45
Definition: errorbase.hpp:35
Definition: option.hpp:28
HANDLE get_handle() const
Definition: windows.hpp:96
Definition: windows.hpp:78
Definition: fd.hpp:47
Definition: check.hpp:33
int execlp(const char *file, T...t)
Definition: shell.hpp:189
Try< ProcessData > create_process(const std::string &command, const std::vector< std::string > &argv, const Option< std::map< std::string, std::string >> &environment, const bool create_suspended=false, const Option< std::array< os::WindowsFD, 3 >> pipes=None())
Definition: shell.hpp:239
Definition: shell.hpp:212
Result< ProcessStatus > status(pid_t pid)
Definition: proc.hpp:166
Definition: error.hpp:106
Option< int > spawn(const std::string &command, const std::vector< std::string > &arguments)
Definition: shell.hpp:162
Definition: posix_signalhandler.hpp:23
bool isSome() const
Definition: option.hpp:115
constexpr const char * arg0
Definition: shell.hpp:44
Environment * environment
DWORD pid_t
Definition: windows.hpp:187
URI file(const std::string &path)
Creates a file URI with the given path on the local host.
Definition: file.hpp:33
SharedHandle process_handle
Definition: shell.hpp:213
SharedHandle thread_handle
Definition: shell.hpp:214
const T & get() const &
Definition: option.hpp:118
#define foreachpair(KEY, VALUE, ELEMS)
Definition: foreach.hpp:51
static Try error(const E &e)
Definition: try.hpp:42
process::Future< Nothing > transform(process::Owned< Reader< T >> &&reader, const std::function< std::string(const T &)> &func, process::http::Pipe::Writer writer)
This is a helper function that reads records from a Reader, applies a transformation to the records a...
Definition: recordio.hpp:112
Try< std::string > shell(const std::string &fmt, const T &...t)
Runs a shell command with optional arguments.
Definition: shell.hpp:73
Option< std::map< std::wstring, std::wstring > > get_system_env()
Definition: shell.hpp:42
Definition: none.hpp:27
Definition: attributes.hpp:24
bool isError() const
Definition: try.hpp:71
int execvpe(const std::string &command, const std::vector< std::string > &argv, const std::map< std::string, std::string > &envp)
Definition: shell.hpp:427
int execvp(const std::string &command, const std::vector< std::string > &argv)
Definition: shell.hpp:415
Option< std::wstring > create_process_env(const Option< std::map< std::string, std::string >> &env)
Definition: shell.hpp:104
pid_t pid
Definition: shell.hpp:215
std::string stringify(int flags)
Option< int > spawn(const std::string &command, const std::vector< std::string > &arguments, const Option< std::map< std::string, std::string >> &environment=None())
Definition: shell.hpp:367
Try< Nothing > set_inherit(const os::WindowsFD &fd, const bool inherit)
Definition: inherit.hpp:32
constexpr const char * name
Definition: shell.hpp:43
Option< int > system(const std::string &command)
Definition: shell.hpp:133