Apache Mesos
shell.hpp
Go to the documentation of this file.
1 // Licensed under the Apache License, Version 2.0 (the "License");
2 // you may not use this file except in compliance with the License.
3 // You may obtain a copy of the License at
4 //
5 // http://www.apache.org/licenses/LICENSE-2.0
6 //
7 // Unless required by applicable law or agreed to in writing, software
8 // distributed under the License is distributed on an "AS IS" BASIS,
9 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 // See the License for the specific language governing permissions and
11 // limitations under the License.
12 
13 #ifndef __STOUT_OS_WINDOWS_SHELL_HPP__
14 #define __STOUT_OS_WINDOWS_SHELL_HPP__
15 
16 #include <process.h>
17 #include <stdarg.h> // For va_list, va_start, etc.
18 
19 #include <algorithm>
20 #include <map>
21 #include <ostream>
22 #include <string>
23 #include <tuple>
24 #include <vector>
25 
26 #include <stout/error.hpp>
27 #include <stout/foreach.hpp>
28 #include <stout/option.hpp>
29 #include <stout/os.hpp>
30 #include <stout/os/int_fd.hpp>
31 #include <stout/try.hpp>
32 #include <stout/unimplemented.hpp>
33 
34 #include <stout/windows.hpp>
35 
36 namespace internal {
37 
38 namespace windows {
39 
40 // Retrieves system environment in a `std::map`, ignoring
41 // the current process's environment variables.
43 {
44  std::map<std::wstring, std::wstring> system_env;
45  wchar_t* env_entry = nullptr;
46 
47  // Get the system environment.
48  // The third parameter (bool) tells the function *not* to inherit
49  // variables from the current process.
50  if (!::CreateEnvironmentBlock((LPVOID*)&env_entry, nullptr, FALSE)) {
51  return None();
52  }
53 
54  // Save the environment block in order to destroy it later.
55  wchar_t* env_block = env_entry;
56 
57  while (*env_entry != L'\0') {
58  // Each environment block contains the environment variables as follows:
59  // Var1=Value1\0
60  // Var2=Value2\0
61  // Var3=Value3\0
62  // ...
63  // VarN=ValueN\0\0
64  // The name of an environment variable cannot include an equal sign (=).
65 
66  // Construct a string from the pointer up to the first '\0',
67  // e.g. "Var1=Value1\0", then split into name and value.
68  std::wstring entry(env_entry);
69  std::wstring::size_type separator = entry.find(L"=");
70  std::wstring var_name(entry.substr(0, separator));
71  std::wstring varVal(entry.substr(separator + 1));
72 
73  // Mesos variables are upper case. Convert system variables to
74  // match the name provided by the scheduler in case of a collision.
75  // This is safe because Windows environment variables are case insensitive.
77  var_name.begin(), var_name.end(), var_name.begin(), ::towupper);
78 
79  // The system environment has priority.
80  system_env.insert_or_assign(var_name.data(), varVal.data());
81 
82  // Advance the pointer the length of the entry string plus the '\0'.
83  env_entry += entry.length() + 1;
84  }
85 
86  ::DestroyEnvironmentBlock(env_block);
87 
88  return system_env;
89 }
90 
91 
92 // Creates a null-terminated array of null-terminated strings that will be
93 // passed to `CreateProcessW` as the `lpEnvironment` argument, as described by
94 // MSDN[1]. This array needs to be sorted in alphabetical order, but the `map`
95 // already takes care of that. Note that this function explicitly handles
96 // UTF-16 environments, so it must be used in conjunction with the
97 // `CREATE_UNICODE_ENVIRONMENT` flag.
98 //
99 // NOTE: This function will add the system's environment variables into
100 // the returned string. These variables take precedence over the provided
101 // `env` and are generally necessary in order to launch things on Windows.
102 //
103 // [1] https://msdn.microsoft.com/en-us/library/windows/desktop/ms682425(v=vs.85).aspx
105  const Option<std::map<std::string, std::string>>& env)
106 {
107  if (env.isNone() || (env.isSome() && env.get().size() == 0)) {
108  return None();
109  }
110 
112 
113  // The system environment must be non-empty.
114  // No subprocesses will be able to launch if the system environment is blank.
115  CHECK(system_env.isSome() && system_env.get().size() > 0);
116 
117  std::map<std::wstring, std::wstring> combined_env;
118 
119  // Populate the combined environment first with the system environment.
120  foreachpair (const std::wstring& key,
121  const std::wstring& value,
122  system_env.get()) {
123  combined_env[key] = value;
124  }
125 
126  // Now override with the supplied environment.
127  foreachpair (const std::string& key,
128  const std::string& value,
129  env.get()) {
130  combined_env[wide_stringify(key)] = wide_stringify(value);
131  }
132 
133  std::wstring env_string;
134  foreachpair (const std::wstring& key,
135  const std::wstring& value,
136  combined_env) {
137  env_string += key + L'=' + value + L'\0';
138  }
139 
140  // Append final null terminating character.
141  env_string.push_back(L'\0');
142  return env_string;
143 }
144 
145 
146 // Concatenates multiple command-line arguments and escapes the values.
147 // NOTE: This is necessary even when using Windows APIs that "appear"
148 // to take arguments as a list, because those APIs will themselves
149 // concatenate command-line arguments *without* escaping them.
150 //
151 // This function escapes arguments with the following rules:
152 // 1) Any argument with a space, tab, newline, vertical tab,
153 // or double-quote must be surrounded in double-quotes.
154 // 2) Backslashes at the very end of an argument must be escaped.
155 // 3) Backslashes that precede a double-quote must be escaped.
156 // The double-quote must also be escaped.
157 //
158 // NOTE: The below algorithm is adapted from Daniel Colascione's public domain
159 // algorithm for quoting command line arguments on Windows for `CreateProcess`.
160 //
161 // https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/
162 // NOLINT(whitespace/line_length)
163 inline std::wstring stringify_args(const std::vector<std::string>& argv)
164 {
165  std::wstring command;
166  for (auto argit = argv.cbegin(); argit != argv.cend(); ++argit) {
167  std::wstring arg = wide_stringify(*argit);
168  // Don't quote empty arguments or those without troublesome characters.
169  if (!arg.empty() && arg.find_first_of(L" \t\n\v\"") == arg.npos) {
170  command.append(arg);
171  } else {
172  // Beginning double quotation mark.
173  command.push_back(L'"');
174  for (auto it = arg.cbegin(); it != arg.cend(); ++it) {
175  // Count existent backslashes in argument.
176  unsigned int backslashes = 0;
177  while (it != arg.cend() && *it == L'\\') {
178  ++it;
179  ++backslashes;
180  }
181 
182  if (it == arg.cend()) {
183  // Escape all backslashes, but let the terminating double quotation
184  // mark we add below be interpreted as a metacharacter.
185  command.append(backslashes * 2, L'\\');
186  break;
187  } else if (*it == L'"') {
188  // Escape all backslashes and the following double quotation mark.
189  command.append(backslashes * 2 + 1, L'\\');
190  command.push_back(*it);
191  } else {
192  // Backslashes aren't special here.
193  command.append(backslashes, L'\\');
194  command.push_back(*it);
195  }
196  }
197 
198  // Terminating double quotation mark.
199  command.push_back(L'"');
200  }
201  // Space separate arguments (but don't append at end).
202  if (argit != argv.cend() - 1) {
203  command.push_back(L' ');
204  }
205  }
206  // Append final null terminating character.
207  command.push_back(L'\0');
208  return command;
209 }
210 
211 struct ProcessData {
215 };
216 
217 // Provides an interface for creating a child process on Windows.
218 //
219 // The `command` argument is given for compatibility, and is ignored. This is
220 // because the `CreateProcess` will use `argv[0]` as the command to be executed,
221 // and will perform a `PATH` lookup. If `command` were to be used instead,
222 // `CreateProcess` would require an absolute path.
223 //
224 // If `create_suspended` is `true`, the process will not be started, and the
225 // caller must use `ResumeThread` to start the process.
226 //
227 // The caller can specify explicit `stdin`, `stdout`, and `stderr` handles,
228 // in that order, for the process via the `pipes` argument.
229 //
230 // The return value is a `ProcessData` struct, with the process and thread
231 // handles each saved in a `SharedHandle`, ensuring they are closed when struct
232 // goes out of scope.
234  const std::string& command,
235  const std::vector<std::string>& argv,
236  const Option<std::map<std::string, std::string>>& environment,
237  const bool create_suspended = false,
238  const Option<std::tuple<int_fd, int_fd, int_fd>> pipes = None())
239 {
240  // TODO(andschwa): Assert that `command` and `argv[0]` are the same.
241  std::wstring arg_string = stringify_args(argv);
242  std::vector<wchar_t> arg_buffer(arg_string.begin(), arg_string.end());
243  arg_buffer.push_back(L'\0');
244 
245  // Create the process with a Unicode environment.
246  DWORD creation_flags = CREATE_UNICODE_ENVIRONMENT;
247  if (create_suspended) {
248  creation_flags |= CREATE_SUSPENDED;
249  }
250 
251  // Construct the environment that will be passed to `::CreateProcessW`.
253  std::vector<wchar_t> env_buffer;
254  if (env_string.isSome()) {
255  // This string contains the necessary null characters.
256  env_buffer.assign(env_string.get().begin(), env_string.get().end());
257  }
258 
259  wchar_t* process_env = env_buffer.empty() ? nullptr : env_buffer.data();
260 
261  PROCESS_INFORMATION process_info;
262  memset(&process_info, 0, sizeof(PROCESS_INFORMATION));
263 
264  STARTUPINFOW startup_info;
265  memset(&startup_info, 0, sizeof(STARTUPINFOW));
266  startup_info.cb = sizeof(STARTUPINFOW);
267 
268  // Hook up the stdin/out/err pipes and use the `STARTF_USESTDHANDLES`
269  // flag to instruct the child to use them [1].
270  // A more user-friendly example can be found in [2].
271  //
272  // [1] https://msdn.microsoft.com/en-us/library/windows/desktop/ms686331(v=vs.85).aspx
273  // [2] https://msdn.microsoft.com/en-us/library/windows/desktop/ms682499(v=vs.85).aspx
274  if (pipes.isSome()) {
275  startup_info.hStdInput = std::get<0>(pipes.get());
276  startup_info.hStdOutput = std::get<1>(pipes.get());
277  startup_info.hStdError = std::get<2>(pipes.get());
278  startup_info.dwFlags |= STARTF_USESTDHANDLES;
279  }
280 
281  BOOL create_process_result = ::CreateProcessW(
282  // This is replaced by the first token of `arg_buffer` string.
283  static_cast<LPCWSTR>(nullptr),
284  static_cast<LPWSTR>(arg_buffer.data()),
285  static_cast<LPSECURITY_ATTRIBUTES>(nullptr),
286  static_cast<LPSECURITY_ATTRIBUTES>(nullptr),
287  TRUE, // Inherited parent process handles.
288  creation_flags,
289  static_cast<LPVOID>(process_env),
290  static_cast<LPCWSTR>(nullptr), // Inherited working directory.
291  &startup_info,
292  &process_info);
293 
294  if (!create_process_result) {
295  return WindowsError(
296  "Failed to call `CreateProcess`: " + stringify(arg_string));
297  }
298 
299  return ProcessData{
300  SharedHandle{process_info.hProcess, ::CloseHandle},
301  SharedHandle{process_info.hThread, ::CloseHandle},
302  static_cast<pid_t>(process_info.dwProcessId)};
303 }
304 
305 } // namespace windows {
306 
307 } // namespace internal {
308 
309 namespace os {
310 
311 namespace Shell {
312 
313 // Canonical constants used as platform-dependent args to `exec` calls.
314 // `name` is the command name, `arg0` is the first argument received
315 // by the callee, usually the command name and `arg1` is the second
316 // command argument received by the callee.
317 constexpr const char* name = "cmd.exe";
318 constexpr const char* arg0 = "cmd.exe";
319 constexpr const char* arg1 = "/c";
320 
321 } // namespace Shell {
322 
323 template <typename... T>
324 Try<std::string> shell(const std::string& fmt, const T&... t) = delete;
325 
326 
327 template<typename... T>
328 inline int execlp(const char* file, T... t) = delete;
329 
330 
331 // Executes a command by calling "<command> <arguments...>", and
332 // returns after the command has been completed. Returns process exit code if
333 // succeeds, and -1 on error.
334 inline int spawn(
335  const std::string& command,
336  const std::vector<std::string>& arguments,
337  const Option<std::map<std::string, std::string>>& environment = None())
338 {
341 
342  if (process_data.isError()) {
343  LOG(WARNING) << process_data.error();
344  return -1;
345  }
346 
347  // Wait for the process synchronously.
348  ::WaitForSingleObject(
349  process_data.get().process_handle.get_handle(), INFINITE);
350 
351  DWORD status;
352  if (!::GetExitCodeProcess(
353  process_data.get().process_handle.get_handle(),
354  &status)) {
355  LOG(WARNING) << "Failed to `GetExitCodeProcess`: " << command;
356  return -1;
357  }
358 
359  // Return the child exit code.
360  return static_cast<int>(status);
361 }
362 
363 
364 // Executes a command by calling "cmd /c <command>", and returns
365 // after the command has been completed. Returns exit code if succeeds, and
366 // return -1 on error.
367 //
368 // Note: Be cautious about shell injection
369 // (https://en.wikipedia.org/wiki/Code_injection#Shell_injection)
370 // when using this method and use proper validation and sanitization
371 // on the `command`. For this reason in general `os::spawn` is
372 // preferred if a shell is not required.
373 inline int system(const std::string& command)
374 {
375  return os::spawn(Shell::name, {Shell::arg0, Shell::arg1, command});
376 }
377 
378 
379 // In order to emulate the semantics of `execvp`, `os::spawn` waits for the new
380 // process to exit, and returns its error code, which is propogated back to the
381 // parent via `exit` here.
382 inline int execvp(
383  const std::string& command,
384  const std::vector<std::string>& argv)
385 {
386  exit(os::spawn(command, argv));
387  return -1;
388 }
389 
390 
391 // NOTE: This function can accept `Argv` and `Envp` constructs through their
392 // explicit type conversions, but unlike the POSIX implementations, it cannot
393 // accept the raw forms.
394 inline int execvpe(
395  const std::string& command,
396  const std::vector<std::string>& argv,
397  const std::map<std::string, std::string>& envp)
398 {
399  exit(os::spawn(command, argv, envp));
400  return -1;
401 }
402 
403 } // namespace os {
404 
405 #endif // __STOUT_OS_WINDOWS_SHELL_HPP__
std::wstring stringify_args(const std::vector< std::string > &argv)
Definition: shell.hpp:163
constexpr const char * arg1
Definition: shell.hpp:43
Definition: option.hpp:28
HANDLE get_handle() const
Definition: windows.hpp:96
Definition: windows.hpp:78
Definition: try.hpp:34
int spawn(const std::string &command, const std::vector< std::string > &arguments)
Definition: shell.hpp:159
int execlp(const char *file, T...t)
Definition: shell.hpp:186
Definition: shell.hpp:211
Result< ProcessStatus > status(pid_t pid)
Definition: proc.hpp:166
Definition: error.hpp:106
UPID spawn(ProcessBase *process, bool manage=false)
Spawn a new process.
bool isSome() const
Definition: option.hpp:115
constexpr const char * arg0
Definition: shell.hpp:42
int execvp(const char *file, char *const argv[])
Definition: shell.hpp:192
Environment * environment
DWORD pid_t
Definition: windows.hpp:187
URI file(const std::string &path)
Creates a file URI with the given path on the local host.
Definition: file.hpp:33
SharedHandle process_handle
Definition: shell.hpp:212
SharedHandle thread_handle
Definition: shell.hpp:213
const T & get() const &
Definition: option.hpp:118
#define foreachpair(KEY, VALUE, ELEMS)
Definition: foreach.hpp:51
static Try error(const E &e)
Definition: try.hpp:42
process::Future< Nothing > transform(process::Owned< Reader< T >> &&reader, const std::function< std::string(const T &)> &func, process::http::Pipe::Writer writer)
This is a helper function that reads records from a Reader, applies a transformation to the records a...
Definition: recordio.hpp:112
Try< std::string > shell(const std::string &fmt, const T &...t)
Runs a shell command with optional arguments.
Definition: shell.hpp:71
Option< std::map< std::wstring, std::wstring > > get_system_env()
Definition: shell.hpp:42
Definition: none.hpp:27
bool isError() const
Definition: try.hpp:71
int execvpe(const std::string &command, const std::vector< std::string > &argv, const std::map< std::string, std::string > &envp)
Definition: shell.hpp:394
Option< std::wstring > create_process_env(const Option< std::map< std::string, std::string >> &env)
Definition: shell.hpp:104
pid_t pid
Definition: shell.hpp:214
std::string stringify(int flags)
const T & get() const
Definition: try.hpp:73
constexpr const char * name
Definition: shell.hpp:41
Try< ProcessData > create_process(const std::string &command, const std::vector< std::string > &argv, const Option< std::map< std::string, std::string >> &environment, const bool create_suspended=false, const Option< std::tuple< int_fd, int_fd, int_fd >> pipes=None())
Definition: shell.hpp:233
int system(const std::string &command)
Definition: shell.hpp:130