import subprocess import logging import sys import os import wizard from wizard import util """This is the path to the wizard executable as specified by the caller; it lets us recursively invoke wizard""" wizard_bin = sys.argv[0] def is_python(args): return args[0] == "python" or args[0] == wizard_bin class Shell(object): """An advanced shell, with the ability to do dry-run and log commands""" def __init__(self, dry = False): """ `dry` Don't run any commands, just print them""" self.dry = dry def call(self, *args, **kwargs): kwargs.setdefault("python", None) logging.info("Running `" + ' '.join(args) + "`") if self.dry: return if kwargs["python"] is None and is_python(args): kwargs["python"] = True # XXX: There is a possible problem here where we can fill up # the kernel buffer if we have 64KB of data. This shouldn't # be a problem, and the fix for such case would be to write to # temporary files instead of a pipe. # Another possible way of fixing this is converting from a # waitpid() pump to a select() pump, creating a pipe to # ourself, and then setting up a # SIGCHILD handler to write a single byte to the pipe to get # us out of select() when a subprocess exits. proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if hasattr(self, "async"): self.async(proc, args, **kwargs) return proc stdout, stderr = proc.communicate() self.log(stdout, stderr) if proc.returncode: if kwargs["python"]: eclass = PythonCallError else: eclass = CallError raise eclass(proc.returncode, args, stdout, stderr) return (stdout, stderr) def log(self, stdout, stderr): if stdout: logging.debug("STDOUT:\n" + stdout) if stderr: logging.debug("STDERR:\n" + stderr) def callAsUser(self, *args, **kwargs): user = kwargs.pop("user", None) uid = kwargs.pop("uid", None) kwargs.setdefault("python", is_python(args)) if not user and not uid: return self.call(*args, **kwargs) if util.get_operator_name(): # This might be generalized as "preserve some environment" args.insert(0, "SSH_GSSAPI_NAME=" + util.get_operator_name()) if uid: return self.call("sudo", "-u", "#" + str(uid), *args, **kwargs) if user: return self.call("sudo", "-u", user, *args, **kwargs) class ParallelShell(Shell): """Commands are queued here, and executed in parallel (with threading) in accordance with the maximum number of allowed subprocesses, and result in callback execution when they finish.""" def __init__(self, dry = False, max = 10): super(ParallelShell, self).__init__(dry=dry) self.running = {} self.max = max # maximum of commands to run in parallel def async(self, proc, args, python, on_success, on_error): """Gets handed a subprocess.Proc object from our deferred execution""" self.running[proc.pid] = (proc, args, python, on_success, on_error) def wait(self): # bail out immediately on initial ramp up if len(self.running) < self.max: return # now, wait for open pids. try: pid, status = os.waitpid(-1, 0) except OSError as e: if e.errno == errno.ECHILD: return raise e # ooh, zombie process. reap it proc, args, python, on_success, on_error = self.running.pop(pid) # XXX: this is slightly dangerous; should actually use # temporary files stdout = proc.stdout.read() stderr = proc.stderr.read() self.log(stdout, stderr) if status: if python: eclass = PythonCallError else: eclass = CallError on_error(eclass(proc.returncode, args, stdout, stderr)) return on_success(stdout, stderr) def join(self): """Waits for all of our subprocesses to terminate.""" try: while os.waitpid(-1, 0): pass except OSError as e: if e.errno == errno.ECHILD: return raise e class DummyParallelShell(ParallelShell): """Same API as ParallelShell, but doesn't actually parallelize (by using only one thread)""" def __init__(self, dry = False): super(DummyParallelShell, self).__init__(dry=dry, max=1) class CallError(wizard.Error): def __init__(self, code, args, stdout, stderr): self.code = code self.args = args self.stdout = stdout self.stderr = stderr def __str__(self): return "CallError [%d]" % self.code class PythonCallError(CallError): def __init__(self, code, args, stdout, stderr): self.name = util.get_exception_name(stderr) CallError.__init__(self, code, args, stdout, stderr) def __str__(self): return "PythonCallError [%s]" % self.name