Fix kernel buffer overflow by avoiding passing --debug to subprocesses.

Signed-off-by: Edward Z. Yang <ezyang@mit.edu>

diff --git a/TODO b/TODO
index f27087b55c662c45b20b851f9cafa8985656649c..89116b16d905ee522772838b5aa89865ab950a75 100644 (file)
--- a/TODO
+++ b/TODO
@@ -1,6 +1,9 @@
 - Make scripts_plugin email heuristic less stupid, or maybe even ask for an
   email. This is tracked as Scripts #224 (this issue) and Scripts #193
   (tracking a contact address).
+- Current parallelization probably does a bad job distributing
+  working tasks over different components of the pipeline. Fix
+  this by adding jitter? Trying to smear things out?
 
 - Test head doesn't do quite the right thing with version numbers
   (shouldn't git describe, instead should give a version infinitely

diff --git a/wizard/command/__init__.py b/wizard/command/__init__.py
index 8d5e14c3c434b922db807dffc7214c4ceb4aab0c..e3558f6dfb3fdbe82dd015ddaa5c1f529f2fdfd0 100644 (file)
--- a/wizard/command/__init__.py
+++ b/wizard/command/__init__.py
@@ -57,11 +57,14 @@ def make_base_args(options, **grab):
     """Takes parsed options, and breaks them back into a command
     line string that we can pass into a subcommand"""
     args = []
-    grab["debug"]   = "--debug"
-    grab["verbose"] = "--verbose"
-    grab["quiet"]   = "--quiet"
-    #grab["log_db"] = "--log-db"
+    # Note: you can override these by passing in debug = None
+    # in your kwargs.
+    grab.setdefault("debug", "--debug")
+    grab.setdefault("verbose", "--verbose")
+    grab.setdefault("quiet", "--quiet")
+    #grab.setdefault("log_db", "--log-db")
     for k,flag in grab.items():
+        if not flag: continue
         value = getattr(options, k)
         if not value: continue
         args.append(flag)

diff --git a/wizard/command/mass_upgrade.py b/wizard/command/mass_upgrade.py
index 154455f190a6e7a672549fb0a38a96be9e83f92f..f9aa8bbbc3e90d436395a6b749fa05d3a70a82a4 100644 (file)
--- a/wizard/command/mass_upgrade.py
+++ b/wizard/command/mass_upgrade.py
@@ -177,5 +177,7 @@ the scripts AFS patch."""
     return options, args
 
 def calculate_base_args(options):
-    return command.make_base_args(options, dry_run="--dry-run", srv_path="--srv-path", force="--force")
+    # Do not pass --debug to subprocesses, since it will trigger the OS
+    # kernel buffer issue
+    return command.make_base_args(options, dry_run="--dry-run", srv_path="--srv-path", force="--force", debug=None)
 

diff --git a/wizard/shell.py b/wizard/shell.py
index b8d4bd1e6dcfa8cda550c192b63a74a3146a435c..2197cdecc3cf219beebe5d5afe10744917682869 100644 (file)
--- a/wizard/shell.py
+++ b/wizard/shell.py
@@ -121,13 +121,16 @@ class Shell(object):
             stderr=kwargs["stderr"]
         # XXX: There is a possible problem here where we can fill up
         # the kernel buffer if we have 64KB of data.  This shouldn't
-        # be a problem, and the fix for such case would be to write to
+        # normally be a problem, and the fix for such case would be to write to
         # temporary files instead of a pipe.
+        #
+        # However, it *is* a problem when you do something silly, like
+        # pass --debug to mass-upgrade.
+        #
         # Another possible way of fixing this is converting from a
         # waitpid() pump to a select() pump, creating a pipe to
-        # ourself, and then setting up a
-        # SIGCHILD handler to write a single byte to the pipe to get
-        # us out of select() when a subprocess exits.
+        # ourself, and then setting up a SIGCHILD handler to write a single
+        # byte to the pipe to get us out of select() when a subprocess exits.
         proc = subprocess.Popen(args, stdout=stdout, stderr=stderr, stdin=stdin, cwd=self.cwd, )
         if self._async(proc, args, **kwargs):
             return proc