Fix numerous bugs from our test runs.

- Only create subdirectories if we're forcing.
- Add support for --limit on mass-migrate
- Match for AlreadyMigratedError as well as fully qualified name.
- Add --force on mass-migrate
- Improve logging messages to remove unnecessary cruft
  and add important information
- Move .git to .git.bak, for record-keeping purposes
- Be more selective about logging for shell
- Resolve error case for exception parsing
- Make exceptions show up by removing --quiet

Signed-off-by: Edward Z. Yang <ezyang@mit.edu>

diff --git a/wizard/command/mass_migrate.py b/wizard/command/mass_migrate.py
index d8ed71036e6bb6b96b6fac38161ed9230b5cbd3a..def06fcba28c9d3a56c48936c04306f22a17a0dd 100644 (file)
--- a/wizard/command/mass_migrate.py
+++ b/wizard/command/mass_migrate.py
@@ -25,11 +25,13 @@ def main(argv, baton):
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
-            options.log_dir = os.path.join(options.log_dir, str(int(time.time())))
-            os.mkdir(options.log_dir) # if fails, be fatal
+            if options.force:
+                options.log_dir = os.path.join(options.log_dir, str(int(time.time())))
+                os.mkdir(options.log_dir) # if fails, be fatal
         os.chmod(options.log_dir, 0o777)
     # loop stuff
     errors = {}
+    i = 0
     for line in deploy.get_install_lines(options.versions_path):
         child_args = list(base_args)
         # validate and filter the deployments
@@ -53,12 +55,17 @@ def main(argv, baton):
         if options.log_dir:
             log_file = os.path.join(options.log_dir, shorten(d.location))
             child_args.append("--log-file=" + log_file)
+        # check if we want to punt due to --limit
+        i += 1
+        if i > options.limit:
+            break
         # actual meat
         def make_on_pair(d):
             def on_success(stdout, stderr):
                 seen.add(d.location)
             def on_error(e):
-                if e.name == "wizard.command.migrate.AlreadyMigratedError":
+                if e.name == "wizard.command.migrate.AlreadyMigratedError" or \
+                   e.name == "AlreadyMigratedError":
                     seen.add(d.location)
                     logging.info("Skipped already migrated %s" % d.location)
                 else:
@@ -100,6 +107,10 @@ untrusted repositories."""
             default=10, help="Maximum subprocesses to run concurrently")
     parser.add_option("--seen", dest="seen",
             default=None, help="File to read/write paths of already processed installs. These will be skipped.")
+    parser.add_option("--force", dest="force", action="store_true",
+            default=False, help="Force migrations to occur even if .scripts or .git exists.")
+    parser.add_option("--limit", dest="limit", type="int",
+            default=0, help="Limit the number of autoinstalls to look at.")
     baton.push(parser, "versions_path")
     baton.push(parser, "srv_path")
     baton.push(parser, "log_dir")
@@ -113,9 +124,7 @@ untrusted repositories."""
     return options, args
 
 def calculate_base_args(options):
-    base_args = command.makeBaseArgs(options, dry_run="--dry-run", srv_path="--srv-path")
-    if not options.debug:
-        base_args.append("--quiet")
+    base_args = command.makeBaseArgs(options, dry_run="--dry-run", srv_path="--srv-path", force="--force")
     return base_args
 
 def shorten(dir):

diff --git a/wizard/command/migrate.py b/wizard/command/migrate.py
index f859a025347b3af00f7d2e1baed4888e27fb052a..ff8175dbebf8b3066153b9978400e6f607760c2a 100644 (file)
--- a/wizard/command/migrate.py
+++ b/wizard/command/migrate.py
@@ -13,6 +13,7 @@ def main(argv, baton):
 
     shell.drop_priviledges(options)
 
+    logging.info("Migrating %s" % dir)
     logging.debug("uid is %d" % os.getuid())
 
     check_if_already_migrated(options)
@@ -31,10 +32,8 @@ def main(argv, baton):
 
     if not options.dry_run:
         deployment.scriptsifyVersion()
-        os.rename(".scripts-version", ".scripts/old-version") # archive
     else:
         logging.info("# create .scripts/version containing \"%s-%s-scripts\"" % (deployment.application.name, deployment.version))
-        logging.info("mv .scripts-version .scripts/old-version")
 
 def parse_args(argv, baton):
     usage = """usage: %prog migrate [ARGS] DIR
@@ -67,7 +66,9 @@ def check_if_already_migrated(options):
         else:
             if os.path.isdir(".git"):
                 logging.warning("Force removing .git directory")
-                if not options.dry_run: shutil.rmtree(".git")
+                if not options.dry_run:
+                    shutil.rmtree(".git.bak", ignore_errors=True)
+                    os.rename(".git", ".git.bak")
             if os.path.isdir(".scripts"):
                 logging.warning("Force removing .scripts directory")
                 if not options.dry_run: shutil.rmtree(".scripts")
@@ -116,16 +117,7 @@ def make_repository(sh, options, repo, tag):
     sh.call("git", "reset", tag, "--")
     # checkout the .scripts directory
     sh.call("git", "checkout", ".scripts")
-    # for verbose purposes, give us a git status and git diff
-    if options.verbose:
-        try:
-            sh.call("git", "status")
-        except shell.CallError:
-            pass
-        try:
-            sh.call("git", "diff")
-        except shell.CallError:
-            pass
+    logging.info("Diffstat:\n" + sh.eval("git", "diff", "--stat"))
     # commit user local changes
     message = "Autoinstall migration of %s locker.\n\n%s" % (util.get_dir_owner(), util.get_git_footer())
     util.set_git_env()

diff --git a/wizard/shell.py b/wizard/shell.py
index d40fa3dff02b3cd9cdbe28684f851cba04f9aaee..8748b93180865a5e910c1b3dccd55317a5761cb1 100644 (file)
--- a/wizard/shell.py
+++ b/wizard/shell.py
@@ -39,7 +39,7 @@ def drop_priviledges(options):
         return
     args = []
     for k,v in os.environ.items():
-        if k.startswith('WIZARD_'):
+        if k.startswith('WIZARD_') or k == "SSH_GSSAPI_NAME":
             args.append("%s=%s" % (k,v))
     args += sys.argv
     logging.debug("Dropping priviledges")
@@ -80,8 +80,13 @@ class Shell(object):
         >>> sh.call("cat", input='Foobar')
         ('Foobar', '')
         """
+        kwargs.setdefault("interactive", False)
+        kwargs.setdefault("strip", False)
         kwargs.setdefault("python", None)
-        logging.info("Running `" + ' '.join(args) + "`")
+        if kwargs["strip"]:
+            logging.debug("Evaluating `" + ' '.join(args) + "`")
+        else:
+            logging.info("Running `" + ' '.join(args) + "`")
         if self.dry:
             return
         if kwargs["python"] is None and is_python(args):
@@ -90,8 +95,6 @@ class Shell(object):
             args = list(args)
             args[0] = wizard_bin
         kwargs.setdefault("input", None)
-        kwargs.setdefault("interactive", False)
-        kwargs.setdefault("strip", False)
         if kwargs["interactive"]:
             stdout=sys.stdout
             stdin=sys.stdin
@@ -115,7 +118,10 @@ class Shell(object):
             return proc
         stdout, stderr = proc.communicate(kwargs["input"])
         if not kwargs["interactive"]:
-            self._log(stdout, stderr)
+            if kwargs["strip"]:
+                self._log(None, stderr)
+            else:
+                self._log(stdout, stderr)
         if proc.returncode:
             if kwargs["python"]: eclass = PythonCallError
             else: eclass = CallError

diff --git a/wizard/util.py b/wizard/util.py
index 2d4e4ff4d1993268dbd0822fd326c7e42049c504..4b5ded4a7d00508f734862172beaae85989f8317 100644 (file)
--- a/wizard/util.py
+++ b/wizard/util.py
@@ -93,6 +93,7 @@ def get_exception_name(output):
     """
     lines = output.split("\n")
     cue = False
+    result = "(unknown)"
     for line in lines[1:]:
         line = line.rstrip()
         if not line: continue