Implement cache for already processed installs.

[wizard.git] / wizard / command / massmigrate.py

diff --git a/wizard/command/massmigrate.py b/wizard/command/massmigrate.py
index f247a0dab034f022134c800630823a5b31f3109f..d371ff6d97381d002a91bbbd49eb25fd9d9cfa40 100644 (file)
--- a/wizard/command/massmigrate.py
+++ b/wizard/command/massmigrate.py
@@ -5,6 +5,7 @@ import wizard
 from wizard import deploy
 from wizard import util
 from wizard import shell
+from wizard import cache
 from wizard.command import _base
 from wizard.command import migrate
 
@@ -15,6 +16,10 @@ Mass migrates an application to the new repository format.
 Essentially equivalent to running '%prog migrate' on all
 autoinstalls for a particular application found by parallel-find,
 but with advanced reporting.
+
+When doing an actual run, it is recommended to use --cache to
+be able to resume gracefully (without it, massmigrate must
+stat every install to find out if it migrated it yet).
 """
     parser = _base.WizardOptionParser(usage)
     parser.add_option("--no-parallelize", dest="no_parallelize", action="store_true",
@@ -23,6 +28,8 @@ but with advanced reporting.
             default=False, help="Print commands that would be run. Implies --no-parallelize")
     parser.add_option("--max", dest="max",
             default=10, help="Maximum subprocesses to run concurrently")
+    parser.add_option("--cache", dest="cache",
+            default=None, help="Cache file to read/write paths of already processed installs. These will be skipped.")
     options, args, logger = parser.parse_all(argv, logger)
     if len(args) > 1:
         parser.error("too many arguments")
@@ -41,6 +48,10 @@ but with advanced reporting.
         sh = shell.DummyParallelShell(logger=logger)
     else:
         sh = shell.ParallelShell(logger=logger, max=int(options.max))
+    if options.cache:
+        seen = cache.Cache(options.cache)
+    else:
+        seen = cache.DummyCache()
     for line in deploy.getInstallLines(global_options):
         # validate and filter the deployments
         try:
@@ -49,19 +60,23 @@ but with advanced reporting.
             continue
         name = d.getApplication().name
         if name != app: continue
+        if d.location in seen:
+            continue
         # actual meat
-        def on_success(stdout, stderr): pass # yay! don't do anything
-        def make_on_error(d): # make it early binding
+        def make_on_pair(d):
+            def on_success(stdout, stderr):
+                seen.add(d.location)
             def on_error(e):
                 if e.name == "wizard.command.migrate.AlreadyMigratedError":
+                    seen.add(d.location)
                     logger.info("Skipped already migrated %s" % d.location)
                 else:
                     name = e.name
                     if name not in errors: errors[name] = []
                     errors[name].append(d)
                     logger.error("ERROR [%s] in %s" % (name, d.location))
-            return on_error
-        on_error = make_on_error(d)
+            return (on_success, on_error)
+        on_success, on_error = make_on_pair(d)
         sh.wait() # wait for a parallel processing slot to be available
         sh.callAsUser(shell.wizard, "migrate", d.location, *base_args,
                       user=user, on_success=on_success, on_error=on_error)