wizard/command/mass_upgrade.py

   1 import logging
   2 import os
   3 import os.path
   4 import itertools
   5 import sys
   6 import shutil
   7
   8 from wizard import deploy, scripts, shell, sset, command
   9 from command import upgrade
  10
  11 def main(argv, baton):
  12     options, args = parse_args(argv, baton)
  13     app = args[0]
  14     base_args = calculate_base_args(options)
  15     sh = shell.ParallelShell.make(options.no_parallelize, options.max_processes)
  16     command.create_logdir(options.log_dir)
  17     seen = sset.make(options.seen)
  18     is_root = not os.getuid()
  19     report = command.open_reports(options.log_dir, ('lookup', 'warnings', 'errors'),
  20         options.redo, ('merge', 'verify', 'backup_failure', 'blacklisted'))
  21     # loop stuff
  22     errors = {}
  23     i = 0
  24     deploys = deploy.parse_install_lines(app, options.versions_path, user=options.user)
  25     requested_deploys = itertools.islice(deploys, options.limit)
  26     # clean up /dev/shm/wizard
  27     if os.path.exists("/dev/shm/wizard"):
  28         shutil.rmtree("/dev/shm/wizard")
  29         os.mkdir("/dev/shm/wizard")
  30         os.chmod("/dev/shm/wizard", 0o777)
  31     try:
  32         for i, d in enumerate(requested_deploys, 1):
  33             report.lookup.write("%04d %s\n" % (i, d.location)) # pylint: disable-msg=E1101
  34             report.flush()
  35             # check if we want to punt due to --limit
  36             if d.location in seen:
  37                 continue
  38             if i in report.skip:
  39                 continue
  40             if is_root and not command.security_check_homedir(d.location):
  41                 continue
  42             # XXX: we may be able to punt based on detected versions from d, which
  43             # would be faster than spinning up a new process. On the other hand,
  44             # `seen` makes this mostly not a problem
  45             logging.info("[%04d] Processing %s" % (i, d.location))
  46             child_args = list(base_args)
  47             # calculate the log file, if a log dir was specified
  48             if options.log_dir:
  49                 log_file = command.calculate_log_name(options.log_dir, i)
  50                 child_args.append("--log-file=" + log_file)
  51             # actual meat
  52             def make_on_pair(d, i):
  53                 # we need to make another stack frame so that d and i get specific bindings.
  54                 def on_success(stdout, stderr):
  55                     if stderr:
  56                         report.lookup.write("[%04d] %s\n" % (i, d.location)) # pylint: disable-msg=E1101
  57                         logging.warning("[%04d] Warnings at [%s]:\n%s" % (i, d.location, stderr))
  58                     seen.add(d.location)
  59                     report.flush()
  60                 def on_error(e):
  61                     if e.name == "AlreadyUpgraded":
  62                         seen.add(d.location)
  63                         logging.info("[%04d] Skipped already upgraded %s" % (i, d.location))
  64                     elif e.name == "MergeFailed":
  65                         seen.add(d.location)
  66                         conflicts, _, tmpdir = e.stdout.rstrip().partition(" ")
  67                         logging.warning("[%04d] Conflicts in %d files: resolve at [%s], source at [%s]" % (i, int(conflicts), tmpdir, d.location))
  68                         report.merge.write("[%04d] %s %d %s\n" % (i, tmpdir, int(conflicts), d.location)) # pylint: disable-msg=E1101
  69                         report.fails['merge'] += 1
  70                     elif e.name == "BlacklistedError":
  71                         reason = e.stdout.rstrip()
  72                         reason = reason.replace("\n", " ")
  73                         shortmsg = "[%04d] %s %s\n" % (i, d.location, reason)
  74                         report.blacklisted.write(shortmsg) # pylint: disable-msg=E1101
  75                         report.fails['blacklisted'] += 1
  76                         logging.warning("[%04d] Blacklisted because of '%s' at %s" % (i, reason, d.location))
  77                     else:
  78                         name = e.name
  79                         if name == "WebVerificationError":
  80                             try:
  81                                 host, path = scripts.get_web_host_and_path(d.location)
  82                                 url = "http://%s%s" % (host, path)
  83                             except ValueError:
  84                                 url = d.location
  85                             # This should actually be a warning, but
  86                             # it's a really common error
  87                             logging.info("[%04d] Could not verify application at %s" % (i, url))
  88                             report.verify.write("[%04d] %s\n" % (i, url)) # pylint: disable-msg=E1101
  89                             report.fails['verify'] += 1
  90                         else:
  91                             if name not in errors: errors[name] = []
  92                             errors[name].append(d)
  93                             msg = "[%04d] %s in %s" % (i, name, d.location)
  94                             logging.error(msg)
  95                             report.errors.write(msg + "\n") # pylint: disable-msg=E1101
  96                             shortmsg = "[%04d] %s\n" % (i, d.location)
  97                             if name == "BackupFailure":
  98                                 report.backup_failure.write(shortmsg) # pylint: disable-msg=E1101
  99                                 report.fails['backup_failure'] += 1
 100                     report.flush()
 101                 return (on_success, on_error)
 102             on_success, on_error = make_on_pair(d, i)
 103             sh.call("wizard", "upgrade", d.location, *child_args,
 104                           on_success=on_success, on_error=on_error)
 105         sh.join()
 106     finally:
 107         sys.stderr.write("\n")
 108         for name, deploys in errors.items():
 109             logging.warning("%s from %d installs" % (name, len(deploys)))
 110         def printPercent(description, number, total):
 111             logging.warning("%d out of %d installs (%.1f%%) had %s" % (number, total, float(number)/total*100, description))
 112         if report.fails['merge']:
 113             printPercent("merge conflicts", report.fails['merge'], i)
 114         if report.fails['verify']:
 115             printPercent("web verification failure", report.fails['verify'], i)
 116
 117 def parse_args(argv, baton):
 118     usage = """usage: %prog mass-upgrade [ARGS] APPLICATION
 119
 120 Mass upgrades an application to the latest scripts version.
 121 Essentially equivalent to running '%prog upgrade' on all
 122 autoinstalls for a particular application found by parallel-find,
 123 but with advanced reporting.
 124
 125 This command is intended to be run as root on a server with
 126 the scripts AFS patch."""
 127     parser = command.WizardOptionParser(usage)
 128     baton.push(parser, "log_dir")
 129     baton.push(parser, "seen")
 130     baton.push(parser, "no_parallelize")
 131     baton.push(parser, "dry_run")
 132     baton.push(parser, "max_processes")
 133     baton.push(parser ,"limit")
 134     baton.push(parser, "versions_path")
 135     baton.push(parser, "srv_path")
 136     baton.push(parser, "user")
 137     parser.add_option("--force", dest="force", action="store_true",
 138             default=False, help="Force running upgrade even if it's already at latest version.")
 139     parser.add_option("--redo", dest="redo", action="store_true",
 140             default=False, help="Redo failed upgrades; use this if you updated Wizard's code.")
 141     options, args, = parser.parse_all(argv)
 142     if len(args) > 1:
 143         parser.error("too many arguments")
 144     elif not args:
 145         parser.error("must specify application to upgrade")
 146     return options, args
 147
 148 def calculate_base_args(options):
 149     return command.make_base_args(options, dry_run="--dry-run", srv_path="--srv-path", force="--force")
 150