wizard/command/mass_upgrade.py

   1 import logging
   2 import os
   3 import os.path
   4 import itertools
   5 import sys
   6 import shutil
   7 import errno
   8
   9 from wizard import deploy, report, shell, sset, command
  10 from wizard.command import upgrade
  11
  12 def main(argv, baton):
  13     options, args = parse_args(argv, baton)
  14     app = args[0]
  15     sh = shell.ParallelShell.make(options.no_parallelize, options.max_processes)
  16     command.create_logdir(options.log_dir)
  17     # setup reports
  18     human_status = {
  19         'up_to_date': 'are now up-to-date',
  20         'not_migrated': 'were not migrated',
  21         'merge': 'had merge failures',
  22         'verify': 'had web verification errors',
  23         'backup_failure': 'had a backup failure',
  24         'blacklisted': 'were blacklisted',
  25         'db': 'had database errors',
  26         'quota': 'had too low quota',
  27         'permissions': 'had insufficient permissions for upgrade'
  28     }
  29     if options.remerge:
  30         os.unlink(os.path.join(options.log_dir, 'merge.txt'))
  31     status = (report.make_fresh if options.redo else report.make)(options.log_dir, *human_status.keys())
  32     runtime = report.make_fresh(options.log_dir, 'success', 'lookup', 'warnings', 'errors')
  33     # setup rr_cache
  34     rr_cache = os.path.join(options.log_dir, "rr-cache")
  35     try:
  36         os.mkdir(rr_cache)
  37     except OSError as e:
  38         if e.errno != errno.EEXIST:
  39             raise
  40     os.chmod(rr_cache, 0o777)
  41     # setup base arguments
  42     base_args = calculate_base_args(options)
  43     base_args.append("--non-interactive")
  44     base_args.append("--rr-cache=" + rr_cache)
  45     # loop variables
  46     errors = {}
  47     i = 0
  48     deploys = deploy.parse_install_lines(app, options.versions_path, user=options.user)
  49     requested_deploys = itertools.islice(deploys, options.limit)
  50     # clean up /dev/shm/wizard
  51     if os.path.exists("/dev/shm/wizard"):
  52         shutil.rmtree("/dev/shm/wizard")
  53         os.mkdir("/dev/shm/wizard")
  54         os.chmod("/dev/shm/wizard", 0o777)
  55     try:
  56         for i, d in enumerate(requested_deploys, 1):
  57             runtime.write("lookup", i, d.location)
  58             if not os.getuid() and not command.security_check_homedir(d.location):
  59                 continue
  60             if not options.redo:
  61                 found = False
  62                 for r in status.reports.values():
  63                     if i in r.values:
  64                         found = True
  65                         break
  66                 if found:
  67                     continue
  68             # XXX: we may be able to punt based on detected versions from d, which
  69             # would be faster than spinning up a new process. On the other hand,
  70             # our aggressive caching strategies using reports make this mostly not a problem
  71             logging.info("[%04d] Processing %s", i, d.location)
  72             child_args = list(base_args) # copy
  73             # calculate the log file, if a log dir was specified
  74             if options.log_dir:
  75                 log_file = command.calculate_log_name(options.log_dir, i)
  76                 child_args.append("--log-file=" + log_file)
  77             # actual meat
  78             def make_on_pair(d, i):
  79                 # we need to make another stack frame so that d and i get specific bindings.
  80                 def on_success(stdout, stderr):
  81                     if stderr:
  82                         runtime.write("warnings", i, d.location)
  83                         logging.warning("[%04d] Warnings at [%s]:\n%s", i, d.location, stderr)
  84                     runtime.write("success", i, d.location)
  85                     status.write("up_to_date", i, d.location)
  86                 def on_error(e):
  87                     if e.name == "AlreadyUpgraded":
  88                         logging.info("[%04d] Skipped already upgraded %s" % (i, d.location))
  89                         status.write("up_to_date", i, d.location)
  90                     elif e.name == "MergeFailed":
  91                         conflicts, _, tmpdir = e.stdout.rstrip().partition(" ")
  92                         logging.warning("[%04d] Conflicts in %s files: resolve at [%s], source at [%s]",
  93                                         i, conflicts, tmpdir, d.location)
  94                         status.write("merge", i, tmpdir, conflicts, d.location)
  95                     elif e.name == "BlacklistedError":
  96                         reason = e.stdout.rstrip().replace("\n", " ")
  97                         logging.warning("[%04d] Blacklisted because of '%s' at %s", i, reason, d.location)
  98                         status.write("blacklisted", i, d.location, reason)
  99                     elif e.name == "WebVerificationError":
 100                         url = d.url.geturl()
 101                         # This should actually be a warning, but it's a really common error
 102                         logging.info("[%04d] Could not verify application at %s", i, url)
 103                         status.write("verify", i, url)
 104                     elif e.name == "DatabaseVerificationError":
 105                         logging.info("[%04d] Could not verify database ast %s", i, d.location)
 106                         status.write("db", i, d.location)
 107                     elif e.name == "NotMigratedError":
 108                         logging.info("[%04d] Application not migrated at %s", i, d.location)
 109                         status.write("not_migrated", i, d.location)
 110                     elif e.name == "BackupFailure":
 111                         logging.info("[%04d] Failed backups at %s", i, d.location)
 112                         status.write("backup_failure", i, d.location)
 113                     elif e.name == "QuotaTooLow":
 114                         logging.info("[%04d] Quota too low at %s", i, d.location)
 115                         status.write("quota", i, d.location)
 116                     elif e.name == "PermissionsError":
 117                         logging.info("[%04d] Insufficient permissions to upgrade %s", i, d.location)
 118                         status.write("permissions", i, d.location)
 119                     else:
 120                         errors.setdefault(e.name, []).append(d)
 121                         logging.error("[%04d] %s in %s", i, e.name, d.location)
 122                         runtime.write("errors", i, e.name, d.location)
 123                         # lack of status write means that we'll always retry
 124                 return (on_success, on_error)
 125             on_success, on_error = make_on_pair(d, i)
 126             sh.call("wizard", "upgrade", d.location, *child_args,
 127                           on_success=on_success, on_error=on_error)
 128         sh.join()
 129     finally:
 130         sys.stderr.write("\n")
 131         for name, deploys in errors.items():
 132             logging.warning("%s from %d installs", name, len(deploys))
 133         print
 134         total = sum(len(x.values) for x in status.reports.values())
 135         def printPercent(description, number):
 136             print "% 4d out of % 4d installs (% 5.1f%%) %s" % (number, total, float(number)/total*100, description)
 137         error_count = sum(len(e) for e in errors.values())
 138         if error_count:
 139             printPercent("had unusual errors", error_count)
 140         for name, description in human_status.items():
 141             values = status.reports[name].values
 142             if values:
 143                 printPercent(description, len(values))
 144         sys.stderr.write("\n")
 145         print "%d installs were upgraded this run" % len(runtime.reports["success"].values)
 146
 147 def parse_args(argv, baton):
 148     usage = """usage: %prog mass-upgrade [ARGS] APPLICATION
 149
 150 Mass upgrades an application to the latest version.  Essentially
 151 equivalent to running '%prog upgrade' on all autoinstalls for a
 152 particular application found by parallel-find, but with advanced
 153 reporting.
 154
 155 This command is intended to be run as root on a server with
 156 the scripts AFS patch."""
 157     parser = command.WizardOptionParser(usage)
 158     baton.push(parser, "log_dir")
 159     baton.push(parser, "no_parallelize")
 160     baton.push(parser, "dry_run")
 161     baton.push(parser, "max_processes")
 162     baton.push(parser ,"limit")
 163     baton.push(parser, "versions_path")
 164     baton.push(parser, "srv_path")
 165     baton.push(parser, "user")
 166     parser.add_option("--force", dest="force", action="store_true",
 167             default=False, help="Force running upgrade even if it's already at latest version.")
 168     parser.add_option("--redo", dest="redo", action="store_true",
 169             default=False, help="Redo all upgrades; use this if you updated Wizard's code.")
 170     parser.add_option("--remerge", dest="remerge", action="store_true",
 171             default=False, help="Redo all merges.")
 172     options, args, = parser.parse_all(argv)
 173     if len(args) > 1:
 174         parser.error("too many arguments")
 175     elif not args:
 176         parser.error("must specify application to upgrade")
 177     return options, args
 178
 179 def calculate_base_args(options):
 180     # Do not pass --debug to subprocesses, since it will trigger the OS
 181     # kernel buffer issue
 182     return command.make_base_args(options, dry_run="--dry-run", srv_path="--srv-path", force="--force", debug=None)
 183