import optparse import logging import os import os.path import pwd import hashlib import errno import time import itertools import wizard from wizard import deploy, util, scripts, shell, sset, command def main(argv, baton): options, args = parse_args(argv, baton) app = args[0] base_args = calculate_base_args(options) sh = shell.ParallelShell.make(options.no_parallelize, options.max_processes) command.create_logdir(options.log_dir) seen = sset.make(options.seen) is_root = not os.getuid() report = command.open_reports(options.log_dir, ('lookup', 'warnings', 'errors'), options.redo, ('merge', 'verify')) # loop stuff errors = {} i = 0 fails = { "merge": 0, "verify": 0, } deploys = deploy.parse_install_lines(app, options.versions_path, user=options.user) requested_deploys = itertools.islice(deploys, options.limit) for i, d in enumerate(requested_deploys, 1): report.lookup.write("%04d %s\n" % (i, d.location)) # check if we want to punt due to --limit if d.location in seen: continue if i in report.skip: continue if is_root and not command.security_check_homedir(d.location): continue # XXX: we may be able to punt based on detected versions from d, which # would be faster than spinning up a new process. On the other hand, # `seen` makes this mostly not a problem logging.info("[%04d] Processing %s" % (i, d.location)) child_args = list(base_args) # calculate the log file, if a log dir was specified if options.log_dir: log_file = command.calculate_log_name(options.log_dir, i) child_args.append("--log-file=" + log_file) # actual meat def make_on_pair(d, i): # we need to make another stack frame so that d and i get specific bindings. def on_success(stdout, stderr): if stderr: report.lookup.write("[%04d] %s\n" % (i, d.location)) logging.warning("[%04d] Warnings at [%s]:\n%s" % (i, d.location, stderr)) seen.add(d.location) def on_error(e): if e.name == "AlreadyUpgraded": seen.add(d.location) logging.info("[%04d] Skipped already upgraded %s" % (i, d.location)) elif e.name == "MergeFailed": seen.add(d.location) conflicts, _, tmpdir = e.stdout.rstrip().partition(" ") logging.warning("[%04d] Conflicts in %d files: resolve at [%s], source at [%s]" % (i, int(conflicts), tmpdir, d.location)) report.merge.write("[%04d] %s %d %s\n" % (i, tmpdir, int(conflicts), d.location)) fails['merge'] += 1 else: name = e.name if name == "WebVerificationError": try: host, path = scripts.get_web_host_and_path(d.location) url = "http://%s%s" % (host, path) except ValueError: url = d.location # This should actually be a warning, but # it's a really common error logging.info("[%04d] Could not verify application at %s" % (i, url)) report.verify.write("[%04d] %s\n" % (i, url)) fails['verify'] += 1 else: if name not in errors: errors[name] = [] errors[name].append(d) msg = "[%04d] %s in %s" % (i, name, d.location) logging.error(msg) report.errors.write(msg + "\n") return (on_success, on_error) on_success, on_error = make_on_pair(d, i) sh.call("wizard", "upgrade", d.location, *child_args, on_success=on_success, on_error=on_error) sh.join() for name, deploys in errors.items(): logging.warning("%s from %d installs" % (name, len(deploys))) def printPercent(description, number, total): logging.warning("%d out of %d installs (%.1f%%) had %s" % (number, total, float(number)/total*100, description)) if fails['merge']: printPercent("merge conflicts", fails['merge'], i) if fails['verify']: printPercent("web verification failure", fails['verify'], i) def parse_args(argv, baton): usage = """usage: %prog mass-upgrade [ARGS] APPLICATION Mass upgrades an application to the latest scripts version. Essentially equivalent to running '%prog upgrade' on all autoinstalls for a particular application found by parallel-find, but with advanced reporting. This command is intended to be run as root on a server with the scripts AFS patch.""" parser = command.WizardOptionParser(usage) baton.push(parser, "log_dir") baton.push(parser, "seen") baton.push(parser, "no_parallelize") baton.push(parser, "dry_run") baton.push(parser, "max_processes") baton.push(parser ,"limit") baton.push(parser, "versions_path") baton.push(parser, "srv_path") baton.push(parser, "user") parser.add_option("--force", dest="force", action="store_true", default=False, help="Force running upgrade even if it's already at latest version.") parser.add_option("--redo", dest="redo", action="store_true", default=False, help="Redo failed upgrades; use this if you updated Wizard's code.") options, args, = parser.parse_all(argv) if len(args) > 1: parser.error("too many arguments") elif not args: parser.error("must specify application to upgrade") return options, args def calculate_base_args(options): return command.makeBaseArgs(options, dry_run="--dry-run", srv_path="--srv-path", force="--force")