import logging import os import os.path import itertools import sys import shutil import errno from wizard import deploy, scripts, shell, sset, command from wizard.command import upgrade def main(argv, baton): options, args = parse_args(argv, baton) app = args[0] base_args = calculate_base_args(options) base_args.append("--non-interactive") sh = shell.ParallelShell.make(options.no_parallelize, options.max_processes) command.create_logdir(options.log_dir) seen = sset.make(options.seen) is_root = not os.getuid() report = command.open_reports(options.log_dir, ('lookup', 'warnings', 'errors', 'success'), options.redo, ('not_migrated', 'merge', 'verify', 'backup_failure', 'blacklisted')) rr_cache = os.path.join(options.log_dir, "rr-cache") base_args.append("--rr-cache=" + rr_cache) try: os.mkdir(rr_cache) except OSError as e: if e.errno != errno.EEXIST: raise # loop stuff errors = {} i = 0 deploys = deploy.parse_install_lines(app, options.versions_path, user=options.user) requested_deploys = itertools.islice(deploys, options.limit) # clean up /dev/shm/wizard if os.path.exists("/dev/shm/wizard"): shutil.rmtree("/dev/shm/wizard") os.mkdir("/dev/shm/wizard") os.chmod("/dev/shm/wizard", 0o777) try: for i, d in enumerate(requested_deploys, 1): report.lookup.write("%04d %s\n" % (i, d.location)) # pylint: disable-msg=E1101 report.flush() # check if we want to punt due to --limit if d.location in seen: continue if i in report.skip: continue if is_root and not command.security_check_homedir(d.location): continue # XXX: we may be able to punt based on detected versions from d, which # would be faster than spinning up a new process. On the other hand, # `seen` makes this mostly not a problem logging.info("[%04d] Processing %s" % (i, d.location)) child_args = list(base_args) # calculate the log file, if a log dir was specified if options.log_dir: log_file = command.calculate_log_name(options.log_dir, i) child_args.append("--log-file=" + log_file) # actual meat def make_on_pair(d, i): # we need to make another stack frame so that d and i get specific bindings. def on_success(stdout, stderr): if stderr: report.lookup.write("[%04d] %s\n" % (i, d.location)) # pylint: disable-msg=E1101 logging.warning("[%04d] Warnings at [%s]:\n%s" % (i, d.location, stderr)) seen.add(d.location) report.success.write("%s\n" % d.location) # pylint: disable-msg=E1101 report.successes += 1 report.flush() def on_error(e): if e.name == "AlreadyUpgraded": seen.add(d.location) logging.info("[%04d] Skipped already upgraded %s" % (i, d.location)) elif e.name == "MergeFailed": seen.add(d.location) conflicts, _, tmpdir = e.stdout.rstrip().partition(" ") logging.warning("[%04d] Conflicts in %d files: resolve at [%s], source at [%s]" % (i, int(conflicts), tmpdir, d.location)) report.merge.write("[%04d] %s %d %s\n" % (i, tmpdir, int(conflicts), d.location)) # pylint: disable-msg=E1101 report.fails['merge'] += 1 elif e.name == "BlacklistedError": reason = e.stdout.rstrip() reason = reason.replace("\n", " ") shortmsg = "[%04d] %s %s\n" % (i, d.location, reason) report.blacklisted.write(shortmsg) # pylint: disable-msg=E1101 report.fails['blacklisted'] += 1 logging.warning("[%04d] Blacklisted because of '%s' at %s" % (i, reason, d.location)) else: name = e.name if name == "WebVerificationError": url = d.url.geturl() # This should actually be a warning, but # it's a really common error logging.info("[%04d] Could not verify application at %s" % (i, url)) report.verify.write("[%04d] %s\n" % (i, url)) # pylint: disable-msg=E1101 report.fails['verify'] += 1 elif e.name == "NotMigratedError": logging.info("[%04d] Application not migrated at %s" % (i, d.location)) report.not_migrated.write("[%04d] %s\n" % (i, d.location)) # pylint: disable-msg=E1101 report.fails['not_migrated'] += 1 else: if name not in errors: errors[name] = [] errors[name].append(d) msg = "[%04d] %s in %s" % (i, name, d.location) logging.error(msg) report.errors.write(msg + "\n") # pylint: disable-msg=E1101 shortmsg = "[%04d] %s\n" % (i, d.location) if name == "BackupFailure": report.backup_failure.write(shortmsg) # pylint: disable-msg=E1101 report.fails['backup_failure'] += 1 report.flush() return (on_success, on_error) on_success, on_error = make_on_pair(d, i) sh.call("wizard", "upgrade", d.location, *child_args, on_success=on_success, on_error=on_error) sh.join() finally: sys.stderr.write("\n") for name, deploys in errors.items(): logging.warning("%s from %d installs" % (name, len(deploys))) print def printPercent(description, number, total): print "%d out of %d installs (%.1f%%) had %s" % (number, total, float(number)/total*100, description) if report.fails['merge']: printPercent("merge conflicts", report.fails['merge'], i) if report.fails['verify']: printPercent("web verification failure", report.fails['verify'], i) printPercent("successful upgrades", report.successes, i) def parse_args(argv, baton): usage = """usage: %prog mass-upgrade [ARGS] APPLICATION Mass upgrades an application to the latest scripts version. Essentially equivalent to running '%prog upgrade' on all autoinstalls for a particular application found by parallel-find, but with advanced reporting. This command is intended to be run as root on a server with the scripts AFS patch.""" parser = command.WizardOptionParser(usage) baton.push(parser, "log_dir") baton.push(parser, "seen") baton.push(parser, "no_parallelize") baton.push(parser, "dry_run") baton.push(parser, "max_processes") baton.push(parser ,"limit") baton.push(parser, "versions_path") baton.push(parser, "srv_path") baton.push(parser, "user") parser.add_option("--force", dest="force", action="store_true", default=False, help="Force running upgrade even if it's already at latest version.") parser.add_option("--redo", dest="redo", action="store_true", default=False, help="Redo failed upgrades; use this if you updated Wizard's code.") options, args, = parser.parse_all(argv) if len(args) > 1: parser.error("too many arguments") elif not args: parser.error("must specify application to upgrade") return options, args def calculate_base_args(options): return command.make_base_args(options, dry_run="--dry-run", srv_path="--srv-path", force="--force")