import os
import os.path
import pwd
+import hashlib
+import errno
+import time
+import itertools
import wizard
from wizard import deploy, util, shell, sset, command
base_args = calculate_base_args(options)
sh = make_shell(options)
seen = make_serialized_set(options)
- my_uid = os.getuid() # to see if we have root
+ is_root = not os.getuid()
+ warnings_log, errors_log = open_aggregate_logs(options)
# loop stuff
errors = {}
- for line in deploy.get_install_lines(options.versions_path):
- # validate and filter the deployments
- try:
- d = deploy.Deployment.parse(line)
- except deploy.DeploymentParseError, deploy.NoSuchApplication:
- continue
- name = d.application.name
- if name != app: continue
+ i = 0
+ # [] needed to workaround subtle behavior of frozenset("")
+ deploys = deploy.parse_install_lines([app], options.versions_path)
+ requested_deploys = itertools.islice(deploys, options.limit)
+ for i, d in enumerate(requested_deploys, 1)
+ # check if we want to punt due to --limit
if d.location in seen:
continue
- # security check: see if the user's directory is the prefix of
- # the deployment we're upgrading
- if not my_uid:
- uid = util.get_dir_uid(d.location)
- real = os.path.realpath(d.location)
- if not real.startswith(pwd.getpwuid(uid).pw_dir + "/"):
- logging.error("Security check failed, owner of deployment and owner of home directory mismatch for %s" % d.location)
- continue
+ if is_root and not security_check_homedir(d):
+ continue
+ child_args = list(base_args)
+ # calculate the log file, if a log dir was specified
+ if options.log_dir:
+ log_file = os.path.join(options.log_dir, calculate_log_name(i, d.location))
+ child_args.append("--log-file=" + log_file)
# actual meat
- def make_on_pair(d):
+ def make_on_pair(d, i):
+ # we need to make another stack frame so that d and i get specific bindings.
def on_success(stdout, stderr):
+ if stderr:
+ warnings_log.write("%s\n" % d.location)
+ logging.warning("Warnings [%04d] %s:\n%s" % (i, d.location, stderr))
seen.add(d.location)
def on_error(e):
- if e.name == "wizard.command.migrate.AlreadyMigratedError":
+ if e.name == "wizard.command.migrate.AlreadyMigratedError" or \
+ e.name == "AlreadyMigratedError":
seen.add(d.location)
logging.info("Skipped already migrated %s" % d.location)
else:
name = e.name
if name not in errors: errors[name] = []
errors[name].append(d)
- logging.error("%s in %s" % (name, d.location))
+ logging.error("%s in [%04d] %s" % (name, i, d.location))
+ errors_log.write("%s\n" % d.location)
return (on_success, on_error)
- on_success, on_error = make_on_pair(d)
- sh.wait() # wait for a parallel processing slot to be available
- sh.call("wizard", "migrate", d.location, *base_args,
+ on_success, on_error = make_on_pair(d, i)
+ sh.call("wizard", "migrate", d.location, *child_args,
on_success=on_success, on_error=on_error)
sh.join()
for name, deploys in errors.items():
user for testing purposes, but then you MUST NOT run this on
untrusted repositories."""
parser = command.WizardOptionParser(usage)
+ baton.push(parser, "log_dir")
+ parser.add_option("--seen", dest="seen",
+ default=None, help="File to read/write paths of already processed installs."
+ "These will be skipped.")
parser.add_option("--no-parallelize", dest="no_parallelize", action="store_true",
default=False, help="Turn off parallelization")
parser.add_option("--dry-run", dest="dry_run", action="store_true",
default=False, help="Print commands that would be run. Implies --no-parallelize")
parser.add_option("--max", dest="max",
default=10, help="Maximum subprocesses to run concurrently")
- parser.add_option("--seen", dest="seen",
- default=None, help="File to read/write paths of already processed installs. These will be skipped.")
+ parser.add_option("--force", dest="force", action="store_true",
+ default=False, help="Force migrations to occur even if .scripts or .git exists.")
+ parser.add_option("--limit", dest="limit", type="int",
+ default=0, help="Limit the number of autoinstalls to look at.")
baton.push(parser, "versions_path")
baton.push(parser, "srv_path")
options, args, = parser.parse_all(argv)
options.no_parallelize = True
return options, args
+def open_aggregate_logs(options):
+ warnings_logname = "/tmp/wizard-migrate-warnings.log"
+ errors_logname = "/tmp/wizard-migrate-errors.log"
+ if options.log_dir:
+ # must not be on AFS, since subprocesses won't be
+ # able to write to the logfiles do the to the AFS patch.
+ try:
+ os.mkdir(options.log_dir)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ if options.force:
+ options.log_dir = os.path.join(options.log_dir, str(int(time.time())))
+ os.mkdir(options.log_dir) # if fails, be fatal
+ os.chmod(options.log_dir, 0o777)
+ warnings_logname = os.path.join(options.log_dir, "warnings.log")
+ errors_logname = os.path.join(options.log_dir, "errors.log")
+ warnings_log = open(warnings_logname, "a")
+ errors_log = open(errors_logname, "a")
+ return warnings_log, errors_log
+
+def security_check_homedir(d):
+ uid = util.get_dir_uid(d.location)
+ real = os.path.realpath(d.location)
+ try:
+ if not real.startswith(pwd.getpwuid(uid).pw_dir + "/"):
+ logging.error("Security check failed, owner of deployment and"
+ "owner of home directory mismatch for %s" % d.location)
+ return False
+ except KeyError:
+ logging.error("Security check failed, could not look up"
+ "owner of %s (uid %d)" % (d.location, uid))
+ return False
+ return True
+
def calculate_base_args(options):
- base_args = command.makeBaseArgs(options, dry_run="--dry-run", srv_path="--srv-path")
- if not options.debug:
- base_args.append("--quiet")
- return base_args
+ return command.makeBaseArgs(options, dry_run="--dry-run", srv_path="--srv-path",
+ force="--force")
+
+def calculate_log_name(i, dir):
+ return "%04d" % i + dir.replace('/', '-') + ".log"
def make_shell(options):
if options.no_parallelize:
- sh = shell.DummyParallelShell()
+ return shell.DummyParallelShell()
else:
- sh = shell.ParallelShell(max=int(options.max))
- return sh
+ return shell.ParallelShell(max=int(options.max))
def make_serialized_set(options):
if options.seen:
- seen = sset.SerializedSet(options.seen)
+ return sset.SerializedSet(options.seen)
else:
- seen = sset.DummySerializedSet()
- return seen
+ return sset.DummySerializedSet()
scripts.mit.edu / wizard.git / blobdiff