TODO NOW:
-- Make it faster
- - Certain classes of error will continually fail, so they should
- put in a different "seen" file which also skips them, unless
- we have some sort of gentle force
-
- Keep my sanity when upgrading 1000 installs
- Distinguish between errors(?)
- Custom merge algo: absolute php.ini symlinks to relative symlinks (this
fail, otherwise things are left conflicted, and not easy to fix.
- Prune -7 call errors and automatically reprocess them (with a
strike out counter of 3)--this requires better error parsing
- - Snap-in conflict resolution teaching:
- 1. View the merge conflicts after doing a short run
- 2. Identify common merge conflicts
- 3. Copypaste the conflict markers to the application. Scrub
- user-specific data; this may mean removing the entire
- upper bit which is the user-version.
- 4. Specify which section to keep. /Usually/ this means
- punting the new change, but if the top was specified
- it means we get a little more flexibility. Try to
- minimize wildcarding: those things need to be put into
- subpatterns and then reconstituted into the output.
+ - Report stats if I C-C the process
- Distinguish from logging and reporting (so we can easily send mail
to users)
- - Logs aren't actually useful, /because/ most operations are idempotent.
- Thus, scratch logfile and make our report files more useful: error.log
- needs error information; we don't care too much about machinability.
- All report files should be overwritten on the next run, since we like
- using --limit to incrementally increase the number of things we run. Note
- that if we add soft ignores, you /do/ lose information, so there needs
- to be some way to also have the soft ignore report a "cached error"
- - Report the identifier number at the beginning of all of the stdout logs
- - Don't really care about having the name in the logfile name, but
- have a lookup txt file
+ - Certain classes of error will continually fail, so they should
+ put in a different "seen" file which also skips them, unless
+ we have some sort of gentle force. These are "soft ignores".
+ - If we add soft ignores, you lose information from reports, so there needs
+ to be some way to also have the soft ignore report a "cached error". This
+ probably means augmenting the serialized set to be a serialized dict.
- Figure out a way of collecting blacklist data from .scripts/blacklisted
and aggregate it together
- Failed migrations should be wired to have wizard commands in them
default=False, help="Performs the operation without actually modifying any files. Use in combination with --verbose to see commands that will be run.")
# common variables for mass commands
baton.add("--seen", dest="seen",
- default=None, help="File to read/write paths of already processed installs."
- "These will be skipped.")
+ default=None, help="File to read/write paths of successfully modified installs;"
+ "these will be skipped on re-runs. If --log-dir is specified, this is automatically enabled.")
baton.add("--no-parallelize", dest="no_parallelize", action="store_true",
default=False, help="Turn off parallelization")
baton.add("--max-processes", dest="max_processes", type="int", metavar="N",
return False
return True
-def calculate_log_name(log_dir, i, dir):
+def calculate_log_name(log_dir, i):
"""
- Calculates a log entry given a log directory, numeric identifier, and
+ Calculates a log entry given a numeric identifier, and
directory under operation.
"""
- return os.path.join(log_dir, "%04d" % i + dir.replace('/', '-') + ".log")
+ return os.path.join(log_dir, "%04d.log" % i)
-def open_logs(log_dir, log_names=('warnings', 'errors')):
+def create_logdir(log_dir):
"""
- Opens a number of log files for auxiliary reporting. You can override what
- log files to generate using ``log_names``, which corresponds to the tuple
- of log files you will receive, i.e. the default returns a tuple
- ``(warnings.log file object, errors.log file object)``.
-
- .. note::
-
- The log directory is chmod'ed 777 after creation, to enable
- de-priviledged processes to create files.
+ Creates a log directory and chmods it 777 to enable de-priviledged
+ processes to create files.
"""
- # must not be on AFS, since subprocesses won't be
- # able to write to the logfiles do the to the AFS patch.
try:
os.mkdir(log_dir)
except OSError as e:
# os.mkdir(log_dir) # if fails, be fatal
# # XXX: update last symlink
os.chmod(log_dir, 0o777)
- return (open(os.path.join(os.path.join(log_dir, "%s.log" % x)), "a") for x in log_names)
+
+def open_reports(log_dir, names=('warnings', 'errors')):
+ """
+ Opens a number of reports files for auxiliary reporting. You can override what
+ log files to generate using ``names``, which corresponds to the tuple
+ of report files you will receive, i.e. the default returns a tuple
+ ``(warnings.txt file object, errors.txt file object)``. Note that this will
+ delete any information that was previously in the file (but those logfiles
+ are backed up).
+ """
+ # must not be on AFS, since subprocesses won't be
+ # able to write to the logfiles do the to the AFS patch.
+ files = [os.path.join(os.path.join(log_dir, "%s.txt" % x)) for x in names]
+ for f in files:
+ util.safe_unlink(f)
+ return (open(f, "w") for f in files)
class NullLogHandler(logging.Handler):
"""Log handler that doesn't do anything"""
self.add_option_group(group)
options, numeric_args = self.parse_args(argv)
makeLogger(options, numeric_args)
+ # we're going to process the global --log-dir/--seen dependency here
+ if hasattr(options, "seen") and hasattr(options, "log_dir"):
+ if not options.seen and options.log_dir:
+ options.seen = os.path.join(options.log_dir, "seen.txt")
return options, numeric_args
class OptionBaton(object):
sh = shell.ParallelShell.make(options.no_parallelize, options.max_processes)
seen = sset.make(options.seen)
is_root = not os.getuid()
- warnings_log, errors_log = command.open_logs(options.log_dir)
+ command.create_logdir(options.log_dir)
+ warnings_report, errors_report = command.open_reports(options.log_dir)
# loop stuff
errors = {}
i = 0
child_args = list(base_args)
# calculate the log file, if a log dir was specified
if options.log_dir:
- log_file = command.calculate_log_name(options.log_dir, i, d.location)
+ log_file = command.calculate_log_name(options.log_dir, i)
child_args.append("--log-file=" + log_file)
# actual meat
def make_on_pair(d, i):
# we need to make another stack frame so that d and i get specific bindings.
def on_success(stdout, stderr):
if stderr:
- warnings_log.write("%s\n" % d.location)
+ warnings_report.write("%s\n" % d.location)
logging.warning("Warnings [%04d] %s:\n%s" % (i, d.location, stderr))
seen.add(d.location)
def on_error(e):
if name not in errors: errors[name] = []
errors[name].append(d)
logging.error("%s in [%04d] %s" % (name, i, d.location))
- errors_log.write("%s\n" % d.location)
+ errors_report.write("%s\n" % d.location)
return (on_success, on_error)
on_success, on_error = make_on_pair(d, i)
sh.call("wizard", "migrate", d.location, *child_args,
sh = shell.ParallelShell.make(options.no_parallelize, options.max_processes)
seen = sset.make(options.seen)
is_root = not os.getuid()
- warnings_log, errors_log, merge_log = command.open_logs(options.log_dir, ('warnings', 'errors', 'merge'))
+ command.create_logdir(options.log_dir)
+ lookup_report, warnings_report, errors_report, merge_report, verify_report = command.open_reports(options.log_dir, ('lookup', 'warnings', 'errors', 'merge', 'verify'))
# loop stuff
errors = {}
i = 0
- merge_fails = [0] # otherwise I get a UnboundLocalError later on when I increment
+ fails = {
+ "merge": 0,
+ "verify": 0,
+ }
deploys = deploy.parse_install_lines(app, options.versions_path, user=options.user)
requested_deploys = itertools.islice(deploys, options.limit)
for i, d in enumerate(requested_deploys, 1):
+ lookup_report.write("%04d %s\n" % (i, d.location))
# check if we want to punt due to --limit
if d.location in seen:
continue
if is_root and not command.security_check_homedir(d.location):
continue
- # XXX: we may be able to punt based on detected versions from d
- logging.info("Processing %s" % d.location)
+ # XXX: we may be able to punt based on detected versions from d, which
+ # would be faster than spinning up a new process. On the other hand,
+ # `seen` makes this mostly not a problem
+ logging.info("[%04d] Processing %s" % (i, d.location))
child_args = list(base_args)
# calculate the log file, if a log dir was specified
if options.log_dir:
- log_file = command.calculate_log_name(options.log_dir, i, d.location)
+ log_file = command.calculate_log_name(options.log_dir, i)
child_args.append("--log-file=" + log_file)
# actual meat
def make_on_pair(d, i):
# we need to make another stack frame so that d and i get specific bindings.
def on_success(stdout, stderr):
if stderr:
- warnings_log.write("%s\n" % d.location)
- logging.warning("Warnings [%04d] %s:\n%s" % (i, d.location, stderr))
+ warnings_report.write("[%04d] %s\n" % (i, d.location))
+ logging.warning("[%04d] Warnings at [%s]:\n%s" % (i, d.location, stderr))
seen.add(d.location)
def on_error(e):
- if e.name == "wizard.command.upgrade.AlreadyUpgraded" or \
- e.name == "AlreadyUpgraded":
+ if e.name == "AlreadyUpgraded":
seen.add(d.location)
- logging.info("Skipped already upgraded %s" % d.location)
+ logging.info("[%04d] Skipped already upgraded %s" % (i, d.location))
elif e.name == "MergeFailed":
seen.add(d.location)
- logging.warning("Merge failed: resolve at [%s], source at [%s]" % (e.stdout.rstrip(), d.location))
- merge_log.write("%s\n" % d.location)
- merge_fails[0] += 1
+ tmpdir = e.stdout.rstrip()
+ logging.warning("[%04d] Merge failed: resolve at [%s], source at [%s]" % (i, tmpdir, d.location))
+ merge_report.write("[%04d] %s %s\n" % (i, tmpdir, d.location))
+ fails['merge'] += 1
else:
name = e.name
if name not in errors: errors[name] = []
url = d.location
# This should actually be a warning, but
# it's a really common error
- logging.info("Could not verify application at %s" % url)
+ logging.info("[%04d] Could not verify application at %s" % (i, url))
+ verify_report.write("[%04d] %s\n" % (i, url))
+ fails['verify'] += 1
else:
- logging.error("%s in [%04d] %s" % (name, i, d.location))
- errors_log.write("%s\n" % d.location)
+ msg = "[%04d] %s in %s" % (i, name, d.location)
+ logging.error(msg)
+ errors_report.write(msg + "\n")
return (on_success, on_error)
on_success, on_error = make_on_pair(d, i)
sh.call("wizard", "upgrade", d.location, *child_args,
sh.join()
for name, deploys in errors.items():
logging.warning("%s from %d installs" % (name, len(deploys)))
- if merge_fails[0]:
- logging.warning("%d out of %d installs (%.1f%%) had merge failure" % (merge_fails[0], i, float(merge_fails[0])/i*100))
+ def printPercent(description, number, total):
+ return "%d out of %d installs (%.1f%%) had %s" % (number, total, float(number)/total*100, description)
+ if fails['merge']:
+ printPercent("merge conflicts", fails['merge'], i)
+ if fails['verify']:
+ printPercent("web verification failure", fails['verify'], i)
def parse_args(argv, baton):
usage = """usage: %prog mass-upgrade [ARGS] APPLICATION
def safe_unlink(file):
"""Moves a file/dir to a backup location."""
+ if not os.path.exists(file):
+ return None
prefix = "%s.bak" % file
name = None
for i in itertools.count():