X-Git-Url: https://scripts.mit.edu/gitweb/wizard.git/blobdiff_plain/1d9e7be9725182abb9d4f59ff315b961dffed9f8..6554c6378a6e801b4fe47c50688cdae1d627bc18:/wizard/command/upgrade.py diff --git a/wizard/command/upgrade.py b/wizard/command/upgrade.py index 475ddd7..39c73c9 100644 --- a/wizard/command/upgrade.py +++ b/wizard/command/upgrade.py @@ -1,6 +1,7 @@ import sys import distutils.version import os +import os.path import shutil import logging.handlers import tempfile @@ -8,17 +9,15 @@ import itertools import time import errno -from wizard import app, command, deploy, scripts, shell, util +from wizard import app, command, deploy, merge, scripts, shell, util kib_buffer = 1024 * 30 # 30 MiB we will always leave available errno_blacklisted = 64 def main(argv, baton): options, args = parse_args(argv, baton) - if args: - dir = args[0] - else: - dir = "." + dir = os.path.abspath(args[0]) if args else os.getcwd() + os.chdir(dir) shell.drop_priviledges(dir, options.log_file) util.set_git_env() upgrade = Upgrade(options) @@ -75,27 +74,28 @@ class Upgrade(object): def execute(self, dir): """ - Executes an upgrade. This is the entry-point. + Executes an upgrade. This is the entry-point. This expects + that it's current working directory is the same as ``dir``. """ - with util.ChangeDirectory(dir): - try: - if self.options.continue_: - logging.info("Continuing upgrade...") - self.resume() - else: - logging.info("Upgrading %s" % os.getcwd()) - self.preflight() - self.merge() - self.postflight() - # Till now, all of our operations were in a tmp sandbox. - if self.options.dry_run: - logging.info("Dry run, bailing. See results at %s" % self.temp_wc_dir) - return - backup = self.backup() - self.upgrade(backup) - finally: - if self.use_shm and self.temp_dir and os.path.exists(self.temp_dir): - shutil.rmtree(self.temp_dir) + assert os.path.abspath(dir) == os.getcwd() + try: + if self.options.continue_: + logging.info("Continuing upgrade...") + self.resume() + else: + logging.info("Upgrading %s" % os.getcwd()) + self.preflight() + self.merge() + self.postflight() + # Till now, all of our operations were in a tmp sandbox. + if self.options.dry_run: + logging.info("Dry run, bailing. See results at %s" % self.temp_wc_dir) + return + backup = self.backup() + self.upgrade(backup) + finally: + if self.use_shm and self.temp_dir and os.path.exists(self.temp_dir): + shutil.rmtree(self.temp_dir) def resume(self): """ @@ -140,6 +140,9 @@ class Upgrade(object): raise LocalChangesError() except shell.CallError: pass + # Working copy is not anchored anywhere useful for git describe, + # so we need to give it a hint. + self.wc.setAppVersion(self.prod.app_version) def preflight(self): """ @@ -147,17 +150,34 @@ class Upgrade(object): attempting anything. """ options = self.options - self.prod = deploy.ProductionCopy(".") - self.repo = self.prod.application.repository(options.srv_path) - # XXX: put this in Application - self.version = shell.eval("git", "--git-dir="+self.repo, "describe", "--tags", "master") - self.preflightBlacklist() - self.prod.verify() - self.prod.verifyTag(options.srv_path) - self.prod.verifyGit(options.srv_path) - self.prod.verifyConfigured() - shell.call("git", "fetch", "--tags") # XXX: hack since some installs have stale tags - self.prod.verifyVersion() + for i in range(0,2): + self.prod = deploy.ProductionCopy(".") + self.prod.verify() + self.repo = self.prod.application.repository(options.srv_path) + # XXX: put this in Application + self.version = shell.eval("git", "--git-dir="+self.repo, "describe", "--tags", "master") + self.preflightBlacklist() + self.prod.verify() + self.prod.verifyDatabase() + self.prod.verifyTag(options.srv_path) + self.prod.verifyGit(options.srv_path) + self.prod.verifyConfigured() + try: + shell.call("git", "fetch", "--tags") # XXX: hack since some installs have stale tags + except shell.CallError as e: + if "Disk quota exceeded" in e.stderr: + raise QuotaTooLow + raise + try: + self.prod.verifyVersion() + except deploy.VersionMismatchError as e: + # XXX: kind of hacky, mainly it does change the Git working copy + # state (although /very/ non-destructively) + shell.call("git", "merge", "--strategy=ours", self.prod.application.makeVersion(str(e.real_version)).scripts_tag) + continue + break + else: + raise VersionRematchFailed self.prod.verifyWeb() self.preflightAlreadyUpgraded() self.preflightQuota() @@ -197,6 +217,16 @@ class Upgrade(object): self.mergeSaveState() self.mergePerform() def mergePreCommit(self): + def get_file_set(rev): + return set(shell.eval("git", "ls-tree", "-r", "--name-only", rev).split("\n")) + # add all files that are unversioned but would be replaced by the pull, + # and generate a new commit + old_files = get_file_set("HEAD") + new_files = get_file_set(self.version) + added_files = new_files - old_files + for f in added_files: + if os.path.exists(f): + shell.call("git", "add", f) message = "Pre-commit of %s locker before autoinstall upgrade.\n\n%s" % (util.get_dir_owner(), util.get_git_footer()) try: message += "\nPre-commit-by: " + util.get_operator_git() @@ -204,7 +234,11 @@ class Upgrade(object): pass try: shell.call("git", "commit", "-a", "-m", message) - except shell.CallError: + except shell.CallError as e: + if "Permission denied" in e.stderr: + raise util.PermissionsError + elif e.stderr: + raise logging.info("No changes detected") def mergeClone(self): # If /dev/shm exists, it's a tmpfs and we can use it @@ -233,113 +267,86 @@ class Upgrade(object): if self.options.log_file: open(".git/WIZARD_LOG_FILE", "w").write(self.options.log_file) def mergePerform(self): - def make_virtual_commit(rev, parents = []): - """ - Takes a revision and generates a "virtual" commit with - user-specific variables instantiated for a smooth, easy - merge. - - .. warning:: - - Changes the state of the working copy. - """ - shell.call("git", "checkout", "-q", rev, "--") - self.wc.parametrize(self.prod) - for file in self.wc.application.parametrized_files: - try: - shell.call("git", "add", "--", file) - except shell.CallError: - pass - virtual_tree = shell.eval("git", "write-tree", log=True) - parent_args = itertools.chain(*(["-p", p] for p in parents)) - virtual_commit = shell.eval("git", "commit-tree", virtual_tree, - *parent_args, input="", log=True) - shell.call("git", "reset", "--hard") - return virtual_commit - user_tree = shell.eval("git", "rev-parse", "HEAD^{tree}") - base_virtual_commit = make_virtual_commit(self.wc.app_version.scripts_tag) - next_virtual_commit = make_virtual_commit(self.version, [base_virtual_commit]) - user_virtual_commit = shell.eval("git", "commit-tree", user_tree, - "-p", base_virtual_commit, input="", log=True) - shell.call("git", "checkout", user_virtual_commit, "--") - self.wc.prepareMerge() - try: - shell.call("git", "commit", "--amend", "-a", "-m", "amendment") - except shell.CallError as e: - pass + def prepare_config(): + self.wc.prepareConfig() + shell.call("git", "add", ".") + def resolve_conflicts(): + return self.wc.resolveConflicts() shell.call("git", "config", "merge.conflictstyle", "diff3") + # setup rerere + if self.options.rr_cache is None: + self.options.rr_cache = os.path.join(self.prod.location, ".git", "rr-cache") + if not os.path.exists(self.options.rr_cache): + os.mkdir(self.options.rr_cache) + os.symlink(self.options.rr_cache, os.path.join(self.wc.location, ".git", "rr-cache")) + shell.call("git", "config", "rerere.enabled", "true") try: - shell.call("git", "merge", next_virtual_commit) - except shell.CallError as e: - logging.info("Merge failed with these messages:\n\n" + e.stderr) - # Run the application's specific merge resolution algorithms - # and see if we can salvage it - if self.wc.resolveConflicts(): - logging.info("Resolved conflicts with application specific knowledge") - shell.call("git", "commit", "-a", "-m", "merge") - return - files = set() - for line in shell.eval("git", "ls-files", "--unmerged").splitlines(): - files.add(line.split(None, 3)[-1]) - conflicts = len(files) - # XXX: this is kind of fiddly; note that temp_dir still points at the OLD - # location after this code. - self.temp_wc_dir = mv_shm_to_tmp(os.getcwd(), self.use_shm) - self.wc.location = self.temp_wc_dir - os.chdir(self.temp_wc_dir) - open(os.path.join(self.prod.location, ".scripts/pending"), "w").write(self.temp_wc_dir) - if self.options.non_interactive: - print "%d %s" % (conflicts, self.temp_wc_dir) - raise MergeFailed - else: - user_shell = os.getenv("SHELL") - if not user_shell: user_shell = "/bin/bash" - # XXX: scripts specific hack, since mbash doesn't respect the current working directory - # When the revolution comes (i.e. $ATHENA_HOMEDIR/Scripts is your Scripts home - # directory) this isn't strictly necessary, but we'll probably need to support - # web_scripts directories ad infinitum. - if user_shell == "/usr/local/bin/mbash": user_shell = "/bin/bash" - while 1: - print - print "ERROR: The merge failed with %d conflicts in these files:" % conflicts - print - for file in sorted(files): - print " * %s" % file + merge.merge(self.wc.app_version.scripts_tag, self.version, + prepare_config, resolve_conflicts) + except merge.MergeError: + self.mergeFail() + def mergeFail(self): + files = set() + for line in shell.eval("git", "ls-files", "--unmerged").splitlines(): + files.add(line.split(None, 3)[-1]) + conflicts = len(files) + # XXX: this is kind of fiddly; note that temp_dir still points at the OLD + # location after this code. + self.temp_wc_dir = mv_shm_to_tmp(os.getcwd(), self.use_shm) + self.wc.location = self.temp_wc_dir + os.chdir(self.temp_wc_dir) + open(os.path.join(self.prod.location, ".scripts/pending"), "w").write(self.temp_wc_dir) + if self.options.non_interactive: + print "%d %s" % (conflicts, self.temp_wc_dir) + raise MergeFailed + else: + user_shell = os.getenv("SHELL") + if not user_shell: user_shell = "/bin/bash" + # XXX: scripts specific hack, since mbash doesn't respect the current working directory + # When the revolution comes (i.e. $ATHENA_HOMEDIR/Scripts is your Scripts home + # directory) this isn't strictly necessary, but we'll probably need to support + # web_scripts directories ad infinitum. + if user_shell == "/usr/local/bin/mbash": user_shell = "/bin/bash" + while 1: + print + print "ERROR: The merge failed with %d conflicts in these files:" % conflicts + print + for file in sorted(files): + print " * %s" % file + print + print "Please resolve these conflicts (edit and then `git add`), and" + print "then type 'exit'. You will now be dropped into a shell whose working" + print "directory is %s" % self.temp_wc_dir + try: + shell.call(user_shell, "-i", interactive=True) + except shell.CallError as e: + logging.warning("Shell returned non-zero exit code %d" % e.code) + if shell.eval("git", "ls-files", "--unmerged").strip(): print - print "Please resolve these conflicts (edit and then `git add`), and" - print "then type 'exit'. You will now be dropped into a shell whose working" - print "directory is %s" % self.temp_wc_dir - try: - shell.call(user_shell, "-i", interactive=True) - except shell.CallError as e: - logging.warning("Shell returned non-zero exit code %d" % e.code) - if shell.eval("git", "ls-files", "--unmerged").strip(): + print "WARNING: There are still unmerged files." + out = raw_input("Continue editing? [y/N]: ") + if out == "y" or out == "Y": + continue + else: + print "Aborting. The conflicted working copy can be found at:" print - print "WARNING: There are still unmerged files." - out = raw_input("Continue editing? [y/N]: ") - if out == "y" or out == "Y": - continue - else: - print "Aborting. The conflicted working copy can be found at:" - print - print " %s" % self.temp_wc_dir - print - print "and you can resume the upgrade process by running in that directory:" - print - print " wizard upgrade --continue" - sys.exit(1) - break + print " %s" % self.temp_wc_dir + print + print "and you can resume the upgrade process by running in that directory:" + print + print " wizard upgrade --continue" + sys.exit(1) + break def postflight(self): with util.ChangeDirectory(self.temp_wc_dir): - try: - shell.call("git", "status") - except shell.CallError: - pass - else: - shell.call("git", "commit", "--allow-empty", "-am", "throw-away commit") + if shell.eval("git", "ls-files", "-u").strip(): + raise UnmergedChangesError + shell.call("git", "commit", "--allow-empty", "-am", "throw-away commit") + self.wc.parametrize(self.prod) + shell.call("git", "add", ".") message = self.postflightCommitMessage() - new_tree = shell.eval("git", "rev-parse", "HEAD^{tree}") + new_tree = shell.eval("git", "write-tree") final_commit = shell.eval("git", "commit-tree", new_tree, "-p", self.user_commit, "-p", self.next_commit, input=message, log=True) # a master branch may not necessarily exist if the user @@ -440,6 +447,9 @@ of the conflicted working tree to stdout, separated by a space.""" default=False, help="Force running upgrade even if it's already at latest version.") parser.add_option("--non-interactive", dest="non_interactive", action="store_true", default=False, help="Don't drop to shell in event of conflict.") + parser.add_option("--rr-cache", dest="rr_cache", metavar="PATH", + default=None, help="Use this folder to reuse recorded merge resolutions. Defaults to" + "your production copy's rr-cache, if it exists.") baton.push(parser, "srv_path") options, args = parser.parse_all(argv) if len(args) > 1: @@ -486,6 +496,14 @@ The best way to resolve this is probably to attempt an upgrade again, with git rerere to remember merge resolutions (XXX: not sure if this actually works).""" +class UnmergedChangesError(Error): + def __str__(self): + return """ + +ERROR: You attempted to continue an upgrade, but there were +still local unmerged changes in your working copy. Please resolve +them all and try again.""" + class BlacklistedError(Error): #: Reason why the autoinstall was blacklisted reason = None @@ -509,3 +527,12 @@ ERROR: We cannot resume the upgrade process; either this working copy is missing essential metadata, or you've attempt to continue from a production copy that does not have any pending upgrades. """ + +class VersionRematchFailed(Error): + def __str__(self): + return """ + +ERROR: Your Git version information was not consistent with your +files on the system, and we were unable to create a fake merge +to make the two consistent. Please contact scripts@mit.edu. +"""