wizard/command/upgrade.py

   1 import optparse
   2 import sys
   3 import distutils.version
   4 import os
   5 import shutil
   6 import logging.handlers
   7 import errno
   8 import tempfile
   9 import itertools
  10
  11 from wizard import app, command, deploy, shell, util
  12
  13 def main(argv, baton):
  14     options, args = parse_args(argv, baton)
  15     if args:
  16         dir = args[0]
  17     else:
  18         dir = "."
  19     shell.drop_priviledges(dir, options.log_file)
  20     util.chdir(dir)
  21     if os.path.exists(".scripts/blacklisted"):
  22         raise BlacklistedError()
  23     sh = shell.Shell()
  24     util.set_git_env()
  25     use_shm = False # if you are running --continue, this is guaranteed to be False
  26     if options.continue_:
  27         temp_wc_dir = os.getcwd()
  28         user_commit, next_commit = open(".git/WIZARD_PARENTS", "r").read().split()
  29         repo = open(".git/WIZARD_REPO", "r").read()
  30         version = open(".git/WIZARD_UPGRADE_VERSION", "r").read()
  31         util.chdir(sh.eval("git", "config", "remote.origin.url"))
  32         d = deploy.Deployment(".")
  33         try:
  34             sh.call("git", "status")
  35             raise LocalChangesError()
  36         except shell.CallError:
  37             pass
  38     else:
  39         d = deploy.Deployment(".")
  40         d.verify()
  41         d.verifyTag(options.srv_path)
  42         d.verifyGit(options.srv_path)
  43         d.verifyConfigured()
  44         d.verifyVersion()
  45         if not options.dry_run:
  46             d.verifyWeb()
  47         repo = d.application.repository(options.srv_path)
  48         version = calculate_newest_version(sh, repo)
  49         if version == d.app_version.scripts_tag and not options.force:
  50             # don't log this error
  51             # XXX: maybe we should build this in as a flag to add
  52             # to exceptions w/ our exception handler
  53             sys.stderr.write("Traceback:\n  (n/a)\nAlreadyUpgraded\n")
  54             sys.exit(1)
  55         logging.info("Upgrading %s" % os.getcwd())
  56         if not options.dry_run:
  57             perform_pre_commit(sh)
  58         # If /dev/shm exists, it's a tmpfs and we can use it
  59         # to do a fast git merge. Don't forget to move it to
  60         # /tmp if it fails.
  61         # XXX: git merge-tree is another possibility for
  62         # reducing filesystem interactions, and also probably
  63         # works better with extra merging functionality.  However,
  64         # I don't know how I would put the results back in the
  65         # working tree.
  66         use_shm = os.path.exists("/dev/shm")
  67         temp_dir, temp_wc_dir = perform_tmp_clone(sh, use_shm)
  68         with util.ChangeDirectory(temp_wc_dir):
  69             sh.call("git", "remote", "add", "scripts", repo)
  70             sh.call("git", "fetch", "-q", "scripts")
  71             user_commit, next_commit = calculate_parents(sh, version)
  72             # save variables so that --continue will work
  73             # yeah yeah no trailing newline whatever
  74             open(".git/WIZARD_REPO", "w").write(repo)
  75             open(".git/WIZARD_UPGRADE_VERSION", "w").write(version)
  76             open(".git/WIZARD_PARENTS", "w").write("%s\n%s" % (user_commit, next_commit))
  77             perform_merge(sh, repo, d, version, use_shm)
  78     # variables: version, user_commit, next_commit, temp_wc_dir
  79     with util.ChangeDirectory(temp_wc_dir):
  80         try:
  81             sh.call("git", "status")
  82             sh.call("git", "commit", "-m", "throw-away commit")
  83         except shell.CallError:
  84             pass
  85         message = make_commit_message(version)
  86         new_tree = sh.eval("git", "rev-parse", "HEAD^{tree}")
  87         final_commit = sh.eval("git", "commit-tree", new_tree,
  88                 "-p", user_commit, "-p", next_commit, input=message, log=True)
  89         # a master branch may not necessarily exist if the user
  90         # was manually installed to an earlier version
  91         try:
  92             sh.call("git", "checkout", "-q", "-b", "master", "--")
  93         except shell.CallError:
  94             sh.call("git", "checkout", "-q", "master", "--")
  95         sh.call("git", "reset", "-q", "--hard", final_commit)
  96         # This is a quick sanity check to make sure we didn't completely
  97         # mess up the merge
  98         d.verifyVersion()
  99     # Till now, all of our operations were in a tmp sandbox.
 100     if options.dry_run:
 101         logging.info("Dry run, bailing.  See results at %s" % temp_wc_dir)
 102         return
 103     # perform database backup
 104     backup = d.backup(options)
 105     # XXX: frob .htaccess to make site inaccessible
 106     with util.IgnoreKeyboardInterrupts():
 107         with util.LockDirectory(".scripts-upgrade-lock"):
 108             # git merge (which performs a fast forward)
 109             sh.call("git", "pull", "-q", temp_wc_dir, "master")
 110             # after the pull is successful, the directory now
 111             # has the objects for this commit, so we can safely
 112             # nuke the shm directory.  We refrain from nuking the
 113             # tmp directory in case we messed up the merge resolution
 114             # and want to be able to use it again.
 115             if use_shm:
 116                 shutil.rmtree(temp_dir)
 117             version_obj = distutils.version.LooseVersion(version.partition('-')[2])
 118             try:
 119                 # run update script
 120                 d.upgrade(version_obj, options)
 121                 d.verifyWeb()
 122             except app.UpgradeFailure:
 123                 logging.warning("Upgrade failed: rolling back")
 124                 perform_restore(d, backup)
 125                 raise
 126             except deploy.WebVerificationError as e:
 127                 logging.warning("Web verification failed: rolling back")
 128                 logging.info("Web page that was output was:\n\n%s" % e.contents)
 129                 perform_restore(d, backup)
 130                 raise app.UpgradeVerificationFailure("Upgrade caused website to become inaccessible; site was rolled back")
 131     # XXX: frob .htaccess to make site accessible
 132     #       to do this, check if .htaccess changed, first.  Upgrade
 133     #       process might have frobbed it.  Don't be
 134     #       particularly worried if the segment disappeared
 135
 136 def perform_restore(d, backup):
 137     # You don't want d.restore() because it doesn't perform
 138     # the file level backup
 139     shell.Shell().call("wizard", "restore", backup)
 140     try:
 141         d.verifyWeb()
 142     except deploy.WebVerificationError:
 143         logging.critical("Web verification failed after rollback")
 144
 145 def make_commit_message(version):
 146     message = "Upgraded autoinstall in %s to %s.\n\n%s" % (util.get_dir_owner(), version, util.get_git_footer())
 147     try:
 148         message += "\nUpgraded-by: " + util.get_operator_git()
 149     except util.NoOperatorInfo:
 150         pass
 151     return message
 152
 153 def calculate_newest_version(sh, repo):
 154     # XXX: put this in Application
 155     return sh.eval("git", "--git-dir="+repo, "describe", "--tags", "master")
 156
 157 def calculate_parents(sh, version):
 158     user_commit = sh.eval("git", "rev-parse", "HEAD")
 159     next_commit = sh.eval("git", "rev-parse", version)
 160     return user_commit, next_commit
 161
 162 def perform_pre_commit(sh):
 163     message = "Pre-commit of %s locker before autoinstall upgrade.\n\n%s" % (util.get_dir_owner(), util.get_git_footer())
 164     try:
 165         message += "\nPre-commit-by: " + util.get_operator_git()
 166     except util.NoOperatorInfo:
 167         pass
 168     try:
 169         sh.call("git", "commit", "-a", "-m", message)
 170     except shell.CallError:
 171         logging.info("No changes detected")
 172
 173 def perform_tmp_clone(sh, use_shm):
 174     if use_shm:
 175         dir = "/dev/shm/wizard"
 176         if not os.path.exists(dir):
 177             os.mkdir(dir)
 178     else:
 179         dir = None
 180     temp_dir = tempfile.mkdtemp(prefix="wizard", dir=dir)
 181     temp_wc_dir = os.path.join(temp_dir, "repo")
 182     logging.info("Using temporary directory: " + temp_wc_dir)
 183     sh.call("git", "clone", "-q", "--shared", ".", temp_wc_dir)
 184     return temp_dir, temp_wc_dir
 185
 186 def perform_merge(sh, repo, d, version, use_shm):
 187     # naive merge algorithm:
 188     # sh.call("git", "merge", "-m", message, "scripts/master")
 189     # crazy merge algorithm:
 190     def make_virtual_commit(tag, parents = []):
 191         """WARNING: Changes state of Git repository"""
 192         sh.call("git", "checkout", "-q", tag, "--")
 193         d.parametrize(".")
 194         for file in d.application.parametrized_files:
 195             try:
 196                 sh.call("git", "add", "--", file)
 197             except shell.CallError:
 198                 pass
 199         virtual_tree = sh.eval("git", "write-tree", log=True)
 200         parent_args = itertools.chain(*(["-p", p] for p in parents))
 201         virtual_commit = sh.eval("git", "commit-tree", virtual_tree,
 202                 *parent_args, input="", log=True)
 203         sh.call("git", "reset", "--hard")
 204         return virtual_commit
 205     user_tree = sh.eval("git", "rev-parse", "HEAD^{tree}")
 206     base_virtual_commit = make_virtual_commit(d.app_version.scripts_tag)
 207     next_virtual_commit = make_virtual_commit(version, [base_virtual_commit])
 208     user_virtual_commit = sh.eval("git", "commit-tree", user_tree,
 209             "-p", base_virtual_commit, input="", log=True)
 210     sh.call("git", "checkout", user_virtual_commit, "--")
 211     try:
 212         sh.call("git", "merge", next_virtual_commit)
 213     except shell.CallError:
 214         # Run the application's specific merge resolution algorithms
 215         # and see if we can salvage it
 216         curdir = os.getcwd()
 217         if d.application.resolveConflicts(curdir):
 218             logging.info("Resolved conflicts with application specific knowledge")
 219             sh.call("git", "commit", "-a", "-m", "merge")
 220             return
 221         logging.info("Conflict info:\n" + sh.eval("git", "diff"))
 222         if use_shm:
 223             # Keeping all of our autoinstalls in shared memory is
 224             # a recipe for disaster, so let's move them to slightly
 225             # less volatile storage (a temporary directory)
 226             os.chdir(tempfile.gettempdir())
 227             newdir = tempfile.mkdtemp(prefix="wizard")
 228             # shutil, not os; at least on Ubuntu os.move fails
 229             # with "[Errno 18] Invalid cross-device link"
 230             shutil.move(curdir, newdir)
 231             shutil.rmtree(os.path.dirname(curdir))
 232             curdir = os.path.join(newdir, "repo")
 233         print curdir
 234         raise MergeFailed
 235
 236 def parse_args(argv, baton):
 237     usage = """usage: %prog upgrade [ARGS] [DIR]
 238
 239 Upgrades an autoinstall to the latest version.  This involves
 240 updating files and running .scripts/update.
 241
 242 WARNING: This is still experimental."""
 243     parser = command.WizardOptionParser(usage)
 244     parser.add_option("--dry-run", dest="dry_run", action="store_true",
 245             default=False, help="Prints would would be run without changing anything")
 246     # notice trailing underscore
 247     parser.add_option("--continue", dest="continue_", action="store_true",
 248             default=False, help="Continues an upgrade that has had its merge manually "
 249             "resolved using the current working directory as the resolved copy.")
 250     parser.add_option("--force", dest="force", action="store_true",
 251             default=False, help="Force running upgrade even if it's already at latest version.")
 252     baton.push(parser, "srv_path")
 253     options, args = parser.parse_all(argv)
 254     if len(args) > 1:
 255         parser.error("too many arguments")
 256     return options, args
 257
 258 class Error(command.Error):
 259     """Base exception for all exceptions raised by upgrade"""
 260     pass
 261
 262 class AlreadyUpgraded(Error):
 263     def __str__(self):
 264         return """
 265
 266 ERROR: This autoinstall is already at the latest version."""
 267
 268 class MergeFailed(Error):
 269     def __str__(self):
 270         return """
 271
 272 ERROR: Merge failed.  Resolve the merge by cd'ing to the
 273 temporary directory, finding conflicted files with `git status`,
 274 resolving the files, adding them using `git add` and then
 275 running `wizard upgrade --continue`."""
 276
 277 class LocalChangesError(Error):
 278     def __str__(self):
 279         return """
 280
 281 ERROR: Local changes occurred in the install while the merge was
 282 being processed so that a pull would not result in a fast-forward.
 283 The best way to resolve this is probably to attempt an upgrade again,
 284 with git rerere to remember merge resolutions (XXX: not sure if
 285 this actually works)."""
 286
 287 class BlacklistedError(Error):
 288     def __str__(self):
 289         return """
 290
 291 ERROR: This autoinstall was manually blacklisted against errors;
 292 if the user has not been notified of this, please send them
 293 mail."""