wizard/command/upgrade.py

   1 import sys
   2 import distutils.version
   3 import os
   4 import shutil
   5 import logging.handlers
   6 import tempfile
   7 import itertools
   8 import time
   9
  10 from wizard import app, command, deploy, scripts, shell, util
  11
  12 kib_buffer = 1024 * 30 # 30 MiB we will always leave available
  13 errno_blacklisted = 64
  14
  15 def main(argv, baton):
  16     options, args = parse_args(argv, baton)
  17     if args:
  18         dir = args[0]
  19     else:
  20         dir = "."
  21     shell.drop_priviledges(dir, options.log_file)
  22     util.set_git_env()
  23     upgrade = Upgrade(options)
  24     upgrade.execute(dir)
  25     if not options.non_interactive:
  26         print "Upgrade complete"
  27
  28 class Upgrade(object):
  29     """
  30     Represents the algorithm for upgrading an application.  This is in
  31     a class and not a function because it's a multi-step process that
  32     requires state betweens steps.  Steps are represented as methods
  33     in this object.
  34     """
  35
  36     #: Version of application we are upgrading to, i.e. the latest version.
  37     version = None # XXX: This is a string... I'm not convinced it should be
  38     #: String commit ID of the user's latest wc; i.e. "ours"
  39     user_commit = None
  40     #: String commit ID of the latest, greatest scripts version; i.e. "theirs"
  41     next_commit = None
  42     #: The temporary directory that the system gave us; may stay as ``None``
  43     #: if we don't ever make ourselves a temporary directory (e.g. ``--continue``).
  44     #: While we should clean this up if it is set to something, it may
  45     #: not correspond to anything useful.
  46     temp_dir = None
  47     #: The temporary directory containing our working copy for merging
  48     temp_wc_dir = None
  49     #: We place the temporary repositories inside a tmpfs while merging;
  50     #: this makes merges not disk-bound and affords a modest speed increase.
  51     #: If you are running ``--continue``, this is guaranteed to be ``False``.
  52     use_shm = None
  53     #: Upstream repository to use.  This does not need to be saved.
  54     repo = None
  55
  56     #: Instance of :class:`wizard.deploy.WorkingCopy` for this upgrade
  57     wc = None
  58     #: Instance of :class:`wizard.deploy.ProductionCopy` for this upgrade
  59     prod = None
  60
  61     #: Options object that the installer was called with
  62     options = None
  63
  64     def __init__(self, options):
  65         self.version = None
  66         self.user_commit = None
  67         self.next_commit = None
  68         self.temp_dir = None
  69         self.temp_wc_dir = None
  70         self.use_shm = False # False until proven otherwise.
  71         self.wc = None
  72         self.prod = None
  73         self.options = options
  74
  75     def execute(self, dir):
  76         """
  77         Executes an upgrade.  This is the entry-point.
  78         """
  79         with util.ChangeDirectory(dir):
  80             try:
  81                 if self.options.continue_:
  82                     logging.info("Continuing upgrade...")
  83                     self.resume()
  84                 else:
  85                     logging.info("Upgrading %s" % os.getcwd())
  86                     self.preflight()
  87                     self.merge()
  88                 self.postflight()
  89                 # Till now, all of our operations were in a tmp sandbox.
  90                 if self.options.dry_run:
  91                     logging.info("Dry run, bailing.  See results at %s" % self.temp_wc_dir)
  92                     return
  93                 backup = self.backup()
  94                 self.upgrade(backup)
  95             finally:
  96                 if self.use_shm and self.temp_dir and os.path.exists(self.temp_dir):
  97                     shutil.rmtree(self.temp_dir)
  98
  99     def resume(self):
 100         """
 101         In the event of a ``--continue`` flag, we have to restore state and
 102         perform some sanity checks.
 103         """
 104         self.resumeState()
 105         self.resumeLogging()
 106         util.chdir(shell.eval("git", "config", "remote.origin.url"))
 107         self.resumeProd()
 108     def resumeState(self):
 109         self.temp_wc_dir = os.getcwd()
 110         self.wc = deploy.WorkingCopy(".")
 111         self.user_commit, self.next_commit = open(".git/WIZARD_PARENTS", "r").read().split()
 112         self.version = open(".git/WIZARD_UPGRADE_VERSION", "r").read()
 113     def resumeLogging(self):
 114         options = self.options
 115         if not options.log_file and os.path.exists(".git/WIZARD_LOG_FILE"):
 116             options.log_file = open(".git/WIZARD_LOG_FILE", "r").read()
 117             command.setup_file_logger(options.log_file, options.debug)
 118     def resumeProd(self):
 119         """Restore :attr:`prod` attribute, and check if the production copy has drifted."""
 120         self.prod = deploy.ProductionCopy(".")
 121         try:
 122             shell.call("git", "status")
 123             raise LocalChangesError()
 124         except shell.CallError:
 125             pass
 126
 127     def preflight(self):
 128         """
 129         Make sure that a number of pre-upgrade invariants are met before
 130         attempting anything.
 131         """
 132         options = self.options
 133         self.prod = deploy.ProductionCopy(".")
 134         self.repo = self.prod.application.repository(options.srv_path)
 135         # XXX: put this in Application
 136         self.version = shell.eval("git", "--git-dir="+self.repo, "describe", "--tags", "master")
 137         self.preflightBlacklist()
 138         self.prod.verify()
 139         self.prod.verifyTag(options.srv_path)
 140         self.prod.verifyGit(options.srv_path)
 141         self.prod.verifyConfigured()
 142         shell.call("git", "fetch", "--tags") # XXX: hack since some installs have stale tags
 143         self.prod.verifyVersion()
 144         self.prod.verifyWeb()
 145         self.preflightAlreadyUpgraded()
 146         self.preflightQuota()
 147     def preflightBlacklist(self):
 148         if os.path.exists(".scripts/blacklisted"):
 149             reason = open(".scripts/blacklisted").read()
 150             # ignore blank blacklisted files
 151             if reason:
 152                 print reason
 153                 raise BlacklistedError(reason)
 154             else:
 155                 logging.warning("Application was blacklisted, but no reason was found");
 156     def preflightAlreadyUpgraded(self):
 157         if self.version == self.prod.app_version.scripts_tag and not self.options.force:
 158             # don't log this error; we need to have the traceback line
 159             # so that the parsing code can catch it
 160             # XXX: maybe we should build this in as a flag to add
 161             # to exceptions w/ our exception handler
 162             sys.stderr.write("Traceback:\n  (n/a)\nAlreadyUpgraded\n")
 163             sys.exit(1)
 164     def preflightQuota(self):
 165         kib_usage, kib_limit = scripts.get_quota_usage_and_limit()
 166         if kib_limit is not None and (kib_limit - kib_usage) < kib_buffer:
 167             raise QuotaTooLow
 168
 169     def merge(self):
 170         if not self.options.dry_run:
 171             self.mergePreCommit()
 172         self.mergeClone()
 173         logging.debug("Temporary WC dir is %s", self.temp_wc_dir)
 174         with util.ChangeDirectory(self.temp_wc_dir):
 175             self.wc = deploy.WorkingCopy(".")
 176             shell.call("git", "remote", "add", "scripts", self.repo)
 177             shell.call("git", "fetch", "-q", "scripts")
 178             self.user_commit = shell.eval("git", "rev-parse", "HEAD")
 179             self.next_commit = shell.eval("git", "rev-parse", self.version)
 180             self.mergeSaveState()
 181             self.mergePerform()
 182     def mergePreCommit(self):
 183         message = "Pre-commit of %s locker before autoinstall upgrade.\n\n%s" % (util.get_dir_owner(), util.get_git_footer())
 184         try:
 185             message += "\nPre-commit-by: " + util.get_operator_git()
 186         except util.NoOperatorInfo:
 187             pass
 188         try:
 189             shell.call("git", "commit", "-a", "-m", message)
 190         except shell.CallError:
 191             logging.info("No changes detected")
 192     def mergeClone(self):
 193         # If /dev/shm exists, it's a tmpfs and we can use it
 194         # to do a fast git merge. Don't forget to move it to
 195         # /tmp if it fails.
 196         if not self.options.dry_run and not self.options.debug:
 197             self.use_shm = os.path.exists("/dev/shm")
 198         if self.use_shm:
 199             dir = "/dev/shm/wizard"
 200             if not os.path.exists(dir):
 201                 os.mkdir(dir)
 202                 # XXX: race
 203                 os.chmod(dir, 0o777)
 204         else:
 205             dir = None
 206         self.temp_dir = tempfile.mkdtemp(prefix="wizard", dir=dir)
 207         self.temp_wc_dir = os.path.join(self.temp_dir, "repo")
 208         logging.info("Using temporary directory: " + self.temp_wc_dir)
 209         shell.call("git", "clone", "-q", "--shared", ".", self.temp_wc_dir)
 210     def mergeSaveState(self):
 211         """Save variables so that ``--continue`` will work."""
 212         # yeah yeah no trailing newline whatever
 213         open(".git/WIZARD_UPGRADE_VERSION", "w").write(self.version)
 214         open(".git/WIZARD_PARENTS", "w").write("%s\n%s" % (self.user_commit, self.next_commit))
 215         open(".git/WIZARD_SIZE", "w").write(str(scripts.get_disk_usage()))
 216         if self.options.log_file:
 217             open(".git/WIZARD_LOG_FILE", "w").write(self.options.log_file)
 218     def mergePerform(self):
 219         def make_virtual_commit(rev, parents = []):
 220             """
 221             Takes a revision and generates a "virtual" commit with
 222             user-specific variables instantiated for a smooth, easy
 223             merge.
 224
 225             .. warning::
 226
 227                 Changes the state of the working copy.
 228             """
 229             shell.call("git", "checkout", "-q", rev, "--")
 230             self.wc.parametrize(self.prod)
 231             for file in self.wc.application.parametrized_files:
 232                 try:
 233                     shell.call("git", "add", "--", file)
 234                 except shell.CallError:
 235                     pass
 236             virtual_tree = shell.eval("git", "write-tree", log=True)
 237             parent_args = itertools.chain(*(["-p", p] for p in parents))
 238             virtual_commit = shell.eval("git", "commit-tree", virtual_tree,
 239                     *parent_args, input="", log=True)
 240             shell.call("git", "reset", "--hard")
 241             return virtual_commit
 242         user_tree = shell.eval("git", "rev-parse", "HEAD^{tree}")
 243         base_virtual_commit = make_virtual_commit(self.wc.app_version.scripts_tag)
 244         next_virtual_commit = make_virtual_commit(self.version, [base_virtual_commit])
 245         user_virtual_commit = shell.eval("git", "commit-tree", user_tree,
 246                 "-p", base_virtual_commit, input="", log=True)
 247         shell.call("git", "checkout", user_virtual_commit, "--")
 248         self.wc.prepareMerge()
 249         try:
 250             shell.call("git", "commit", "--amend", "-a", "-m", "amendment")
 251         except shell.CallError as e:
 252             pass
 253         shell.call("git", "config", "merge.conflictstyle", "diff3")
 254         try:
 255             shell.call("git", "merge", next_virtual_commit)
 256         except shell.CallError as e:
 257             logging.info("Merge failed with these messages:\n\n" + e.stderr)
 258             # Run the application's specific merge resolution algorithms
 259             # and see if we can salvage it
 260             if self.wc.resolveConflicts():
 261                 logging.info("Resolved conflicts with application specific knowledge")
 262                 shell.call("git", "commit", "-a", "-m", "merge")
 263                 return
 264             files = set()
 265             for line in shell.eval("git", "ls-files", "--unmerged").splitlines():
 266                 files.add(line.split(None, 3)[-1])
 267             conflicts = len(files)
 268             # XXX: this is kind of fiddly; note that temp_dir still points at the OLD
 269             # location after this code.
 270             self.temp_wc_dir = mv_shm_to_tmp(os.getcwd(), self.use_shm)
 271             self.wc.location = self.temp_wc_dir
 272             os.chdir(self.temp_wc_dir)
 273             open(os.path.join(self.prod.location, ".scripts/pending"), "w").write(self.temp_wc_dir)
 274             if self.options.non_interactive:
 275                 print "%d %s" % (conflicts, self.temp_wc_dir)
 276                 raise MergeFailed
 277             else:
 278                 user_shell = os.getenv("SHELL")
 279                 if not user_shell: user_shell = "/bin/bash"
 280                 # XXX: scripts specific hack, since mbash doesn't respect the current working directory
 281                 # When the revolution comes (i.e. $ATHENA_HOMEDIR/Scripts is your Scripts home
 282                 # directory) this isn't strictly necessary, but we'll probably need to support
 283                 # web_scripts directories ad infinitum.
 284                 if user_shell == "/usr/local/bin/mbash": user_shell = "/bin/bash"
 285                 while 1:
 286                     print
 287                     print "ERROR: The merge failed with %d conflicts in these files:" % conflicts
 288                     print
 289                     for file in sorted(files):
 290                         print "  * %s" % file
 291                     print
 292                     print "Please resolve these conflicts (edit and then `git add`), and"
 293                     print "then type 'exit'.  You will now be dropped into a shell whose working"
 294                     print "directory is %s" % self.temp_wc_dir
 295                     shell.call(user_shell, "-i", interactive=True)
 296                     if shell.eval("git", "ls-files", "--unmerged").strip():
 297                         print
 298                         print "WARNING: There are still unmerged files."
 299                         out = raw_input("Continue editing? [y/N]: ")
 300                         if out == "y" or out == "Y":
 301                             continue
 302                         else:
 303                             print "Aborting.  The conflicted working copy can be found at:"
 304                             print
 305                             print "    %s" % self.temp_wc_dir
 306                             print
 307                             print "and you can resume the upgrade process by running in that directory:"
 308                             print
 309                             print "    wizard upgrade --continue"
 310                             sys.exit(1)
 311                     break
 312
 313     def postflight(self):
 314         with util.ChangeDirectory(self.temp_wc_dir):
 315             try:
 316                 shell.call("git", "status")
 317             except shell.CallError:
 318                 pass
 319             else:
 320                 shell.call("git", "commit", "--allow-empty", "-am", "throw-away commit")
 321             message = self.postflightCommitMessage()
 322             new_tree = shell.eval("git", "rev-parse", "HEAD^{tree}")
 323             final_commit = shell.eval("git", "commit-tree", new_tree,
 324                     "-p", self.user_commit, "-p", self.next_commit, input=message, log=True)
 325             # a master branch may not necessarily exist if the user
 326             # was manually installed to an earlier version
 327             try:
 328                 shell.call("git", "checkout", "-q", "-b", "master", "--")
 329             except shell.CallError:
 330                 shell.call("git", "checkout", "-q", "master", "--")
 331             shell.call("git", "reset", "-q", "--hard", final_commit)
 332             # This is a quick sanity check to make sure we didn't completely
 333             # mess up the merge
 334             self.wc.invalidateCache()
 335             self.wc.verifyVersion()
 336     def postflightCommitMessage(self):
 337         message = "Upgraded autoinstall in %s to %s.\n\n%s" % (util.get_dir_owner(), self.version, util.get_git_footer())
 338         try:
 339             message += "\nUpgraded-by: " + util.get_operator_git()
 340         except util.NoOperatorInfo:
 341             pass
 342         return message
 343
 344     def backup(self):
 345         # Ok, now we have to do a crazy complicated dance to see if we're
 346         # going to have enough quota to finish what we need
 347         pre_size = int(open(os.path.join(self.temp_wc_dir, ".git/WIZARD_SIZE"), "r").read())
 348         post_size = scripts.get_disk_usage(self.temp_wc_dir)
 349         backup = self.prod.backup(self.options)
 350         kib_usage, kib_limit = scripts.get_quota_usage_and_limit()
 351         if kib_limit is not None and (kib_limit - kib_usage) - (post_size - pre_size) / 1024 < kib_buffer:
 352             shutil.rmtree(os.path.join(".scripts/backups", shell.eval("wizard", "restore").splitlines()[0]))
 353             raise QuotaTooLow
 354         return backup
 355
 356     def upgrade(self, backup):
 357         # XXX: frob .htaccess to make site inaccessible
 358         with util.IgnoreKeyboardInterrupts():
 359             with util.LockDirectory(".scripts-upgrade-lock"):
 360                 shell.call("git", "fetch", "--tags")
 361                 # git merge (which performs a fast forward)
 362                 shell.call("git", "pull", "-q", self.temp_wc_dir, "master")
 363                 version_obj = distutils.version.LooseVersion(self.version.partition('-')[2])
 364                 try:
 365                     # run update script
 366                     self.prod.upgrade(version_obj, self.options)
 367                     self.prod.verifyWeb()
 368                 except app.UpgradeFailure:
 369                     logging.warning("Upgrade failed: rolling back")
 370                     self.upgradeRollback(backup)
 371                     raise
 372                 except deploy.WebVerificationError as e:
 373                     logging.warning("Web verification failed: rolling back")
 374                     self.upgradeRollback(backup)
 375                     raise app.UpgradeVerificationFailure()
 376         # XXX: frob .htaccess to make site accessible
 377         #       to do this, check if .htaccess changed, first.  Upgrade
 378         #       process might have frobbed it.  Don't be
 379         #       particularly worried if the segment disappeared
 380     def upgradeRollback(self, backup):
 381         # You don't want d.restore() because it doesn't perform
 382         # the file level backup
 383         shell.call("wizard", "restore", backup)
 384         try:
 385             self.prod.verifyWeb()
 386         except deploy.WebVerificationError:
 387             logging.critical("Web verification failed after rollback")
 388
 389 # utility functions
 390
 391 def mv_shm_to_tmp(curdir, use_shm):
 392     if not use_shm: return curdir
 393     # Keeping all of our autoinstalls in shared memory is
 394     # a recipe for disaster, so let's move them to slightly
 395     # less volatile storage (a temporary directory)
 396     os.chdir(tempfile.gettempdir())
 397     newdir = tempfile.mkdtemp(prefix="wizard")
 398     # shutil, not os; at least on Ubuntu os.move fails
 399     # with "[Errno 18] Invalid cross-device link"
 400     shutil.move(curdir, newdir)
 401     shutil.rmtree(os.path.dirname(curdir))
 402     curdir = os.path.join(newdir, "repo")
 403     return curdir
 404
 405 def parse_args(argv, baton):
 406     usage = """usage: %prog upgrade [ARGS] [DIR]
 407
 408 Upgrades an autoinstall to the latest version.  This involves
 409 updating files and running .scripts/update.  If the merge fails,
 410 this program will write the number of conflicts and the directory
 411 of the conflicted working tree to stdout, separated by a space."""
 412     parser = command.WizardOptionParser(usage)
 413     parser.add_option("--dry-run", dest="dry_run", action="store_true",
 414             default=False, help="Prints would would be run without changing anything")
 415     # notice trailing underscore
 416     parser.add_option("--continue", dest="continue_", action="store_true",
 417             default=False, help="Continues an upgrade that has had its merge manually "
 418             "resolved using the current working directory as the resolved copy.")
 419     parser.add_option("--force", dest="force", action="store_true",
 420             default=False, help="Force running upgrade even if it's already at latest version.")
 421     parser.add_option("--non-interactive", dest="non_interactive", action="store_true",
 422             default=False, help="Don't drop to shell in event of conflict.")
 423     baton.push(parser, "srv_path")
 424     options, args = parser.parse_all(argv)
 425     if len(args) > 1:
 426         parser.error("too many arguments")
 427     return options, args
 428
 429 class Error(command.Error):
 430     """Base exception for all exceptions raised by upgrade"""
 431     pass
 432
 433 class QuotaTooLow(Error):
 434     def __str__(self):
 435         return """
 436
 437 ERROR: The locker quota was too low to complete the autoinstall
 438 upgrade.
 439 """
 440
 441 class AlreadyUpgraded(Error):
 442     quiet = True
 443     def __str__(self):
 444         return """
 445
 446 ERROR: This autoinstall is already at the latest version."""
 447
 448 class MergeFailed(Error):
 449     quiet = True
 450     def __str__(self):
 451         return """
 452
 453 ERROR: Merge failed.  Above is the temporary directory that
 454 the conflicted merge is in: resolve the merge by cd'ing to the
 455 temporary directory, finding conflicted files with `git status`,
 456 resolving the files, adding them using `git add` and then
 457 running `wizard upgrade --continue`."""
 458
 459 class LocalChangesError(Error):
 460     def __str__(self):
 461         return """
 462
 463 ERROR: Local changes occurred in the install while the merge was
 464 being processed so that a pull would not result in a fast-forward.
 465 The best way to resolve this is probably to attempt an upgrade again,
 466 with git rerere to remember merge resolutions (XXX: not sure if
 467 this actually works)."""
 468
 469 class BlacklistedError(Error):
 470     #: Reason why the autoinstall was blacklisted
 471     reason = None
 472     exitcode = errno_blacklisted
 473     def __init__(self, reason):
 474         self.reason = reason
 475     def __str__(self):
 476         return """
 477
 478 ERROR: This autoinstall was manually blacklisted against errors;
 479 if the user has not been notified of this, please send them
 480 mail.
 481
 482 The reason was: %s""" % self.reason