2 import distutils.version
6 import logging.handlers
12 # XXX: read().strip() is an accident waiting to happen if anyone ever
13 # makes a directory with a trailing or leading space. Unlikely, but yeah.
15 from wizard import app, command, deploy, merge, shell, user, util
17 buffer = 1024 * 1024 * 30 # 30 MiB we will always leave available
18 errno_blacklisted = 64
20 def main(argv, baton):
21 options, args = parse_args(argv, baton)
22 dir = os.path.abspath(args[0]) if args else os.getcwd()
24 shell.drop_priviledges(dir, options.log_file)
26 upgrade = Upgrade(options)
28 if not options.non_interactive:
29 print "Upgrade complete"
31 class Upgrade(object):
33 Represents the algorithm for upgrading an application. This is in
34 a class and not a function because it's a multi-step process that
35 requires state betweens steps. Steps are represented as methods
39 #: Version of application we are upgrading to, i.e. the latest version.
40 version = None # XXX: This is a string... I'm not convinced it should be
41 #: String commit ID of the user's latest wc; i.e. "ours"
43 #: String commit ID of the latest, greatest wizard version; i.e. "theirs"
45 #: The temporary directory that the system gave us; may stay as ``None``
46 #: if we don't ever make ourselves a temporary directory (e.g. ``--continue``).
47 #: While we should clean this up if it is set to something, it may
48 #: not correspond to anything useful.
50 #: The temporary directory containing our working copy for merging
52 #: We place the temporary repositories inside a tmpfs while merging;
53 #: this makes merges not disk-bound and affords a modest speed increase.
54 #: If you are running ``--continue``, this is guaranteed to be ``False``.
56 #: Upstream repository to use. This does not need to be saved.
59 #: Instance of :class:`wizard.deploy.WorkingCopy` for this upgrade
61 #: Instance of :class:`wizard.deploy.ProductionCopy` for this upgrade
64 #: Options object that the installer was called with
67 def __init__(self, options):
69 self.user_commit = None
70 self.next_commit = None
72 self.temp_wc_dir = None
73 self.use_shm = False # False until proven otherwise.
76 self.options = options
78 def execute(self, dir):
80 Executes an upgrade. This is the entry-point. This expects
81 that it's current working directory is the same as ``dir``.
83 assert os.path.abspath(dir) == os.getcwd()
85 if self.options.continue_:
86 logging.info("Continuing upgrade...")
89 logging.info("Upgrading %s" % os.getcwd())
92 # Note invariant: we expect you to be in the production
93 # directory at this point, even if you --continue'd from
94 # the temporary directory!
96 # Till now, all of our operations were in a tmp sandbox.
97 if self.options.dry_run:
98 logging.info("Dry run, bailing. See results at %s" % self.temp_wc_dir)
100 backup = self.backup()
102 # Note: disable_rollback assumes that upgrade is the last
103 # step, if you add another setp you may have to modify this
104 # to accomodate that.
106 if self.use_shm and self.temp_dir and os.path.exists(self.temp_dir):
107 shutil.rmtree(self.temp_dir)
111 In the event of a ``--continue`` flag, we have to restore state and
112 perform some sanity checks.
118 def resumeChdir(self):
120 If we called ``--continue`` inside a working copy (the usual
121 situation), check if :file:`.wizard/pending` exists and change
122 to that directory if so.
124 self.temp_wc_dir = os.getcwd()
125 didChdir = command.chdir_to_production()
126 self.prod = deploy.ProductionCopy(".")
127 pending_dir = open(self.prod.pending_file).read().strip()
129 logging.warning("Continued from a production copy; using working copy at %s", pending_dir)
130 self.temp_wc_dir = pending_dir
131 elif self.temp_wc_dir != pending_dir:
132 # prefer the original working copy, but warn that someone
133 # else someone a started an upgrade in the meantime (XXX:
134 # actually, that someone else should bug out, not clobber)
135 logging.warning("Someone else appears to have started an upgrade at %s", pending_dir)
136 def resumeState(self):
137 with util.ChangeDirectory(self.temp_wc_dir):
138 self.wc = deploy.WorkingCopy(".")
140 self.user_commit, self.next_commit = open(".git/WIZARD_PARENTS", "r").read().split()
141 self.version = open(".git/WIZARD_UPGRADE_VERSION", "r").read()
143 if e.errno == errno.ENOENT:
144 raise CannotResumeError()
147 def resumeLogging(self):
148 with util.ChangeDirectory(self.temp_wc_dir):
149 options = self.options
150 if not options.log_file and os.path.exists(".git/WIZARD_LOG_FILE"):
151 options.log_file = open(".git/WIZARD_LOG_FILE", "r").read()
152 command.setup_file_logger(options.log_file, options.debug)
153 def resumeProd(self):
154 """Restore :attr:`prod` attribute, and check if the production copy has drifted."""
156 # simulate the action of `git status`, based on cmd_status()'s call to
157 # refresh_cache() in builtin-commit.c
158 shell.call("git", "update-index", "-q", "--unmerged", "--refresh")
159 r1 = shell.eval("git", "diff-files", "--name-only").strip()
160 r2 = shell.eval("git", "diff-index", "--name-only", "HEAD").strip()
162 raise LocalChangesError()
163 except shell.CallError:
165 # Working copy is not anchored anywhere useful for git describe,
166 # so we need to give it a hint.
167 self.wc.setAppVersion(self.prod.app_version)
171 Make sure that a number of pre-upgrade invariants are met before
174 options = self.options
176 self.prod = deploy.ProductionCopy(".")
178 self.repo = self.prod.application.repository(options.srv_path)
179 # XXX: put this in Application
180 self.version = shell.eval("git", "--git-dir="+self.repo, "describe", "--tags", "master")
181 self.preflightBlacklist()
183 self.prod.verifyDatabase()
184 self.prod.verifyTag(options.srv_path)
185 self.prod.verifyGit(options.srv_path)
186 if not options.skip_verification:
187 self.prod.verifyConfigured()
189 shell.call("git", "fetch", "--tags") # XXX: hack since some installs have stale tags
190 except shell.CallError as e:
191 if "Disk quota exceeded" in e.stderr:
195 self.prod.verifyVersion()
196 except deploy.VersionMismatchError as e:
197 # XXX: kind of hacky, mainly it does change the Git working copy
198 # state (although /very/ non-destructively)
200 shell.call("git", "merge", "--strategy=ours", self.prod.application.makeVersion(str(e.real_version)).wizard_tag)
201 except shell.CallError as e2:
202 if "does not point to a commit" in e2.stderr:
203 raise UnknownVersionError(e.real_version)
209 raise VersionRematchFailed
211 self.preflightAlreadyUpgraded()
212 self.preflightQuota()
213 def preflightBlacklist(self):
214 # XXX: should use deploy info
215 if os.path.exists(".wizard/blacklisted"):
216 reason = open(".wizard/blacklisted").read()
217 # ignore blank blacklisted files
220 raise BlacklistedError(reason)
222 logging.warning("Application was blacklisted, but no reason was found");
223 def preflightAlreadyUpgraded(self):
224 if self.version == self.prod.app_version.wizard_tag and not self.options.force:
225 # don't log this error; we need to have the traceback line
226 # so that the parsing code can catch it
227 # XXX: maybe we should build this in as a flag to add
228 # to exceptions w/ our exception handler
229 sys.stderr.write("Traceback:\n (n/a)\nAlreadyUpgraded\n")
231 def preflightQuota(self):
235 if limit is not None and (limit - usage) < buffer:
236 logging.info("preflightQuota: limit = %d, usage = %d, buffer = %d", limit, usage, buffer)
240 if not self.options.dry_run:
241 self.mergePreCommit()
243 logging.debug("Temporary WC dir is %s", self.temp_wc_dir)
244 with util.ChangeDirectory(self.temp_wc_dir):
245 self.wc = deploy.WorkingCopy(".")
246 shell.call("git", "remote", "add", "wizard", self.repo)
247 shell.call("git", "fetch", "-q", "wizard")
248 self.user_commit = shell.eval("git", "rev-parse", "HEAD")
249 self.next_commit = shell.eval("git", "rev-parse", self.version)
250 self.mergeSaveState()
252 def mergePreCommit(self):
253 def get_file_set(rev):
254 return set(shell.eval("git", "ls-tree", "-r", "--name-only", rev).split("\n"))
255 # add all files that are unversioned but would be replaced by the pull,
256 # and generate a new commit
257 old_files = get_file_set("HEAD")
258 new_files = get_file_set(self.version)
259 added_files = new_files - old_files
260 for f in added_files:
261 if os.path.lexists(f): # broken symbolic links count too!
262 shell.call("git", "add", f)
263 message = "Pre-commit before autoinstall upgrade.\n\n%s" % util.get_git_footer()
265 message += "\nPre-commit-by: " + util.get_operator_git()
266 except util.NoOperatorInfo:
269 shell.call("git", "commit", "-a", "-m", message)
270 except shell.CallError as e:
271 if "Permission denied" in e.stderr:
272 raise util.PermissionsError
275 logging.info("No changes detected")
276 def mergeClone(self):
277 # If /dev/shm exists, it's a tmpfs and we can use it
278 # to do a fast git merge. Don't forget to move it to
280 if not self.options.dry_run and not self.options.debug:
281 self.use_shm = os.path.exists("/dev/shm")
283 dir = "/dev/shm/wizard"
284 if not os.path.exists(dir):
290 self.temp_dir = tempfile.mkdtemp(prefix="wizard", dir=dir)
291 self.temp_wc_dir = os.path.join(self.temp_dir, "repo")
292 logging.info("Using temporary directory: " + self.temp_wc_dir)
293 shell.call("git", "clone", "-q", "--shared", ".", self.temp_wc_dir)
294 def mergeSaveState(self):
295 """Save variables so that ``--continue`` will work."""
296 # yeah yeah no trailing newline whatever
297 open(".git/WIZARD_UPGRADE_VERSION", "w").write(self.version)
298 open(".git/WIZARD_PARENTS", "w").write("%s\n%s" % (self.user_commit, self.next_commit))
299 open(".git/WIZARD_SIZE", "w").write(str(util.disk_usage()))
300 if self.options.log_file:
301 open(".git/WIZARD_LOG_FILE", "w").write(self.options.log_file)
302 def mergePerform(self):
303 def prepare_config():
304 self.wc.prepareConfig()
305 shell.call("git", "add", ".")
306 def resolve_conflicts():
307 return self.wc.resolveConflicts()
308 shell.call("git", "config", "merge.conflictstyle", "diff3")
310 if self.options.rr_cache is None:
311 self.options.rr_cache = os.path.join(self.prod.location, ".git", "rr-cache")
312 if not os.path.exists(self.options.rr_cache):
313 os.mkdir(self.options.rr_cache)
314 os.symlink(self.options.rr_cache, os.path.join(self.wc.location, ".git", "rr-cache"))
315 shell.call("git", "config", "rerere.enabled", "true")
317 merge.merge(self.wc.app_version.wizard_tag, self.version,
318 prepare_config, resolve_conflicts)
319 except merge.MergeError:
323 for line in shell.eval("git", "ls-files", "--unmerged").splitlines():
324 files.add(line.split(None, 3)[-1])
325 conflicts = len(files)
326 # XXX: this is kind of fiddly; note that temp_dir still points at the OLD
327 # location after this code.
328 self.temp_wc_dir = mv_shm_to_tmp(os.getcwd(), self.use_shm)
329 self.wc.location = self.temp_wc_dir
330 os.chdir(self.temp_wc_dir)
331 if os.path.exists(self.prod.pending_file):
332 mtime = os.path.getmtime(self.prod.pending_file)
333 pending_location = open(self.prod.pending_file).read().strip()
334 # don't complain if .wizard/pending is a day old
335 if mtime > (time.time() - 60 * 60 * 24):
336 raise UpgradeInProgressError(pending_location, mtime)
338 logging.warning("Probably harmless old pending upgrade at %s from %s", pending_location, time.ctime(mtime))
339 open(self.prod.pending_file, "w").write(self.temp_wc_dir)
340 if self.options.non_interactive:
341 print "%d %s" % (conflicts, self.temp_wc_dir)
344 user_shell = os.getenv("SHELL")
345 if not user_shell: user_shell = "/bin/bash"
346 # XXX: scripts specific hack, since mbash doesn't respect the current working directory
347 # When the revolution comes (i.e. $ATHENA_HOMEDIR/Scripts is your Scripts home
348 # directory) this isn't strictly necessary, but we'll probably need to support
349 # web_scripts directories ad infinitum.
350 if user_shell == "/usr/local/bin/mbash": user_shell = "/bin/bash"
353 print "ERROR: The merge failed with %d conflicts in these files:" % conflicts
355 for file in sorted(files):
358 print "Please resolve these conflicts (edit and then `git add`), and"
359 print "then type 'exit'. You will now be dropped into a shell whose working"
360 print "directory is %s" % self.temp_wc_dir
362 print "NOTE: If you resolve these conflicts, and then the upgrade fails for"
363 print "an unrelated reason, you can run 'wizard upgrade --continue' from this"
364 print "directory to try again."
366 shell.call(user_shell, "-i", interactive=True)
367 except shell.CallError as e:
368 logging.warning("Shell returned non-zero exit code %d" % e.code)
369 if shell.eval("git", "ls-files", "--unmerged").strip():
371 print "WARNING: There are still unmerged files."
372 out = raw_input("Continue editing? [y/N]: ")
373 if out == "y" or out == "Y":
376 print "Aborting. The conflicted working copy can be found at:"
378 print " %s" % self.temp_wc_dir
380 print "and you can resume the upgrade process by running in that directory:"
382 print " wizard upgrade --continue"
386 def postflight(self):
387 with util.ChangeDirectory(self.temp_wc_dir):
388 if shell.eval("git", "ls-files", "-u").strip():
389 raise UnmergedChangesError
390 shell.call("git", "commit", "--allow-empty", "-am", "throw-away commit")
391 self.wc.parametrize(self.prod)
392 shell.call("git", "add", ".")
393 message = self.postflightCommitMessage()
394 new_tree = shell.eval("git", "write-tree")
395 final_commit = shell.eval("git", "commit-tree", new_tree,
396 "-p", self.user_commit, "-p", self.next_commit, input=message, log=True)
397 # a master branch may not necessarily exist if the user
398 # was manually installed to an earlier version
400 shell.call("git", "checkout", "-q", "-b", "master", "--")
401 except shell.CallError:
402 shell.call("git", "checkout", "-q", "master", "--")
403 shell.call("git", "reset", "-q", "--hard", final_commit)
404 # This is a quick sanity check to make sure we didn't completely
406 self.wc.invalidateCache()
407 self.wc.verifyVersion()
408 def postflightCommitMessage(self):
409 message = "Upgraded autoinstall to %s.\n\n%s" % (self.version, util.get_git_footer())
411 message += "\nUpgraded-by: " + util.get_operator_git()
412 except util.NoOperatorInfo:
417 # Ok, now we have to do a crazy complicated dance to see if we're
418 # going to have enough quota to finish what we need
419 pre_size = int(open(os.path.join(self.temp_wc_dir, ".git/WIZARD_SIZE"), "r").read())
420 post_size = util.disk_usage(self.temp_wc_dir)
421 backup = self.prod.backup(self.options)
425 if limit is not None and (limit - usage) - (post_size - pre_size) < buffer:
426 shutil.rmtree(os.path.join(self.prod.backup_dir, shell.eval("wizard", "restore").splitlines()[0]))
430 def upgrade(self, backup):
431 # XXX: frob .htaccess to make site inaccessible
432 # XXX: frob Git to disallow Git operations after the pull
433 with util.IgnoreKeyboardInterrupts():
434 with util.LockDirectory(".wizard-upgrade-lock"):
435 shell.call("git", "fetch", "--tags")
436 # git merge (which performs a fast forward)
437 shell.call("git", "pull", "-q", self.temp_wc_dir, "master")
438 version_obj = distutils.version.LooseVersion(self.version.partition('-')[2])
441 self.prod.upgrade(version_obj, self.options)
444 os.unlink(self.prod.pending_file)
446 if e.errno != errno.ENOENT:
448 except app.UpgradeFailure:
449 logging.warning("Upgrade failed: rolling back")
450 self.upgradeRollback(backup)
452 except deploy.WebVerificationError as e:
453 logging.warning("Web verification failed: rolling back")
454 self.upgradeRollback(backup)
455 raise app.UpgradeVerificationFailure()
456 # XXX: frob .htaccess to make site accessible
457 # to do this, check if .htaccess changed, first. Upgrade
458 # process might have frobbed it. Don't be
459 # particularly worried if the segment disappeared
460 def upgradeRollback(self, backup):
461 # You don't want d.restore() because it doesn't perform
462 # the file level backup
463 if not self.options.disable_rollback:
464 shell.call("wizard", "restore", backup)
467 except deploy.WebVerificationError:
468 logging.critical("Web verification failed after rollback")
470 logging.warning("Rollback was disabled; you can rollback with `wizard restore %s`", backup)
473 if not self.options.skip_verification:
474 self.prod.verifyWeb()
479 def mv_shm_to_tmp(curdir, use_shm):
480 if not use_shm: return curdir
481 # Keeping all of our autoinstalls in shared memory is
482 # a recipe for disaster, so let's move them to slightly
483 # less volatile storage (a temporary directory)
484 os.chdir(tempfile.gettempdir())
485 newdir = tempfile.mkdtemp(prefix="wizard")
486 # shutil, not os; at least on Ubuntu os.move fails
487 # with "[Errno 18] Invalid cross-device link"
488 shutil.move(curdir, newdir)
489 shutil.rmtree(os.path.dirname(curdir))
490 curdir = os.path.join(newdir, "repo")
493 def parse_args(argv, baton):
494 usage = """usage: %prog upgrade [ARGS] [DIR]
496 Upgrades an autoinstall to the latest version. This involves updating
497 files the upgrade script associated with this application. If the merge
498 fails, this program will write the number of conflicts and the directory
499 of the conflicted working tree to stdout, separated by a space."""
500 parser = command.WizardOptionParser(usage)
501 parser.add_option("--dry-run", dest="dry_run", action="store_true",
502 default=False, help="Prints would would be run without changing anything")
503 # notice trailing underscore
504 parser.add_option("--continue", dest="continue_", action="store_true",
505 default=False, help="Continues an upgrade that has had its merge manually "
506 "resolved using the current working directory as the resolved copy.")
507 parser.add_option("--force", dest="force", action="store_true",
508 default=False, help="Force running upgrade even if it's already at latest version.")
509 parser.add_option("--skip-verification", dest="skip_verification", action="store_true",
510 default=False, help="Skip running configuration and web verification checks.")
511 parser.add_option("--non-interactive", dest="non_interactive", action="store_true",
512 default=False, help="Don't drop to shell in event of conflict.")
513 parser.add_option("--rr-cache", dest="rr_cache", metavar="PATH",
514 default=None, help="Use this folder to reuse recorded merge resolutions. Defaults to"
515 "your production copy's rr-cache, if it exists.")
516 parser.add_option("--disable-rollback", dest="disable_rollback", action="store_true",
517 default=util.boolish(os.getenv("WIZARD_DISABLE_ROLLBACK")),
518 help="Skips rollback in the event of a failed upgrade. Envvar is WIZARD_DISABLE_ROLLBACK.")
519 baton.push(parser, "srv_path")
520 options, args = parser.parse_all(argv)
522 parser.error("too many arguments")
523 if options.skip_verification:
524 logging.warning("Verification is disabled; Wizard may break your application and will not tell you about it")
527 class Error(command.Error):
528 """Base exception for all exceptions raised by upgrade"""
531 class QuotaTooLow(Error):
535 ERROR: The locker quota was too low to complete the autoinstall
539 class AlreadyUpgraded(Error):
544 ERROR: This autoinstall is already at the latest version."""
546 class MergeFailed(Error):
551 ERROR: Merge failed. Above is the temporary directory that
552 the conflicted merge is in: resolve the merge by cd'ing to the
553 temporary directory, finding conflicted files with `git status`,
554 resolving the files, adding them using `git add` and then
555 running `wizard upgrade --continue`."""
557 class LocalChangesError(Error):
561 ERROR: Local changes occurred in the install while the merge was
562 being processed so that a pull would not result in a fast-forward.
563 The best way to resolve this is probably to attempt an upgrade again,
564 with git rerere to remember merge resolutions (XXX: not sure if
565 this actually works)."""
567 class UnmergedChangesError(Error):
571 ERROR: You attempted to continue an upgrade, but there were
572 still local unmerged changes in your working copy. Please resolve
573 them all and try again."""
575 class BlacklistedError(Error):
576 #: Reason why the autoinstall was blacklisted
578 exitcode = errno_blacklisted
579 def __init__(self, reason):
584 ERROR: This autoinstall was manually blacklisted against errors;
585 if the user has not been notified of this, please send them
586 mail. If you know that this application is blacklisted and
587 would like to attempt an upgrade anyway, run:
589 wizard blacklist --delete
591 The reason was: %s""" % self.reason
593 class CannotResumeError(Error):
597 ERROR: We cannot resume the upgrade process; either this working
598 copy is missing essential metadata, or you've attempt to continue
599 from a production copy that does not have any pending upgrades.
602 class VersionRematchFailed(Error):
606 ERROR: Your Git version information was not consistent with your
607 files on the system, and we were unable to create a fake merge
608 to make the two consistent."""
610 class UnknownVersionError(Error):
611 #: Version that we didn't have
613 def __init__(self, version):
614 self.version = version
618 ERROR: The version you are attempting to upgrade from (%s)
619 is unknown to the repository Wizard is using.""" % str(self.version)
621 class UpgradeInProgressError(Error):
622 #: Location of pending upgrade
624 #: Time of pending upgrade
626 def __init__(self, location, time):
627 self.location = location
632 ERROR: There is already an upgrade in progress at
636 which was last started at %s.
638 To ignore and start another upgrade anyway, remove the file
639 .wizard/pending and try again.""" % (self.location, time.ctime(self.time))