2 Object model for querying information and manipulating deployments
3 of autoinstalls. Every :class:`Deployment` has an :class:`app.ApplicationVersion`
4 which in turn has an :class:`app.Application`.
21 from wizard import app, git, old_log, shell, sql, util
23 ## -- Global Functions --
25 def get_install_lines(versions_store, user=None):
27 Low level function that retrieves a list of lines from the
28 :term:`versions store` that can be passed to :meth:`Deployment.parse`.
30 if os.path.isfile(versions_store):
31 return fileinput.input([versions_store])
33 return fileinput.input([versions_store + "/" + user])
34 return fileinput.input([versions_store + "/" + f for f in sorted(os.listdir(versions_store))])
36 def parse_install_lines(show, versions_store, yield_errors = False, user = None):
38 Generator function for iterating through all autoinstalls.
39 Each item is an instance of :class:`Deployment`, or possibly
40 a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can
41 filter out applications and versions by specifying ``app``
42 or ``app-1.2.3`` in ``show``. This function may generate
46 show = app.applications()
47 elif isinstance(show, str):
48 # otherwise, frozenset will treat string as an iterable
49 show = frozenset([show])
51 show = frozenset(show)
52 for line in get_install_lines(versions_store, user):
55 d = Deployment.parse(line)
56 name = d.application.name
57 except app.NoSuchApplication as e:
60 e.location = line.split(':')[0]
67 # we consider this a worse error
68 logging.warning("Error with '%s'" % line.rstrip())
71 if name + "-" + util.truncate(str(d.version)) in show or name in show:
78 def web(dir, url=None):
80 Attempts to determine the URL a directory would be web-accessible at.
81 If ``url`` is specified, automatically use it. Returns a generator which
82 produces a list of candidate urls.
84 This function implements a plugin interface named :ref:`wizard.deploy.web`.
87 if isinstance(url, str):
88 url = urlparse.urlparse(url)
89 logging.info("wizard.deploy.web: Using default URL %s", url)
93 for entry in pkg_resources.iter_entry_points("wizard.deploy.web"):
94 logging.debug("wizard.deploy.web: Processing %s", entry)
97 if isinstance(r, str):
98 r = urlparse.urlparse(r)
99 logging.info("wizard.deploy.web: Using plugin-supplied URL %s", r)
102 # try the environment
103 host = os.getenv("WIZARD_WEB_HOST")
104 path = os.getenv("WIZARD_WEB_PATH")
105 if host is not None and path is not None:
106 r = urlparse.ParseResult(
111 logging.info("wizard.deploy.web: Using environment URL %s", r)
114 logging.info("wizard.deploy.web: Exhausted URLs")
116 ## -- Model Objects --
119 def chdir_to_location(f, self, *args, **kwargs):
121 Decorator for making a function have working directory
122 :attr:`Deployment.location`.
124 with util.ChangeDirectory(self.location):
125 return f(self, *args, **kwargs)
127 class Deployment(object):
129 Represents a deployment of an autoinstall, e.g. directory that has a
130 ``.wizard`` directory in it. Supply ``version`` with an
131 :class:`ApplicationVersion` only if you were reading from the
132 :term:`versions store` and care about speed (data from there can be
135 The Deployment interface is somewhat neutered, so you may
136 want to use :class:`WorkingCopy` or :class:`ProductionCopy` for
137 more powerful operations.
141 For legacy purposes, deployments can also be marked by a
142 ``.scripts`` directory or a ``.scripts-version`` file.
144 #: Absolute path to the deployment
146 def __init__(self, location, version=None):
147 self.location = os.path.abspath(location)
148 self._app_version = version
149 # some cache variables
150 self._read_cache = {}
155 self._wizard_dir = None
156 def invalidateCache(self):
158 Invalidates all cached variables. This currently applies to
159 :attr:`app_version`, :attr:`old_log` and :meth:`read`.
161 self._app_version = None
162 self._read_cache = {}
164 def setAppVersion(self, app_version):
166 Manually resets the application version; useful if the working
167 copy is off in space (i.e. not anchored to something we can
168 git describe off of) or there is no metadata to be heard of.
170 self._app_version = app_version
171 def read(self, file, force = False):
173 Reads a file's contents, possibly from cache unless ``force``
176 if force or file not in self._read_cache:
177 f = open(os.path.join(self.location, file))
178 self._read_cache[file] = f.read()
180 return self._read_cache[file]
183 Extracts all the values of all variables from deployment.
184 These variables may be used for parametrizing generic parent
185 commits and include things such as database access credentials
186 and local configuration.
188 return self.application.extract(self)
190 def verify(self, no_touch=False):
192 Checks if this is an autoinstall, throws an exception if there
193 are problems. If ``no_touch`` is ``True``, it will not attempt
194 edit the installation.
196 with util.ChangeDirectory(self.location):
197 has_git = os.path.isdir(".git")
198 has_wizard = os.path.isdir(".wizard")
199 if not has_wizard and os.path.isdir(".scripts"):
201 os.symlink(".scripts", ".wizard")
203 if not has_git and has_wizard:
204 raise CorruptedAutoinstallError(self.location)
205 elif has_git and not has_wizard:
206 raise AlreadyVersionedError(self.location)
208 elif not has_git and not has_wizard:
209 if os.path.isfile(".scripts-version"):
210 raise NotMigratedError(self.location)
212 raise NotAutoinstallError(self.location)
214 def verifyTag(self, srv_path):
216 Checks if the purported version has a corresponding tag
217 in the upstream repository.
219 repo = self.application.repository(srv_path)
221 shell.eval("git", "--git-dir", repo, "rev-parse", self.app_version.wizard_tag, '--')
222 except shell.CallError:
223 raise NoTagError(self.app_version.wizard_tag)
225 def verifyGit(self, srv_path):
227 Checks if the autoinstall's Git repository makes sense,
228 checking if the tag is parseable and corresponds to
229 a real application, and if the tag in this repository
230 corresponds to the one in the remote repository.
232 with util.ChangeDirectory(self.location):
233 repo = self.application.repository(srv_path)
234 def repo_rev_parse(tag):
235 return shell.eval("git", "--git-dir", repo, "rev-parse", tag)
236 def self_rev_parse(tag):
238 return shell.safeCall("git", "rev-parse", tag, strip=True)
239 except shell.CallError:
240 raise NoLocalTagError(tag)
241 def compare_tags(tag):
242 return repo_rev_parse(tag) == self_rev_parse(tag)
243 if not compare_tags(self.app_version.pristine_tag):
244 raise InconsistentPristineTagError(self.app_version.pristine_tag)
245 if not compare_tags(self.app_version.wizard_tag):
247 raise InconsistentWizardTagError(self.app_version.wizard_tag)
248 parent = repo_rev_parse(self.app_version.wizard_tag)
249 merge_base = shell.safeCall("git", "merge-base", parent, "HEAD", strip=True)
250 if merge_base != parent:
252 raise HeadNotDescendantError(self.app_version.wizard_tag)
254 def verifyConfigured(self):
256 Checks if the autoinstall is configured running.
258 if not self.configured:
259 raise NotConfiguredError(self.location)
262 def verifyVersion(self):
264 Checks if our version and the version number recorded in a file
267 real = self.detectVersion()
268 if not str(real) == self.app_version.pristine_tag.partition('-')[2]:
269 raise VersionMismatchError(real, self.version)
272 def detectVersion(self):
274 Returns the real version, based on filesystem, of install.
276 Throws a :class:`VersionDetectionError` if we couldn't figure out
277 what the real version was.
279 real = self.application.detectVersion(self)
281 raise VersionDetectionError
286 def configured(self):
287 """Whether or not an autoinstall has been configured/installed for use."""
288 return self.application.checkConfig(self)
291 """Whether or not the autoinstalls has been migrated."""
292 return os.path.isdir(self.wizard_dir)
294 def wizard_dir(self):
295 """The absolute path of the Wizard directory."""
296 return os.path.join(self.location, ".wizard")
298 def backup_dir(self):
299 """The absolute path to ``.wizard/backups``."""
300 return os.path.join(self.wizard_dir, "backups")
303 def old_version_file(self):
305 The absolute path of either ``.scripts-version``.
307 return os.path.join(self.location, '.scripts-version')
309 def blacklisted_file(self):
310 """The absolute path of the ``.wizard/blacklisted`` file."""
311 return os.path.join(self.wizard_dir, 'blacklisted')
313 def pending_file(self):
314 """The absolute path of the ``.wizard/pending`` file."""
315 return os.path.join(self.wizard_dir, 'pending')
317 def version_file(self):
318 """The absolute path of the ``.wizard/version`` file."""
319 return os.path.join(self.wizard_dir, 'version')
322 """The absolute path of the :file:`.wizard/dsn` override file."""
323 return os.path.join(self.wizard_dir, 'dsn')
326 """The absolute path of the :file:`.wizard/url` override file."""
327 return os.path.join(self.wizard_dir, 'url')
329 def application(self):
330 """The :class:`app.Application` of this deployment."""
331 return self.app_version.application
335 The :class:`wizard.old_log.Log` of this deployment. This
336 is only applicable to un-migrated autoinstalls.
338 if not self._old_log:
339 self._old_log = old_log.DeployLog.load(self)
344 The :class:`distutils.version.LooseVersion` of this
347 return self.app_version.version
349 def app_version(self):
350 """The :class:`app.ApplicationVersion` of this deployment."""
351 if not self._app_version:
352 if os.path.isdir(os.path.join(self.location, ".git")):
354 with util.ChangeDirectory(self.location):
355 appname, _, version = git.describe().partition('-')
356 self._app_version = app.ApplicationVersion.make(appname, version)
357 except shell.CallError:
359 if not self._app_version:
362 self._app_version = self.old_log[-1].version
363 except old_log.ScriptsVersionNoSuchFile:
365 if not self._app_version:
366 appname = shell.eval("git", "config", "remote.origin.url").rpartition("/")[2].partition(".")[0]
367 self._app_version = app.ApplicationVersion.make(appname, "unknown")
368 return self._app_version
371 """The :class:`sqlalchemy.engine.url.URL` for this deployment."""
373 self._dsn = sql.auth(self.application.dsn(self))
377 """The :class:`urlparse.ParseResult` for this deployment."""
383 Initializes :attr:`url` with a possible URL the web application may
384 be located at. It may be called again to switch to another possible
385 URL, usually in the event of a web access failure.
388 self._urlGen = web(self.location, self.application.url(self))
390 self._url = self._urlGen.next() # pylint: disable-msg=E1101
392 except StopIteration:
397 Parses a line from the :term:`versions store`.
401 Use this method only when speed is of the utmost
402 importance. You should prefer to directly create a deployment
403 with only a ``location`` when possible.
407 location, deploydir = line.split(":")
409 return ProductionCopy(line) # lazy loaded version
411 return ProductionCopy(location, version=app.ApplicationVersion.parse(deploydir))
413 e.location = location
416 class ProductionCopy(Deployment):
418 Represents the production copy of a deployment. This copy
419 is canonical, and is the only one guaranteed to be accessible
420 via web, have a database, etc.
423 def upgrade(self, version, options):
425 Performs an upgrade of database schemas and other non-versioned data.
427 return self.application.upgrade(self, version, options)
429 def backup(self, options):
431 Performs a backup of database schemas and other non-versioned data.
433 # There are retarded amounts of race-safety in this function,
434 # because we do NOT want to claim to have made a backup, when
435 # actually something weird happened to it.
436 if not os.path.exists(self.backup_dir):
438 os.mkdir(self.backup_dir)
440 if e.errno == errno.EEXIST:
444 tmpdir = tempfile.mkdtemp() # actually will be kept around
446 self.application.backup(self, tmpdir, options)
447 except app.BackupFailure:
448 # the backup is bogus, don't let it show up
449 shutil.rmtree(tmpdir)
452 with util.LockDirectory(os.path.join(self.backup_dir, "lock")):
454 backup = str(self.version) + "-" + datetime.datetime.today().strftime("%Y-%m-%dT%H%M%S")
455 outdir = os.path.join(self.backup_dir, backup)
456 if os.path.exists(outdir):
457 logging.warning("Backup: A backup occurred in the last second. Trying again in a second...")
461 shutil.move(tmpdir, outdir)
463 # don't leave half-baked stuff lying around
465 shutil.rmtree(outdir)
472 def restore(self, backup, options):
474 Restores a backup. Destroys state, so be careful! Also, this does
475 NOT restore the file-level backup, which is what 'wizard restore'
476 does, so you probably do NOT want to call this elsewhere unless
477 you know what you're doing (call 'wizard restore' instead).
479 return self.application.restore(self, os.path.join(self.backup_dir, backup), options)
481 def remove(self, options):
483 Deletes all non-local or non-filesystem data (such as databases) that
484 this application uses.
486 self.application.remove(self, options)
487 def verifyDatabase(self):
489 Checks if the autoinstall has a properly configured database.
491 if not self.application.checkDatabase(self):
492 raise DatabaseVerificationError
495 Checks if the autoinstall is viewable from the web. If you do not run
496 this, there is no guarantee that the url returned by this application
500 if not self.application.checkWeb(self):
503 except UnknownWebPath:
504 raise WebVerificationError
507 def fetch(self, path, post=None):
509 Performs a HTTP request on the website.
511 return util.fetch(self.url.netloc, self.url.path, path, post) # pylint: disable-msg=E1103
513 class WorkingCopy(Deployment):
515 Represents a temporary clone of a deployment that we can make
516 modifications to without fear of interfering with a production
517 deployment. More operations are permitted on these copies.
520 def parametrize(self, deployment):
522 Edits files in ``dir`` to replace WIZARD_* variables with literal
523 instances based on ``deployment``. This is used for constructing
524 virtual merge bases, and as such ``deployment`` will generally not
527 return self.application.parametrize(self, deployment)
529 def prepareConfig(self):
531 Edits files in the deployment such that any user-specific configuration
532 is replaced with generic WIZARD_* variables.
534 return self.application.prepareConfig(self)
536 def resolveConflicts(self):
538 Resolves conflicted files in this working copy. Returns whether or
539 not all conflicted files were resolved or not. Fully resolved
540 files are added to the index, but no commit is made.
542 return self.application.resolveConflicts(self)
544 def prepareMerge(self):
546 Performs various edits to files in the current working directory in
547 order to make a merge go more smoothly. This is usually
548 used to fix botched line-endings.
550 return self.application.prepareMerge(self)
554 class Error(wizard.Error):
555 """Base error class for this module"""
559 class NotMigratedError(Error):
561 The deployment contains a .scripts-version file, but no .git
562 or .wizard directory.
564 #: Directory of deployment
566 def __init__(self, dir):
569 return """This installation was not migrated"""
571 class AlreadyVersionedError(Error):
572 """The deployment contained a .git directory but no .wizard directory."""
573 #: Directory of deployment
575 def __init__(self, dir):
580 ERROR: Directory contains a .git directory, but not
581 a .wizard directory. If this is not a corrupt
582 migration, this means that the user was versioning their
583 install using Git."""
585 class NotConfiguredError(Error):
586 """The install was missing essential configuration."""
587 #: Directory of unconfigured install
589 def __init__(self, dir):
594 ERROR: The install was well-formed, but not configured
595 (essential configuration files were not found.)"""
597 class CorruptedAutoinstallError(Error):
598 """The install was missing a .git directory, but had a .wizard directory."""
599 #: Directory of the corrupted install
601 def __init__(self, dir):
606 ERROR: Directory contains a .wizard directory,
607 but not a .git directory."""
609 class NotAutoinstallError(Error):
610 """Application is not an autoinstall."""
611 #: Directory of the not autoinstall
613 def __init__(self, dir):
622 does not appear to be an autoinstall. If you are in a
623 subdirectory of an autoinstall, you need to use the root
624 directory for the autoinstall.""" % self.dir
626 class NoTagError(Error):
627 """Deployment has a tag that does not have an equivalent in upstream repository."""
630 def __init__(self, tag):
635 ERROR: Could not find tag %s in repository.""" % self.tag
637 class NoLocalTagError(Error):
638 """Could not find tag in local repository."""
641 def __init__(self, tag):
646 ERROR: Could not find tag %s in local repository.""" % self.tag
648 class InconsistentPristineTagError(Error):
649 """Pristine tag commit ID does not match upstream pristine tag commit ID."""
652 def __init__(self, tag):
657 ERROR: Local pristine tag %s did not match repository's. This
658 probably means an upstream rebase occured.""" % self.tag
660 class InconsistentWizardTagError(Error):
661 """Wizard tag commit ID does not match upstream wizard tag commit ID."""
664 def __init__(self, tag):
669 ERROR: Local wizard tag %s did not match repository's. This
670 probably means an upstream rebase occurred. Try
671 'git fetch --tags && wizard remaster'.""" % self.tag
673 class HeadNotDescendantError(Error):
674 """HEAD is not connected to tag."""
675 #: Tag that HEAD should have been descendant of.
677 def __init__(self, tag):
682 ERROR: HEAD is not a descendant of %s. This probably
683 means that an upstream rebase occurred, and new tags were
684 pulled, but local user commits were never rebased. Try
685 running 'wizard remaster'.""" % self.tag
687 class VersionDetectionError(Error):
688 """Could not detect real version of application."""
692 ERROR: Could not detect the real version of the application."""
694 class VersionMismatchError(Error):
695 """Git version of application does not match detected version."""
700 def __init__(self, real_version, git_version):
701 self.real_version = real_version
702 self.git_version = git_version
706 ERROR: The detected version %s did not match the Git
707 version %s.""" % (self.real_version, self.git_version)
709 class WebVerificationError(Error):
710 """Could not access the application on the web"""
714 ERROR: We were not able to access the application on the
715 web. This may indicate that the website is behind
716 authentication on the htaccess level. You can find
717 the contents of the page from the debug backtraces."""
719 class DatabaseVerificationError(Error):
720 """Could not access the database"""
724 ERROR: We were not able to access the database for
725 this application; this probably means that your database
726 configuration is misconfigured."""
728 class UnknownWebPath(Error):
729 """Could not determine application's web path."""
733 ERROR: We were not able to determine what the application's
734 host and path were in order to perform a web request
735 on the application. You can specify this manually using
736 the WIZARD_WEB_HOST and WIZARD_WEB_PATH environment