2 Object model for querying information and manipulating deployments
3 of autoinstalls. Every :class:`Deployment` has an :class:`app.ApplicationVersion`
4 which in turn has an :class:`app.Application`.
21 from wizard import app, git, old_log, shell, sql, util
23 ## -- Global Functions --
25 def get_install_lines(versions_store, user=None):
27 Low level function that retrieves a list of lines from the
28 :term:`versions store` that can be passed to :meth:`Deployment.parse`.
30 if os.path.isfile(versions_store):
31 return fileinput.input([versions_store])
33 return fileinput.input([versions_store + "/" + user])
34 return fileinput.input([versions_store + "/" + f for f in sorted(os.listdir(versions_store))])
36 def parse_install_lines(show, versions_store, yield_errors = False, user = None):
38 Generator function for iterating through all autoinstalls.
39 Each item is an instance of :class:`Deployment`, or possibly
40 a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can
41 filter out applications and versions by specifying ``app``
42 or ``app-1.2.3`` in ``show``. This function may generate
46 show = app.applications()
47 elif isinstance(show, str):
48 # otherwise, frozenset will treat string as an iterable
49 show = frozenset([show])
51 show = frozenset(show)
52 for line in get_install_lines(versions_store, user):
55 d = Deployment.parse(line)
56 name = d.application.name
57 except app.NoSuchApplication as e:
60 e.location = line.split(':')[0]
67 # we consider this a worse error
68 logging.warning("Error with '%s'" % line.rstrip())
71 if name + "-" + str(d.version) in show or name in show:
78 def web(dir, url=None):
80 Attempts to determine the URL a directory would be web-accessible at.
81 If ``url`` is specified, automatically use it. Returns a generator whic
82 produces a list of candidate urls.
84 This function implements a plugin interface named :ref:`wizard.deploy.web`.
90 for entry in pkg_resources.iter_entry_points("wizard.deploy.web"):
93 if isinstance(r, str):
94 r = urlparse.urlparse(r)
98 host = os.getenv("WIZARD_WEB_HOST")
99 path = os.getenv("WIZARD_WEB_PATH")
100 if host is not None and path is not None:
101 yield urlparse.ParseResult(
107 ## -- Model Objects --
110 def chdir_to_location(f, self, *args, **kwargs):
112 Decorator for making a function have working directory
113 :attr:`Deployment.location`.
115 with util.ChangeDirectory(self.location):
116 return f(self, *args, **kwargs)
118 class Deployment(object):
120 Represents a deployment of an autoinstall, e.g. directory
121 that has ``.scripts`` directory or ``.scripts-version``
122 file in it. Supply ``version`` with an :class:`ApplicationVersion` only if
123 you were reading from the :term:`versions store` and care about
124 speed (data from there can be stale).
126 The Deployment interface is somewhat neutered, so you may
127 want to use :class:`WorkingCopy` or :class:`ProductionCopy` for
128 more powerful operations.
130 #: Absolute path to the deployment
132 def __init__(self, location, version=None):
133 self.location = os.path.abspath(location)
134 self._app_version = version
135 # some cache variables
136 self._read_cache = {}
141 def invalidateCache(self):
143 Invalidates all cached variables. This currently applies to
144 :attr:`app_version`, :attr:`old_log` and :meth:`read`.
146 self._app_version = None
147 self._read_cache = {}
149 def read(self, file, force = False):
151 Reads a file's contents, possibly from cache unless ``force``
154 if force or file not in self._read_cache:
155 f = open(os.path.join(self.location, file))
156 self._read_cache[file] = f.read()
158 return self._read_cache[file]
161 Extracts all the values of all variables from deployment.
162 These variables may be used for parametrizing generic parent
163 commits and include things such as database access credentials
164 and local configuration.
166 return self.application.extract(self)
170 Checks if this is an autoinstall, throws an exception if there
173 with util.ChangeDirectory(self.location):
174 has_git = os.path.isdir(".git")
175 has_scripts = os.path.isdir(".scripts")
176 if not has_git and has_scripts:
177 raise CorruptedAutoinstallError(self.location)
178 elif has_git and not has_scripts:
179 raise AlreadyVersionedError(self.location)
180 elif not has_git and not has_scripts:
181 if os.path.isfile(".scripts-version"):
182 raise NotMigratedError(self.location)
184 raise NotAutoinstallError(self.location)
186 def verifyTag(self, srv_path):
188 Checks if the purported version has a corresponding tag
189 in the upstream repository.
191 repo = self.application.repository(srv_path)
193 shell.eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--')
194 except shell.CallError:
195 raise NoTagError(self.app_version.scripts_tag)
197 def verifyGit(self, srv_path):
199 Checks if the autoinstall's Git repository makes sense,
200 checking if the tag is parseable and corresponds to
201 a real application, and if the tag in this repository
202 corresponds to the one in the remote repository.
204 with util.ChangeDirectory(self.location):
205 repo = self.application.repository(srv_path)
206 def repo_rev_parse(tag):
207 return shell.eval("git", "--git-dir", repo, "rev-parse", tag)
208 def self_rev_parse(tag):
210 return shell.safeCall("git", "rev-parse", tag, strip=True)
211 except shell.CallError:
212 raise NoLocalTagError(tag)
213 def compare_tags(tag):
214 return repo_rev_parse(tag) == self_rev_parse(tag)
215 if not compare_tags(self.app_version.pristine_tag):
216 raise InconsistentPristineTagError(self.app_version.pristine_tag)
217 if not compare_tags(self.app_version.scripts_tag):
218 raise InconsistentScriptsTagError(self.app_version.scripts_tag)
219 parent = repo_rev_parse(self.app_version.scripts_tag)
220 merge_base = shell.safeCall("git", "merge-base", parent, "HEAD", strip=True)
221 if merge_base != parent:
222 raise HeadNotDescendantError(self.app_version.scripts_tag)
224 def verifyConfigured(self):
226 Checks if the autoinstall is configured running.
228 if not self.configured:
229 raise NotConfiguredError(self.location)
232 def verifyVersion(self):
234 Checks if our version and the version number recorded in a file
237 real = self.detectVersion()
238 if not str(real) == self.app_version.pristine_tag.partition('-')[2]:
239 raise VersionMismatchError(real, self.version)
242 def detectVersion(self):
244 Returns the real version, based on filesystem, of install.
246 Throws a :class:`VersionDetectionError` if we couldn't figure out
247 what the real version was.
249 real = self.application.detectVersion(self)
251 raise VersionDetectionError
256 def configured(self):
257 """Whether or not an autoinstall has been configured/installed for use."""
258 return self.application.checkConfig(self)
261 """Whether or not the autoinstalls has been migrated."""
262 return os.path.isdir(self.scripts_dir)
264 def scripts_dir(self):
265 """The absolute path of the ``.scripts`` directory."""
266 return os.path.join(self.location, '.scripts')
268 def old_version_file(self):
270 The absolute path of either ``.scripts-version`` (for unmigrated
271 installs) or ``.scripts/version``.
275 Use of this is discouraged for migrated installs.
277 return os.path.join(self.location, '.scripts-version')
279 def version_file(self):
280 """The absolute path of the ``.scripts/version`` file."""
281 return os.path.join(self.scripts_dir, 'version')
284 """The absolute path of the :file:`.scripts/dsn` override file."""
285 return os.path.join(self.scripts_dir, 'dsn')
288 """The absolute path of the :file:`.scripts/url` override file."""
289 return os.path.join(self.scripts_dir, 'url')
291 def application(self):
292 """The :class:`app.Application` of this deployment."""
293 return self.app_version.application
297 The :class:`wizard.old_log.Log` of this deployment. This
298 is only applicable to un-migrated autoinstalls.
300 if not self._old_log:
301 self._old_log = old_log.DeployLog.load(self)
306 The :class:`distutils.version.LooseVersion` of this
309 return self.app_version.version
311 def app_version(self):
312 """The :class:`app.ApplicationVersion` of this deployment."""
313 if not self._app_version:
314 if os.path.isdir(os.path.join(self.location, ".git")):
316 with util.ChangeDirectory(self.location):
317 appname, _, version = git.describe().partition('-')
318 self._app_version = app.ApplicationVersion.make(appname, version)
319 except shell.CallError:
321 if not self._app_version:
323 self._app_version = self.old_log[-1].version
324 except old_log.ScriptsVersionNoSuchFile:
326 if not self._app_version:
327 appname = shell.eval("git", "config", "remote.origin.url").rpartition("/")[2].partition(".")[0]
328 self._app_version = app.ApplicationVersion.make(appname, "unknown")
329 return self._app_version
332 """The :class:`sqlalchemy.engine.url.URL` for this deployment."""
334 self._dsn = sql.auth(self.application.dsn(self))
338 """The :class:`urlparse.ParseResult` for this deployment."""
340 self._urlGen = web(self.location, self.application.url(self))
346 self._url = self._urlGen.next() # pylint: disable-msg=E1101
348 except StopIteration:
353 Parses a line from the :term:`versions store`.
357 Use this method only when speed is of the utmost
358 importance. You should prefer to directly create a deployment
359 with only a ``location`` when possible.
363 location, deploydir = line.split(":")
365 return ProductionCopy(line) # lazy loaded version
367 return ProductionCopy(location, version=app.ApplicationVersion.parse(deploydir))
369 e.location = location
372 class ProductionCopy(Deployment):
374 Represents the production copy of a deployment. This copy
375 is canonical, and is the only one guaranteed to be accessible
376 via web, have a database, etc.
379 def upgrade(self, version, options):
381 Performs an upgrade of database schemas and other non-versioned data.
383 return self.application.upgrade(self, version, options)
385 def backup(self, options):
387 Performs a backup of database schemas and other non-versioned data.
389 # There are retarded amounts of race-safety in this function,
390 # because we do NOT want to claim to have made a backup, when
391 # actually something weird happened to it.
392 backupdir = os.path.join(self.scripts_dir, "backups")
393 if not os.path.exists(backupdir):
397 if e.errno == errno.EEXIST:
401 tmpdir = tempfile.mkdtemp() # actually will be kept around
403 self.application.backup(self, tmpdir, options)
404 except app.BackupFailure:
405 # the backup is bogus, don't let it show up
406 shutil.rmtree(tmpdir)
409 with util.LockDirectory(os.path.join(backupdir, "lock")):
411 backup = str(self.version) + "-" + datetime.datetime.today().strftime("%Y-%m-%dT%H%M%S")
412 outdir = os.path.join(backupdir, backup)
413 if os.path.exists(outdir):
414 logging.warning("Backup: A backup occurred in the last second. Trying again in a second...")
418 shutil.move(tmpdir, outdir)
420 # don't leave half-baked stuff lying around
422 shutil.rmtree(outdir)
429 def restore(self, backup, options):
431 Restores a backup. Destroys state, so be careful! Also, this does
432 NOT restore the file-level backup, which is what 'wizard restore'
433 does, so you probably do NOT want to call this elsewhere unless
434 you know what you're doing (call 'wizard restore' instead).
436 backup_dir = os.path.join(".scripts", "backups", backup)
437 return self.application.restore(self, backup_dir, options)
439 def remove(self, options):
441 Deletes all non-local or non-filesystem data (such as databases) that
442 this application uses.
444 self.application.remove(self, options)
445 def verifyDatabase(self):
447 Checks if the autoinstall has a properly configured database.
449 if not self.application.checkDatabase(self):
450 raise DatabaseVerificationError
453 Checks if the autoinstall is viewable from the web. If you do not run
454 this, there is no guarantee that the url returned by this application
458 if not self.application.checkWeb(self):
461 except UnknownWebPath:
462 raise WebVerificationError
465 def fetch(self, path, post=None):
467 Performs a HTTP request on the website.
469 return util.fetch(self.url.netloc, self.url.path, path, post) # pylint: disable-msg=E1103
471 class WorkingCopy(Deployment):
473 Represents a temporary clone of a deployment that we can make
474 modifications to without fear of interfering with a production
475 deployment. More operations are permitted on these copies.
477 def setAppVersion(self, app_version):
479 Manually resets the application version; useful if the working
480 copy is off in space (i.e. not anchored to something we can
481 git describe off of.)
483 self._app_version = app_version
485 def parametrize(self, deployment):
487 Edits files in ``dir`` to replace WIZARD_* variables with literal
488 instances based on ``deployment``. This is used for constructing
489 virtual merge bases, and as such ``deployment`` will generally not
492 return self.application.parametrize(self, deployment)
494 def prepareConfig(self):
496 Edits files in the deployment such that any user-specific configuration
497 is replaced with generic WIZARD_* variables.
499 return self.application.prepareConfig(self)
501 def resolveConflicts(self):
503 Resolves conflicted files in this working copy. Returns whether or
504 not all conflicted files were resolved or not. Fully resolved
505 files are added to the index, but no commit is made.
507 return self.application.resolveConflicts(self)
509 def prepareMerge(self):
511 Performs various edits to files in the current working directory in
512 order to make a merge go more smoothly. This is usually
513 used to fix botched line-endings.
515 return self.application.prepareMerge(self)
519 class Error(wizard.Error):
520 """Base error class for this module"""
523 class NotMigratedError(Error):
525 The deployment contains a .scripts-version file, but no .git
526 or .scripts directory.
528 #: Directory of deployment
530 def __init__(self, dir):
533 return """This installation was not migrated"""
535 class AlreadyVersionedError(Error):
536 """The deployment contained a .git directory but no .scripts directory."""
537 #: Directory of deployment
539 def __init__(self, dir):
544 ERROR: Directory contains a .git directory, but not
545 a .scripts directory. If this is not a corrupt
546 migration, this means that the user was versioning their
547 install using Git."""
549 class NotConfiguredError(Error):
550 """The install was missing essential configuration."""
551 #: Directory of unconfigured install
553 def __init__(self, dir):
558 ERROR: The install was well-formed, but not configured
559 (essential configuration files were not found.)"""
561 class CorruptedAutoinstallError(Error):
562 """The install was missing a .git directory, but had a .scripts directory."""
563 #: Directory of the corrupted install
565 def __init__(self, dir):
570 ERROR: Directory contains a .scripts directory,
571 but not a .git directory."""
573 class NotAutoinstallError(Error):
574 """Application is not an autoinstall."""
575 #: Directory of the not autoinstall
577 def __init__(self, dir):
586 does not appear to be an autoinstall. If you are in a
587 subdirectory of an autoinstall, you need to use the root
588 directory for the autoinstall.""" % self.dir
590 class NoTagError(Error):
591 """Deployment has a tag that does not have an equivalent in upstream repository."""
594 def __init__(self, tag):
599 ERROR: Could not find tag %s in repository.""" % self.tag
601 class NoLocalTagError(Error):
602 """Could not find tag in local repository."""
605 def __init__(self, tag):
610 ERROR: Could not find tag %s in local repository.""" % self.tag
612 class InconsistentPristineTagError(Error):
613 """Pristine tag commit ID does not match upstream pristine tag commit ID."""
616 def __init__(self, tag):
621 ERROR: Local pristine tag %s did not match repository's. This
622 probably means an upstream rebase occured.""" % self.tag
624 class InconsistentScriptsTagError(Error):
625 """Scripts tag commit ID does not match upstream scripts tag commit ID."""
628 def __init__(self, tag):
633 ERROR: Local scripts tag %s did not match repository's. This
634 probably means an upstream rebase occurred.""" % self.tag
636 class HeadNotDescendantError(Error):
637 """HEAD is not connected to tag."""
638 #: Tag that HEAD should have been descendant of.
640 def __init__(self, tag):
645 ERROR: HEAD is not a descendant of %s. This probably
646 means that an upstream rebase occurred, and new tags were
647 pulled, but local user commits were never rebased.""" % self.tag
649 class VersionDetectionError(Error):
650 """Could not detect real version of application."""
654 ERROR: Could not detect the real version of the application."""
656 class VersionMismatchError(Error):
657 """Git version of application does not match detected version."""
662 def __init__(self, real_version, git_version):
663 self.real_version = real_version
664 self.git_version = git_version
668 ERROR: The detected version %s did not match the Git
669 version %s.""" % (self.real_version, self.git_version)
671 class WebVerificationError(Error):
672 """Could not access the application on the web"""
676 ERROR: We were not able to access the application on the
677 web. This may indicate that the website is behind
678 authentication on the htaccess level. You can find
679 the contents of the page from the debug backtraces."""
681 class DatabaseVerificationError(Error):
682 """Could not access the database"""
686 ERROR: We were not able to access the database for
687 this application; this probably means that your database
688 configuration is misconfigured."""
690 class UnknownWebPath(Error):
691 """Could not determine application's web path."""
695 ERROR: We were not able to determine what the application's
696 host and path were in order to perform a web request
697 on the application. You can specify this manually using
698 the WIZARD_WEB_HOST and WIZARD_WEB_PATH environment