2 Object model for querying information and manipulating deployments
3 of autoinstalls. Every :class:`Deployment` has an :class:`app.ApplicationVersion`
4 which in turn has an :class:`app.Application`.
21 from wizard import app, git, old_log, shell, sql, util
23 ## -- Global Functions --
25 def get_install_lines(versions_store, user=None):
27 Low level function that retrieves a list of lines from the
28 :term:`versions store` that can be passed to :meth:`Deployment.parse`.
30 if os.path.isfile(versions_store):
31 return fileinput.input([versions_store])
33 return fileinput.input([versions_store + "/" + user])
34 return fileinput.input([versions_store + "/" + f for f in sorted(os.listdir(versions_store))])
36 def parse_install_lines(show, versions_store, yield_errors = False, user = None):
38 Generator function for iterating through all autoinstalls.
39 Each item is an instance of :class:`Deployment`, or possibly
40 a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can
41 filter out applications and versions by specifying ``app``
42 or ``app-1.2.3`` in ``show``. This function may generate
46 show = app.applications()
47 elif isinstance(show, str):
48 # otherwise, frozenset will treat string as an iterable
49 show = frozenset([show])
51 show = frozenset(show)
52 for line in get_install_lines(versions_store, user):
55 d = Deployment.parse(line)
56 name = d.application.name
57 except app.NoSuchApplication as e:
60 e.location = line.split(':')[0]
67 # we consider this a worse error
68 logging.warning("Error with '%s'" % line.rstrip())
71 if name + "-" + util.truncate(str(d.version)) in show or name in show:
78 def web(dir, url=None):
80 Attempts to determine the URL a directory would be web-accessible at.
81 If ``url`` is specified, automatically use it. Returns a generator which
82 produces a list of candidate urls.
84 This function implements a plugin interface named :ref:`wizard.deploy.web`.
87 if isinstance(url, str):
88 url = urlparse.urlparse(url)
89 logging.info("wizard.deploy.web: Using default URL %s", url)
93 for entry in pkg_resources.iter_entry_points("wizard.deploy.web"):
94 logging.debug("wizard.deploy.web: Processing %s", entry)
97 if isinstance(r, str):
98 r = urlparse.urlparse(r)
99 logging.info("wizard.deploy.web: Using plugin-supplied URL %s", r)
102 # try the environment
103 host = os.getenv("WIZARD_WEB_HOST")
104 path = os.getenv("WIZARD_WEB_PATH")
105 if host is not None and path is not None:
106 r = urlparse.ParseResult(
111 logging.info("wizard.deploy.web: Using environment URL %s", r)
114 logging.info("wizard.deploy.web: Exhausted URLs")
116 ## -- Model Objects --
119 def chdir_to_location(f, self, *args, **kwargs):
121 Decorator for making a function have working directory
122 :attr:`Deployment.location`.
124 with util.ChangeDirectory(self.location):
125 return f(self, *args, **kwargs)
127 class Deployment(object):
129 Represents a deployment of an autoinstall, e.g. directory that has a
130 ``.wizard`` directory in it. Supply ``version`` with an
131 :class:`ApplicationVersion` only if you were reading from the
132 :term:`versions store` and care about speed (data from there can be
135 The Deployment interface is somewhat neutered, so you may
136 want to use :class:`WorkingCopy` or :class:`ProductionCopy` for
137 more powerful operations.
141 For legacy purposes, deployments can also be marked by a
142 ``.scripts`` directory or a ``.scripts-version`` file.
144 #: Absolute path to the deployment
146 def __init__(self, location, version=None):
147 self.location = os.path.abspath(location)
148 self._app_version = version
149 # some cache variables
150 self._read_cache = {}
155 self._wizard_dir = None
156 def invalidateCache(self):
158 Invalidates all cached variables. This currently applies to
159 :attr:`app_version`, :attr:`old_log` and :meth:`read`.
161 self._app_version = None
162 self._read_cache = {}
164 def setAppVersion(self, app_version):
166 Manually resets the application version; useful if the working
167 copy is off in space (i.e. not anchored to something we can
168 git describe off of) or there is no metadata to be heard of.
170 self._app_version = app_version
171 def read(self, file, force = False):
173 Reads a file's contents, possibly from cache unless ``force``
176 if force or file not in self._read_cache:
177 f = open(os.path.join(self.location, file))
178 self._read_cache[file] = f.read()
180 return self._read_cache[file]
183 Extracts all the values of all variables from deployment.
184 These variables may be used for parametrizing generic parent
185 commits and include things such as database access credentials
186 and local configuration.
188 return self.application.extract(self)
190 def verify(self, no_touch=False):
192 Checks if this is an autoinstall, throws an exception if there
193 are problems. If ``no_touch`` is ``True``, it will not attempt
194 edit the installation.
196 with util.ChangeDirectory(self.location):
197 has_git = os.path.isdir(".git")
198 has_wizard = os.path.isdir(".wizard")
199 if not has_wizard and os.path.isdir(".scripts"):
201 os.symlink(".scripts", ".wizard")
203 if not has_git and has_wizard:
204 raise CorruptedAutoinstallError(self.location)
205 elif has_git and not has_wizard:
206 raise AlreadyVersionedError(self.location)
208 elif not has_git and not has_wizard:
209 if os.path.isfile(".scripts-version"):
210 raise NotMigratedError(self.location)
212 raise NotAutoinstallError(self.location)
214 def verifyTag(self, srv_path):
216 Checks if the purported version has a corresponding tag
217 in the upstream repository.
219 repo = self.application.repository(srv_path)
221 shell.eval("git", "--git-dir", repo, "rev-parse", self.app_version.wizard_tag, '--')
222 except shell.CallError:
223 raise NoTagError(self.app_version.wizard_tag)
225 def verifyGit(self, srv_path):
227 Checks if the autoinstall's Git repository makes sense,
228 checking if the tag is parseable and corresponds to
229 a real application, and if the tag in this repository
230 corresponds to the one in the remote repository.
232 with util.ChangeDirectory(self.location):
233 repo = self.application.repository(srv_path)
234 def repo_rev_parse(tag):
235 return shell.eval("git", "--git-dir", repo, "rev-parse", tag)
236 def self_rev_parse(tag):
238 return shell.safeCall("git", "rev-parse", tag, strip=True)
239 except shell.CallError:
240 raise NoLocalTagError(tag)
241 def compare_tags(tag):
242 return repo_rev_parse(tag) == self_rev_parse(tag)
243 if not compare_tags(self.app_version.pristine_tag):
244 raise InconsistentPristineTagError(self.app_version.pristine_tag)
245 if not compare_tags(self.app_version.wizard_tag):
246 raise InconsistentWizardTagError(self.app_version.wizard_tag)
247 parent = repo_rev_parse(self.app_version.wizard_tag)
248 merge_base = shell.safeCall("git", "merge-base", parent, "HEAD", strip=True)
249 if merge_base != parent:
250 raise HeadNotDescendantError(self.app_version.wizard_tag)
252 def verifyConfigured(self):
254 Checks if the autoinstall is configured running.
256 if not self.configured:
257 raise NotConfiguredError(self.location)
260 def verifyVersion(self):
262 Checks if our version and the version number recorded in a file
265 real = self.detectVersion()
266 if not str(real) == self.app_version.pristine_tag.partition('-')[2]:
267 raise VersionMismatchError(real, self.version)
270 def detectVersion(self):
272 Returns the real version, based on filesystem, of install.
274 Throws a :class:`VersionDetectionError` if we couldn't figure out
275 what the real version was.
277 real = self.application.detectVersion(self)
279 raise VersionDetectionError
284 def configured(self):
285 """Whether or not an autoinstall has been configured/installed for use."""
286 return self.application.checkConfig(self)
289 """Whether or not the autoinstalls has been migrated."""
290 return os.path.isdir(self.wizard_dir)
292 def wizard_dir(self):
293 """The absolute path of the Wizard directory."""
294 return os.path.join(self.location, ".wizard")
296 def backup_dir(self):
297 """The absolute path to ``.wizard/backups``."""
298 return os.path.join(self.wizard_dir, "backups")
301 def old_version_file(self):
303 The absolute path of either ``.scripts-version``.
305 return os.path.join(self.location, '.scripts-version')
307 def blacklisted_file(self):
308 """The absolute path of the ``.wizard/blacklisted`` file."""
309 return os.path.join(self.wizard_dir, 'blacklisted')
311 def pending_file(self):
312 """The absolute path of the ``.wizard/pending`` file."""
313 return os.path.join(self.wizard_dir, 'pending')
315 def version_file(self):
316 """The absolute path of the ``.wizard/version`` file."""
317 return os.path.join(self.wizard_dir, 'version')
320 """The absolute path of the :file:`.wizard/dsn` override file."""
321 return os.path.join(self.wizard_dir, 'dsn')
324 """The absolute path of the :file:`.wizard/url` override file."""
325 return os.path.join(self.wizard_dir, 'url')
327 def application(self):
328 """The :class:`app.Application` of this deployment."""
329 return self.app_version.application
333 The :class:`wizard.old_log.Log` of this deployment. This
334 is only applicable to un-migrated autoinstalls.
336 if not self._old_log:
337 self._old_log = old_log.DeployLog.load(self)
342 The :class:`distutils.version.LooseVersion` of this
345 return self.app_version.version
347 def app_version(self):
348 """The :class:`app.ApplicationVersion` of this deployment."""
349 if not self._app_version:
350 if os.path.isdir(os.path.join(self.location, ".git")):
352 with util.ChangeDirectory(self.location):
353 appname, _, version = git.describe().partition('-')
354 self._app_version = app.ApplicationVersion.make(appname, version)
355 except shell.CallError:
357 if not self._app_version:
360 self._app_version = self.old_log[-1].version
361 except old_log.ScriptsVersionNoSuchFile:
363 if not self._app_version:
364 appname = shell.eval("git", "config", "remote.origin.url").rpartition("/")[2].partition(".")[0]
365 self._app_version = app.ApplicationVersion.make(appname, "unknown")
366 return self._app_version
369 """The :class:`sqlalchemy.engine.url.URL` for this deployment."""
371 self._dsn = sql.auth(self.application.dsn(self))
375 """The :class:`urlparse.ParseResult` for this deployment."""
381 Initializes :attr:`url` with a possible URL the web application may
382 be located at. It may be called again to switch to another possible
383 URL, usually in the event of a web access failure.
386 self._urlGen = web(self.location, self.application.url(self))
388 self._url = self._urlGen.next() # pylint: disable-msg=E1101
390 except StopIteration:
395 Parses a line from the :term:`versions store`.
399 Use this method only when speed is of the utmost
400 importance. You should prefer to directly create a deployment
401 with only a ``location`` when possible.
405 location, deploydir = line.split(":")
407 return ProductionCopy(line) # lazy loaded version
409 return ProductionCopy(location, version=app.ApplicationVersion.parse(deploydir))
411 e.location = location
414 class ProductionCopy(Deployment):
416 Represents the production copy of a deployment. This copy
417 is canonical, and is the only one guaranteed to be accessible
418 via web, have a database, etc.
421 def upgrade(self, version, options):
423 Performs an upgrade of database schemas and other non-versioned data.
425 return self.application.upgrade(self, version, options)
427 def backup(self, options):
429 Performs a backup of database schemas and other non-versioned data.
431 # There are retarded amounts of race-safety in this function,
432 # because we do NOT want to claim to have made a backup, when
433 # actually something weird happened to it.
434 if not os.path.exists(self.backup_dir):
436 os.mkdir(self.backup_dir)
438 if e.errno == errno.EEXIST:
442 tmpdir = tempfile.mkdtemp() # actually will be kept around
444 self.application.backup(self, tmpdir, options)
445 except app.BackupFailure:
446 # the backup is bogus, don't let it show up
447 shutil.rmtree(tmpdir)
450 with util.LockDirectory(os.path.join(self.backup_dir, "lock")):
452 backup = str(self.version) + "-" + datetime.datetime.today().strftime("%Y-%m-%dT%H%M%S")
453 outdir = os.path.join(self.backup_dir, backup)
454 if os.path.exists(outdir):
455 logging.warning("Backup: A backup occurred in the last second. Trying again in a second...")
459 shutil.move(tmpdir, outdir)
461 # don't leave half-baked stuff lying around
463 shutil.rmtree(outdir)
470 def restore(self, backup, options):
472 Restores a backup. Destroys state, so be careful! Also, this does
473 NOT restore the file-level backup, which is what 'wizard restore'
474 does, so you probably do NOT want to call this elsewhere unless
475 you know what you're doing (call 'wizard restore' instead).
477 return self.application.restore(self, os.path.join(self.backup_dir, backup), options)
479 def remove(self, options):
481 Deletes all non-local or non-filesystem data (such as databases) that
482 this application uses.
484 self.application.remove(self, options)
485 def verifyDatabase(self):
487 Checks if the autoinstall has a properly configured database.
489 if not self.application.checkDatabase(self):
490 raise DatabaseVerificationError
493 Checks if the autoinstall is viewable from the web. If you do not run
494 this, there is no guarantee that the url returned by this application
498 if not self.application.checkWeb(self):
501 except UnknownWebPath:
502 raise WebVerificationError
505 def fetch(self, path, post=None):
507 Performs a HTTP request on the website.
509 return util.fetch(self.url.netloc, self.url.path, path, post) # pylint: disable-msg=E1103
511 class WorkingCopy(Deployment):
513 Represents a temporary clone of a deployment that we can make
514 modifications to without fear of interfering with a production
515 deployment. More operations are permitted on these copies.
518 def parametrize(self, deployment):
520 Edits files in ``dir`` to replace WIZARD_* variables with literal
521 instances based on ``deployment``. This is used for constructing
522 virtual merge bases, and as such ``deployment`` will generally not
525 return self.application.parametrize(self, deployment)
527 def prepareConfig(self):
529 Edits files in the deployment such that any user-specific configuration
530 is replaced with generic WIZARD_* variables.
532 return self.application.prepareConfig(self)
534 def resolveConflicts(self):
536 Resolves conflicted files in this working copy. Returns whether or
537 not all conflicted files were resolved or not. Fully resolved
538 files are added to the index, but no commit is made.
540 return self.application.resolveConflicts(self)
542 def prepareMerge(self):
544 Performs various edits to files in the current working directory in
545 order to make a merge go more smoothly. This is usually
546 used to fix botched line-endings.
548 return self.application.prepareMerge(self)
552 class Error(wizard.Error):
553 """Base error class for this module"""
557 class NotMigratedError(Error):
559 The deployment contains a .scripts-version file, but no .git
560 or .wizard directory.
562 #: Directory of deployment
564 def __init__(self, dir):
567 return """This installation was not migrated"""
569 class AlreadyVersionedError(Error):
570 """The deployment contained a .git directory but no .wizard directory."""
571 #: Directory of deployment
573 def __init__(self, dir):
578 ERROR: Directory contains a .git directory, but not
579 a .wizard directory. If this is not a corrupt
580 migration, this means that the user was versioning their
581 install using Git."""
583 class NotConfiguredError(Error):
584 """The install was missing essential configuration."""
585 #: Directory of unconfigured install
587 def __init__(self, dir):
592 ERROR: The install was well-formed, but not configured
593 (essential configuration files were not found.)"""
595 class CorruptedAutoinstallError(Error):
596 """The install was missing a .git directory, but had a .wizard directory."""
597 #: Directory of the corrupted install
599 def __init__(self, dir):
604 ERROR: Directory contains a .wizard directory,
605 but not a .git directory."""
607 class NotAutoinstallError(Error):
608 """Application is not an autoinstall."""
609 #: Directory of the not autoinstall
611 def __init__(self, dir):
620 does not appear to be an autoinstall. If you are in a
621 subdirectory of an autoinstall, you need to use the root
622 directory for the autoinstall.""" % self.dir
624 class NoTagError(Error):
625 """Deployment has a tag that does not have an equivalent in upstream repository."""
628 def __init__(self, tag):
633 ERROR: Could not find tag %s in repository.""" % self.tag
635 class NoLocalTagError(Error):
636 """Could not find tag in local repository."""
639 def __init__(self, tag):
644 ERROR: Could not find tag %s in local repository.""" % self.tag
646 class InconsistentPristineTagError(Error):
647 """Pristine tag commit ID does not match upstream pristine tag commit ID."""
650 def __init__(self, tag):
655 ERROR: Local pristine tag %s did not match repository's. This
656 probably means an upstream rebase occured.""" % self.tag
658 class InconsistentWizardTagError(Error):
659 """Wizard tag commit ID does not match upstream wizard tag commit ID."""
662 def __init__(self, tag):
667 ERROR: Local wizard tag %s did not match repository's. This
668 probably means an upstream rebase occurred.""" % self.tag
670 class HeadNotDescendantError(Error):
671 """HEAD is not connected to tag."""
672 #: Tag that HEAD should have been descendant of.
674 def __init__(self, tag):
679 ERROR: HEAD is not a descendant of %s. This probably
680 means that an upstream rebase occurred, and new tags were
681 pulled, but local user commits were never rebased.""" % self.tag
683 class VersionDetectionError(Error):
684 """Could not detect real version of application."""
688 ERROR: Could not detect the real version of the application."""
690 class VersionMismatchError(Error):
691 """Git version of application does not match detected version."""
696 def __init__(self, real_version, git_version):
697 self.real_version = real_version
698 self.git_version = git_version
702 ERROR: The detected version %s did not match the Git
703 version %s.""" % (self.real_version, self.git_version)
705 class WebVerificationError(Error):
706 """Could not access the application on the web"""
710 ERROR: We were not able to access the application on the
711 web. This may indicate that the website is behind
712 authentication on the htaccess level. You can find
713 the contents of the page from the debug backtraces."""
715 class DatabaseVerificationError(Error):
716 """Could not access the database"""
720 ERROR: We were not able to access the database for
721 this application; this probably means that your database
722 configuration is misconfigured."""
724 class UnknownWebPath(Error):
725 """Could not determine application's web path."""
729 ERROR: We were not able to determine what the application's
730 host and path were in order to perform a web request
731 on the application. You can specify this manually using
732 the WIZARD_WEB_HOST and WIZARD_WEB_PATH environment