2 Object model for querying information and manipulating deployments
3 of autoinstalls. Every :class:`Deployment` has an :class:`ApplicationVersion`
4 which in turn has an :class:`Application`.
10 import distutils.version
16 from wizard import git, old_log, scripts, shell, util
18 ## -- Global Functions --
20 def get_install_lines(versions_store, user=None):
22 Low level function that retrieves a list of lines from the
23 :term:`versions store` that can be passed to :meth:`Deployment.parse`.
25 if os.path.isfile(versions_store):
26 return fileinput.input([versions_store])
28 return fileinput.input([versions_store + "/" + user])
29 return fileinput.input([versions_store + "/" + f for f in sorted(os.listdir(versions_store))])
31 def parse_install_lines(show, versions_store, yield_errors = False, user = None):
33 Generator function for iterating through all autoinstalls.
34 Each item is an instance of :class:`Deployment`, or possibly
35 a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can
36 filter out applications and versions by specifying ``app``
37 or ``app-1.2.3`` in ``show``. This function may generate
42 elif isinstance(show, str):
43 # otherwise, frozenset will treat string as an iterable
44 show = frozenset([show])
46 show = frozenset(show)
47 for line in get_install_lines(versions_store, user):
50 d = Deployment.parse(line)
51 name = d.application.name
52 except NoSuchApplication as e:
57 # we consider this a worse error
58 logging.warning("Error with '%s'" % line.rstrip())
61 if name + "-" + str(d.version) in show or name in show:
68 ## -- Model Objects --
70 class Deployment(object):
72 Represents a deployment of an autoinstall, e.g. directory in a user's
73 web_scripts that has ``.scripts`` directory or ``.scripts-version``
74 file in it. Supply ``version`` with an :class:`ApplicationVersion` only if
75 you were reading from the :term:`versions store` and care about
76 speed (data from there can be stale).
78 #: Absolute path to the deployment
80 def __init__(self, location, version=None):
81 self.location = os.path.abspath(location)
82 self._app_version = version
83 # some cache variables
86 def read(self, file, force = False):
88 Reads a file's contents, possibly from cache unless ``force``
91 if force or file not in self._read_cache:
92 f = open(os.path.join(self.location, file))
93 self._read_cache[file] = f.read()
95 return self._read_cache[file]
98 Extracts all the values of all variables from deployment.
99 These variables may be used for parametrizing generic parent
100 commits and include things such as database access credentials
101 and local configuration.
103 return self.application.extract(self)
104 def parametrize(self, dir):
106 Edits files in ``dir`` to replace WIZARD_* variables with literal
107 instances. This is used for constructing virtual merge bases, and
108 as such dir will generally not equal :attr:`location`.
110 return self.application.parametrize(self, dir)
111 def upgrade(self, version, options):
113 Performs an upgrae of database schemas and other non-versioned data.
115 with util.ChangeDirectory(self.location):
116 return self.application.upgrade(self, version, options)
117 def backup(self, options):
119 Performs a backup of database schemas and other non-versioned data.
121 with util.ChangeDirectory(self.location):
122 return self.application.backup(self, options)
123 def restore(self, backup, options):
125 Restores a backup. Destroys state, so be careful! Also, this does
126 NOT restore the file-level backup, which is what 'wizard restore'
127 does, so you probably do NOT want to call this elsewhere unless
128 you know what you're doing.
130 with util.ChangeDirectory(self.location):
131 return self.application.restore(self, backup, options)
132 def prepareConfig(self):
134 Edits files in the deployment such that any user-specific configuration
135 is replaced with generic WIZARD_* variables.
137 return self.application.prepareConfig(self)
138 def checkConfig(self, deployment):
140 Checks if the application is configured.
146 Checks if this is an autoinstall, throws an exception if there
149 with util.ChangeDirectory(self.location):
150 has_git = os.path.isdir(".git")
151 has_scripts = os.path.isdir(".scripts")
152 if not has_git and has_scripts:
153 raise CorruptedAutoinstallError(self.location)
154 elif has_git and not has_scripts:
155 raise AlreadyVersionedError(self.location)
156 elif not has_git and not has_scripts:
157 if os.path.isfile(".scripts-version"):
158 raise NotMigratedError(self.location)
160 def verifyTag(self, srv_path):
162 Checks if the purported version has a corresponding tag
163 in the upstream repository.
165 repo = self.application.repository(srv_path)
167 shell.Shell().eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--')
168 except shell.CallError:
169 raise NoTagError(self.app_version.scripts_tag)
171 def verifyGit(self, srv_path):
173 Checks if the autoinstall's Git repository makes sense,
174 checking if the tag is parseable and corresponds to
175 a real application, and if the tag in this repository
176 corresponds to the one in the remote repository.
178 with util.ChangeDirectory(self.location):
180 repo = self.application.repository(srv_path)
181 def repo_rev_parse(tag):
182 return sh.eval("git", "--git-dir", repo, "rev-parse", tag)
183 def self_rev_parse(tag):
185 return sh.safeCall("git", "rev-parse", tag, strip=True)
186 except shell.CallError:
187 raise NoLocalTagError(tag)
188 def compare_tags(tag):
189 return repo_rev_parse(tag) == self_rev_parse(tag)
190 if not compare_tags(self.app_version.pristine_tag):
191 raise InconsistentPristineTagError(self.app_version.pristine_tag)
192 if not compare_tags(self.app_version.scripts_tag):
193 raise InconsistentScriptsTagError(self.app_version.scripts_tag)
194 parent = repo_rev_parse(self.app_version.scripts_tag)
195 merge_base = sh.safeCall("git", "merge-base", parent, "HEAD", strip=True)
196 if merge_base != parent:
197 raise HeadNotDescendantError(self.app_version.scripts_tag)
199 def verifyConfigured(self):
201 Checks if the autoinstall is configured running.
203 if not self.configured:
204 raise NotConfiguredError(self.location)
206 def verifyVersion(self):
208 Checks if our version and the version number recorded in a file
211 real = self.application.detectVersion(self)
213 raise VersionDetectionError
214 elif not str(real) == self.app_version.pristine_tag.partition('-')[2]:
215 raise VersionMismatchError(real, self.version)
219 Checks if the autoinstall is viewable from the web.
222 if not self.application.checkWeb(self, out):
223 raise WebVerificationError(out[0])
225 def fetch(self, path, post=None):
227 Performs a HTTP request on the website.
230 host, basepath = scripts.get_web_host_and_path(self.location)
231 except (ValueError, TypeError):
233 return util.fetch(host, basepath, path, post)
236 def configured(self):
237 """Whether or not an autoinstall has been configured/installed for use."""
238 return self.application.checkConfig(self)
241 """Whether or not the autoinstalls has been migrated."""
242 return os.path.isdir(self.scripts_dir)
244 def scripts_dir(self):
245 """The absolute path of the ``.scripts`` directory."""
246 return os.path.join(self.location, '.scripts')
248 def old_version_file(self):
250 The absolute path of either ``.scripts-version`` (for unmigrated
251 installs) or ``.scripts/version``.
255 Use of this is discouraged for migrated installs.
257 return os.path.join(self.location, '.scripts-version')
259 def version_file(self):
260 """The absolute path of the ``.scripts/version`` file."""
261 return os.path.join(self.scripts_dir, 'version')
263 def application(self):
264 """The :class:`Application` of this deployment."""
265 return self.app_version.application
269 The :class:`wizard.old_log.Log` of this deployment. This
270 is only applicable to un-migrated autoinstalls.
272 if not self._old_log:
273 self._old_log = old_log.DeployLog.load(self)
278 The :class:`distutils.version.LooseVersion` of this
281 return self.app_version.version
283 def app_version(self):
284 """The :class:`ApplicationVersion` of this deployment."""
285 if not self._app_version:
286 if os.path.isdir(os.path.join(self.location, ".git")):
288 with util.ChangeDirectory(self.location):
289 appname, _, version = git.describe().partition('-')
290 self._app_version = ApplicationVersion.make(appname, version)
291 except shell.CallError:
293 if not self._app_version:
294 self._app_version = self.old_log[-1].version
295 return self._app_version
299 Parses a line from the :term:`versions store`.
303 Use this method only when speed is of the utmost
304 importance. You should prefer to directly create a deployment
305 with only a ``location`` when possible.
309 location, deploydir = line.split(":")
311 return Deployment(line) # lazy loaded version
313 return Deployment(location, version=ApplicationVersion.parse(deploydir))
315 e.location = location
318 class Application(object):
319 """Represents an application, i.e. mediawiki or phpbb."""
320 #: String name of the application
322 #: Dictionary of version strings to :class:`ApplicationVersion`.
323 #: See also :meth:`makeVersion`.
325 #: List of files that need to be modified when parametrizing.
326 #: This is a class-wide constant, and should not normally be modified.
327 parametrized_files = []
328 def __init__(self, name):
332 self._extractors = {}
333 self._substitutions = {}
334 def repository(self, srv_path):
336 Returns the Git repository that would contain this application.
337 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
339 repo = os.path.join(srv_path, self.name + ".git")
340 if not os.path.isdir(repo):
341 repo = os.path.join(srv_path, self.name, ".git")
342 if not os.path.isdir(repo):
343 raise NoRepositoryError(self.name)
345 def makeVersion(self, version):
347 Creates or retrieves the :class:`ApplicationVersion` singleton for the
350 if version not in self.versions:
351 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
352 return self.versions[version]
353 def extract(self, deployment):
354 """Extracts wizard variables from a deployment."""
356 for k,extractor in self.extractors.items():
357 result[k] = extractor(deployment)
359 def parametrize(self, deployment, dir):
361 Takes a generic source checkout at dir and parametrizes
362 it according to the values of deployment.
364 variables = deployment.extract()
365 for file in self.parametrized_files:
366 fullpath = os.path.join(dir, file)
368 contents = open(fullpath, "r").read()
371 for key, value in variables.items():
372 if value is None: continue
373 contents = contents.replace(key, value)
374 f = open(fullpath, "w")
376 def resolveConflicts(self, dir):
378 Takes a directory with conflicted files and attempts to
379 resolve them. Returns whether or not all conflicted
380 files were resolved or not. Fully resolved files are
381 added to the index, but no commit is made.
384 def prepareConfig(self, deployment):
386 Takes a deployment and replaces any explicit instances
387 of a configuration variable with generic WIZARD_* constants.
388 There is a sane default implementation built on substitutions;
389 you can override this method to provide arbitrary extra
392 for key, subst in self.substitutions.items():
393 subs = subst(deployment)
394 if not subs and key not in self.deprecated_keys:
395 logging.warning("No substitutions for %s" % key)
396 def install(self, version, options):
398 Run for 'wizard configure' (and, by proxy, 'wizard install')
399 to configure an application. This assumes that the current
400 working directory is a deployment.
403 def upgrade(self, deployment, version, options):
405 Run for 'wizard upgrade' to upgrade database schemas and other
406 non-versioned data in an application. This assumes that
407 the current working directory is the deployment.
410 def backup(self, deployment, options):
412 Run for 'wizard backup' and upgrades to backup database schemas
413 and other non-versioned data in an application. This assumes
414 that the current working directory is the deployment.
417 def restore(self, deployment, backup, options):
419 Run for 'wizard restore' and failed upgrades to restore database
420 and other non-versioned data to a backed up version. This assumes
421 that the current working directory is the deployment.
424 def detectVersion(self, deployment):
426 Checks source files to determine the version manually.
429 def checkWeb(self, deployment, output=None):
431 Checks if the autoinstall is viewable from the web. Output
432 should be an empty list that will get mutated by this function.
436 def extractors(self):
438 Dictionary of variable names to extractor functions. These functions
439 take a :class:`Deployment` as an argument and return the value of
440 the variable, or ``None`` if it could not be found.
441 See also :func:`wizard.app.filename_regex_extractor`.
445 def substitutions(self):
447 Dictionary of variable names to substitution functions. These functions
448 take a :class:`Deployment` as an argument and modify the deployment such
449 that an explicit instance of the variable is released with the generic
450 WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`.
455 """Makes an application, but uses the correct subtype if available."""
457 __import__("wizard.app." + name)
458 return getattr(wizard.app, name).Application(name)
460 return Application(name)
462 class ApplicationVersion(object):
463 """Represents an abstract notion of a version for an application, where
464 ``version`` is a :class:`distutils.version.LooseVersion` and
465 ``application`` is a :class:`Application`."""
466 #: The :class:`distutils.version.LooseVersion` of this instance.
468 #: The :class:`Application` of this instance.
470 def __init__(self, version, application):
471 self.version = version
472 self.application = application
476 Returns the name of the git describe tag for the commit the user is
477 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
479 return "%s-%s" % (self.application, self.version)
481 def scripts_tag(self):
483 Returns the name of the Git tag for this version.
485 end = str(self.version).partition('-scripts')[2].partition('-')[0]
486 return "%s-scripts%s" % (self.pristine_tag, end)
488 def pristine_tag(self):
490 Returns the name of the Git tag for the pristine version corresponding
493 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
495 return cmp(x.version, y.version)
499 Parses a line from the :term:`versions store` and return
500 :class:`ApplicationVersion`.
502 Use this only for cases when speed is of primary importance;
503 the data in version is unreliable and when possible, you should
504 prefer directly instantiating a Deployment and having it query
505 the autoinstall itself for information.
507 The `value` to parse will vary. For old style installs, it
510 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
512 For new style installs, it will look like::
516 name = value.split("/")[-1]
518 if name.find("-") != -1:
519 app, _, version = name.partition("-")
521 # kind of poor, maybe should error. Generally this
522 # will actually result in a not found error
526 raise DeploymentParseError(deploydir)
527 return ApplicationVersion.make(app, version)
529 def make(app, version):
531 Makes/retrieves a singleton :class:`ApplicationVersion` from
532 a``app`` and ``version`` string.
535 # defer to the application for version creation to enforce
537 return applications()[app].makeVersion(version)
539 raise NoSuchApplication(app)
543 class Error(wizard.Error):
544 """Base error class for this module"""
547 class NoSuchApplication(Error):
549 You attempted to reference a :class:`Application` named
550 ``app``, which is not recognized by Wizard.
552 #: The name of the application that does not exist.
554 #: The location of the autoinstall that threw this variable.
555 #: This should be set by error handling code when it is availble.
557 def __init__(self, app):
560 class DeploymentParseError(Error):
562 Could not parse ``value`` from :term:`versions store`.
564 #: The value that failed to parse.
566 #: The location of the autoinstall that threw this variable.
567 #: This should be set by error handling code when it is available.
569 def __init__(self, value):
572 class NoRepositoryError(Error):
574 :class:`Application` does not appear to have a Git repository
575 in the normal location.
577 #: The name of the application that does not have a Git repository.
579 def __init__(self, app):
582 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
584 class NotMigratedError(Error):
586 The deployment contains a .scripts-version file, but no .git
587 or .scripts directory.
589 #: Directory of deployment
591 def __init__(self, dir):
594 return """This installation was not migrated"""
596 class AlreadyVersionedError(Error):
597 """The deployment contained a .git directory but no .scripts directory."""
598 #: Directory of deployment
600 def __init__(self, dir):
605 ERROR: Directory contains a .git directory, but not
606 a .scripts directory. If this is not a corrupt
607 migration, this means that the user was versioning their
608 install using Git."""
610 class NotConfiguredError(Error):
611 """The install was missing essential configuration."""
612 #: Directory of unconfigured install
614 def __init__(self, dir):
619 ERROR: The install was well-formed, but not configured
620 (essential configuration files were not found.)"""
622 class CorruptedAutoinstallError(Error):
623 """The install was missing a .git directory, but had a .scripts directory."""
624 #: Directory of the corrupted install
626 def __init__(self, dir):
631 ERROR: Directory contains a .scripts directory,
632 but not a .git directory."""
634 class NotAutoinstallError(Error):
635 """The directory was not an autoinstall, due to missing .scripts-version file."""
636 #: Directory in question
638 def __init__(self, dir):
643 ERROR: Could not find .scripts-version file. Are you sure
644 this is an autoinstalled application?
647 class NoTagError(Error):
648 """Deployment has a tag that does not have an equivalent in upstream repository."""
651 def __init__(self, tag):
656 ERROR: Could not find tag %s in repository.""" % self.tag
658 class NoLocalTagError(Error):
659 """Could not find tag in local repository."""
662 def __init__(self, tag):
667 ERROR: Could not find tag %s in local repository.""" % self.tag
669 class InconsistentPristineTagError(Error):
670 """Pristine tag commit ID does not match upstream pristine tag commit ID."""
673 def __init__(self, tag):
678 ERROR: Local pristine tag %s did not match repository's. This
679 probably means an upstream rebase occured.""" % self.tag
681 class InconsistentScriptsTagError(Error):
682 """Scripts tag commit ID does not match upstream scripts tag commit ID."""
685 def __init__(self, tag):
690 ERROR: Local scripts tag %s did not match repository's. This
691 probably means an upstream rebase occurred.""" % self.tag
693 class HeadNotDescendantError(Error):
694 """HEAD is not connected to tag."""
695 #: Tag that HEAD should have been descendant of.
697 def __init__(self, tag):
702 ERROR: HEAD is not a descendant of %s. This probably
703 means that an upstream rebase occurred, and new tags were
704 pulled, but local user commits were never rebased.""" % self.tag
706 class VersionDetectionError(Error):
707 """Could not detect real version of application."""
711 ERROR: Could not detect the real version of the application."""
713 class VersionMismatchError(Error):
714 """Git version of application does not match detected version."""
719 def __init__(self, real_version, git_version):
720 self.real_version = real_version
721 self.git_version = git_version
725 ERROR: The detected version %s did not match the Git
726 version %s.""" % (self.real_version, self.git_version)
728 class WebVerificationError(Error):
729 """Could not access the application on the web"""
730 #: Contents of web page access
732 def __init__(self, contents):
733 self.contents = contents
737 ERROR: We were not able to access the application on the
738 web. This may indicate that the website is behind
739 authentication on the htaccess level. The contents
742 %s""" % self.contents
744 class UnknownWebPath(Error):
745 """Could not determine application's web path."""
749 ERROR: We were not able to determine what the application's
750 host and path were in order to perform a web request
751 on the application. You can specify this manually using
752 the WIZARD_WEB_HOST and WIZARD_WEB_PATH environment
755 _application_list = [
756 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
757 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
758 # these are technically deprecated
759 "advancedpoll", "gallery",
764 """Hash table for looking up string application name to instance"""
766 if not _applications:
767 _applications = dict([(n,Application.make(n)) for n in _application_list ])