2 Object model for querying information and manipulating deployments
3 of autoinstalls. Every :class:`Deployment` has an :class:`ApplicationVersion`
4 which in turn has an :class:`Application`.
10 import distutils.version
15 from wizard import git, old_log, scripts, shell, util
17 ## -- Global Functions --
19 def get_install_lines(versions_store, user=None):
21 Low level function that retrieves a list of lines from the
22 :term:`versions store` that can be passed to :meth:`Deployment.parse`.
24 if os.path.isfile(versions_store):
25 return fileinput.input([versions_store])
27 return fileinput.input([versions_store + "/" + user])
28 return fileinput.input([versions_store + "/" + f for f in sorted(os.listdir(versions_store))])
30 def parse_install_lines(show, versions_store, yield_errors = False, user = None):
32 Generator function for iterating through all autoinstalls.
33 Each item is an instance of :class:`Deployment`, or possibly
34 a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can
35 filter out applications and versions by specifying ``app``
36 or ``app-1.2.3`` in ``show``. This function may generate
41 elif isinstance(show, str):
42 # otherwise, frozenset will treat string as an iterable
43 show = frozenset([show])
45 show = frozenset(show)
46 for line in get_install_lines(versions_store, user):
49 d = Deployment.parse(line)
50 name = d.application.name
51 except NoSuchApplication as e:
56 # we consider this a worse error
57 logging.warning("Error with '%s'" % line.rstrip())
60 if name + "-" + str(d.version) in show or name in show:
67 ## -- Model Objects --
69 class Deployment(object):
71 Represents a deployment of an autoinstall, e.g. directory in a user's
72 web_scripts that has ``.scripts`` directory or ``.scripts-version``
73 file in it. Supply ``version`` with an :class:`ApplicationVersion` only if
74 you were reading from the :term:`versions store` and care about
75 speed (data from there can be stale).
77 #: Absolute path to the deployment
79 def __init__(self, location, version=None):
80 self.location = os.path.abspath(location)
81 self._app_version = version
82 # some cache variables
85 def read(self, file, force = False):
87 Reads a file's contents, possibly from cache unless ``force``
90 if force or file not in self._read_cache:
91 f = open(os.path.join(self.location, file))
92 self._read_cache[file] = f.read()
94 return self._read_cache[file]
97 Extracts all the values of all variables from deployment.
98 These variables may be used for parametrizing generic parent
99 commits and include things such as database access credentials
100 and local configuration.
102 return self.application.extract(self)
103 def parametrize(self, dir):
105 Edits files in ``dir`` to replace WIZARD_* variables with literal
106 instances. This is used for constructing virtual merge bases, and
107 as such dir will generally not equal :attr:`location`.
109 return self.application.parametrize(self, dir)
110 def upgrade(self, version, options):
112 Performs an upgrae of database schemas and other non-versioned data.
114 with util.ChangeDirectory(self.location):
115 return self.application.upgrade(self, version, options)
116 def backup(self, options):
118 Performs a backup of database schemas and other non-versioned data.
120 with util.ChangeDirectory(self.location):
121 return self.application.backup(self, options)
122 def restore(self, backup, options):
124 Restores a backup. Destroys state, so be careful! Also, this does
125 NOT restore the file-level backup, which is what 'wizard restore'
126 does, so you probably do NOT want to call this elsewhere unless
127 you know what you're doing.
129 with util.ChangeDirectory(self.location):
130 return self.application.restore(self, backup, options)
131 def prepareConfig(self):
133 Edits files in the deployment such that any user-specific configuration
134 is replaced with generic WIZARD_* variables.
136 return self.application.prepareConfig(self)
137 def checkConfig(self, deployment):
139 Checks if the application is configured.
145 Checks if this is an autoinstall, throws an exception if there
148 with util.ChangeDirectory(self.location):
149 has_git = os.path.isdir(".git")
150 has_scripts = os.path.isdir(".scripts")
151 if not has_git and has_scripts:
152 raise CorruptedAutoinstallError(self.location)
153 elif has_git and not has_scripts:
154 raise AlreadyVersionedError(self.location)
155 elif not has_git and not has_scripts:
156 if os.path.isfile(".scripts-version"):
157 raise NotMigratedError(self.location)
159 def verifyTag(self, srv_path):
161 Checks if the purported version has a corresponding tag
162 in the upstream repository.
164 repo = self.application.repository(srv_path)
166 shell.Shell().eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--')
167 except shell.CallError:
168 raise NoTagError(self.app_version.scripts_tag)
170 def verifyGit(self, srv_path):
172 Checks if the autoinstall's Git repository makes sense,
173 checking if the tag is parseable and corresponds to
174 a real application, and if the tag in this repository
175 corresponds to the one in the remote repository.
177 with util.ChangeDirectory(self.location):
179 repo = self.application.repository(srv_path)
180 def repo_rev_parse(tag):
181 return sh.eval("git", "--git-dir", repo, "rev-parse", tag)
182 def self_rev_parse(tag):
184 return sh.safeCall("git", "rev-parse", tag, strip=True)
185 except shell.CallError:
186 raise NoLocalTagError(tag)
187 def compare_tags(tag):
188 return repo_rev_parse(tag) == self_rev_parse(tag)
189 if not compare_tags(self.app_version.pristine_tag):
190 raise InconsistentPristineTagError(self.app_version.pristine_tag)
191 if not compare_tags(self.app_version.scripts_tag):
192 raise InconsistentScriptsTagError(self.app_version.scripts_tag)
193 parent = repo_rev_parse(self.app_version.scripts_tag)
194 merge_base = sh.safeCall("git", "merge-base", parent, "HEAD", strip=True)
195 if merge_base != parent:
196 raise HeadNotDescendantError(self.app_version.scripts_tag)
198 def verifyConfigured(self):
200 Checks if the autoinstall is configured running.
202 if not self.configured:
203 raise NotConfiguredError(self.location)
205 def verifyVersion(self):
207 Checks if our version and the version number recorded in a file
210 real = self.application.detectVersion(self)
212 raise VersionDetectionError
213 elif not str(real) == self.app_version.pristine_tag.partition('-')[2]:
214 raise VersionMismatchError(real, self.version)
218 Checks if the autoinstall is viewable from the web.
221 if not self.application.checkWeb(self, out):
222 raise WebVerificationError(out[0])
224 def fetch(self, path, post=None):
226 Performs a HTTP request on the website.
229 host, basepath = scripts.get_web_host_and_path(self.location)
232 return util.fetch(host, basepath, path, post)
235 def configured(self):
236 """Whether or not an autoinstall has been configured/installed for use."""
237 return self.application.checkConfig(self)
240 """Whether or not the autoinstalls has been migrated."""
241 return os.path.isdir(self.scripts_dir)
243 def scripts_dir(self):
244 """The absolute path of the ``.scripts`` directory."""
245 return os.path.join(self.location, '.scripts')
247 def old_version_file(self):
249 The absolute path of either ``.scripts-version`` (for unmigrated
250 installs) or ``.scripts/version``.
254 Use of this is discouraged for migrated installs.
256 return os.path.join(self.location, '.scripts-version')
258 def version_file(self):
259 """The absolute path of the ``.scripts/version`` file."""
260 return os.path.join(self.scripts_dir, 'version')
262 def application(self):
263 """The :class:`Application` of this deployment."""
264 return self.app_version.application
268 The :class:`wizard.old_log.Log` of this deployment. This
269 is only applicable to un-migrated autoinstalls.
271 if not self._old_log:
272 self._old_log = old_log.DeployLog.load(self)
277 The :class:`distutils.version.LooseVersion` of this
280 return self.app_version.version
282 def app_version(self):
283 """The :class:`ApplicationVersion` of this deployment."""
284 if not self._app_version:
285 if os.path.isdir(os.path.join(self.location, ".git")):
287 with util.ChangeDirectory(self.location):
288 appname, _, version = git.describe().partition('-')
289 self._app_version = ApplicationVersion.make(appname, version)
290 except shell.CallError:
292 if not self._app_version:
293 self._app_version = self.old_log[-1].version
294 return self._app_version
298 Parses a line from the :term:`versions store`.
302 Use this method only when speed is of the utmost
303 importance. You should prefer to directly create a deployment
304 with only a ``location`` when possible.
308 location, deploydir = line.split(":")
310 return Deployment(line) # lazy loaded version
312 return Deployment(location, version=ApplicationVersion.parse(deploydir))
314 e.location = location
317 class Application(object):
318 """Represents an application, i.e. mediawiki or phpbb."""
319 #: String name of the application
321 #: Dictionary of version strings to :class:`ApplicationVersion`.
322 #: See also :meth:`makeVersion`.
324 #: List of files that need to be modified when parametrizing.
325 #: This is a class-wide constant, and should not normally be modified.
326 parametrized_files = []
327 def __init__(self, name):
331 self._extractors = {}
332 self._substitutions = {}
333 def repository(self, srv_path):
335 Returns the Git repository that would contain this application.
336 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
338 repo = os.path.join(srv_path, self.name + ".git")
339 if not os.path.isdir(repo):
340 repo = os.path.join(srv_path, self.name, ".git")
341 if not os.path.isdir(repo):
342 raise NoRepositoryError(self.name)
344 def makeVersion(self, version):
346 Creates or retrieves the :class:`ApplicationVersion` singleton for the
349 if version not in self.versions:
350 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
351 return self.versions[version]
352 def extract(self, deployment):
353 """Extracts wizard variables from a deployment."""
355 for k,extractor in self.extractors.items():
356 result[k] = extractor(deployment)
358 def parametrize(self, deployment, dir):
360 Takes a generic source checkout at dir and parametrizes
361 it according to the values of deployment.
363 variables = deployment.extract()
364 for file in self.parametrized_files:
365 fullpath = os.path.join(dir, file)
367 contents = open(fullpath, "r").read()
370 for key, value in variables.items():
371 if value is None: continue
372 contents = contents.replace(key, value)
373 tmp = tempfile.NamedTemporaryFile(delete=False)
375 os.rename(tmp.name, fullpath)
376 def prepareConfig(self, deployment):
378 Takes a deployment and replaces any explicit instances
379 of a configuration variable with generic WIZARD_* constants.
380 There is a sane default implementation built on substitutions;
381 you can override this method to provide arbitrary extra
384 for key, subst in self.substitutions.items():
385 subs = subst(deployment)
386 if not subs and key not in self.deprecated_keys:
387 logging.warning("No substitutions for %s" % key)
388 def install(self, version, options):
390 Run for 'wizard configure' (and, by proxy, 'wizard install')
391 to configure an application. This assumes that the current
392 working directory is a deployment.
395 def upgrade(self, deployment, version, options):
397 Run for 'wizard upgrade' to upgrade database schemas and other
398 non-versioned data in an application. This assumes that
399 the current working directory is the deployment.
402 def backup(self, deployment, options):
404 Run for 'wizard backup' and upgrades to backup database schemas
405 and other non-versioned data in an application. This assumes
406 that the current working directory is the deployment.
409 def restore(self, deployment, backup, options):
411 Run for 'wizard restore' and failed upgrades to restore database
412 and other non-versioned data to a backed up version. This assumes
413 that the current working directory is the deployment.
416 def detectVersion(self, deployment):
418 Checks source files to determine the version manually.
421 def checkWeb(self, deployment, output=None):
423 Checks if the autoinstall is viewable from the web. Output
424 should be an empty list that will get mutated by this function.
428 def extractors(self):
430 Dictionary of variable names to extractor functions. These functions
431 take a :class:`Deployment` as an argument and return the value of
432 the variable, or ``None`` if it could not be found.
433 See also :func:`wizard.app.filename_regex_extractor`.
437 def substitutions(self):
439 Dictionary of variable names to substitution functions. These functions
440 take a :class:`Deployment` as an argument and modify the deployment such
441 that an explicit instance of the variable is released with the generic
442 WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`.
447 """Makes an application, but uses the correct subtype if available."""
449 __import__("wizard.app." + name)
450 return getattr(wizard.app, name).Application(name)
452 return Application(name)
454 class ApplicationVersion(object):
455 """Represents an abstract notion of a version for an application, where
456 ``version`` is a :class:`distutils.version.LooseVersion` and
457 ``application`` is a :class:`Application`."""
458 #: The :class:`distutils.version.LooseVersion` of this instance.
460 #: The :class:`Application` of this instance.
462 def __init__(self, version, application):
463 self.version = version
464 self.application = application
468 Returns the name of the git describe tag for the commit the user is
469 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
471 return "%s-%s" % (self.application, self.version)
473 def scripts_tag(self):
475 Returns the name of the Git tag for this version.
477 end = str(self.version).partition('-scripts')[2].partition('-')[0]
478 return "%s-scripts%s" % (self.pristine_tag, end)
480 def pristine_tag(self):
482 Returns the name of the Git tag for the pristine version corresponding
485 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
487 return cmp(x.version, y.version)
491 Parses a line from the :term:`versions store` and return
492 :class:`ApplicationVersion`.
494 Use this only for cases when speed is of primary importance;
495 the data in version is unreliable and when possible, you should
496 prefer directly instantiating a Deployment and having it query
497 the autoinstall itself for information.
499 The `value` to parse will vary. For old style installs, it
502 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
504 For new style installs, it will look like::
508 name = value.split("/")[-1]
510 if name.find("-") != -1:
511 app, _, version = name.partition("-")
513 # kind of poor, maybe should error. Generally this
514 # will actually result in a not found error
518 raise DeploymentParseError(deploydir)
519 return ApplicationVersion.make(app, version)
521 def make(app, version):
523 Makes/retrieves a singleton :class:`ApplicationVersion` from
524 a``app`` and ``version`` string.
527 # defer to the application for version creation to enforce
529 return applications()[app].makeVersion(version)
531 raise NoSuchApplication(app)
535 class Error(wizard.Error):
536 """Base error class for this module"""
539 class NoSuchApplication(Error):
541 You attempted to reference a :class:`Application` named
542 ``app``, which is not recognized by Wizard.
544 #: The name of the application that does not exist.
546 #: The location of the autoinstall that threw this variable.
547 #: This should be set by error handling code when it is availble.
549 def __init__(self, app):
552 class DeploymentParseError(Error):
554 Could not parse ``value`` from :term:`versions store`.
556 #: The value that failed to parse.
558 #: The location of the autoinstall that threw this variable.
559 #: This should be set by error handling code when it is available.
561 def __init__(self, value):
564 class NoRepositoryError(Error):
566 :class:`Application` does not appear to have a Git repository
567 in the normal location.
569 #: The name of the application that does not have a Git repository.
571 def __init__(self, app):
574 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
576 class NotMigratedError(Error):
578 The deployment contains a .scripts-version file, but no .git
579 or .scripts directory.
581 #: Directory of deployment
583 def __init__(self, dir):
586 return """This installation was not migrated"""
588 class AlreadyVersionedError(Error):
589 """The deployment contained a .git directory but no .scripts directory."""
590 #: Directory of deployment
592 def __init__(self, dir):
597 ERROR: Directory contains a .git directory, but not
598 a .scripts directory. If this is not a corrupt
599 migration, this means that the user was versioning their
600 install using Git."""
602 class NotConfiguredError(Error):
603 """The install was missing essential configuration."""
604 #: Directory of unconfigured install
606 def __init__(self, dir):
611 ERROR: The install was well-formed, but not configured
612 (essential configuration files were not found.)"""
614 class CorruptedAutoinstallError(Error):
615 """The install was missing a .git directory, but had a .scripts directory."""
616 #: Directory of the corrupted install
618 def __init__(self, dir):
623 ERROR: Directory contains a .scripts directory,
624 but not a .git directory."""
626 class NotAutoinstallError(Error):
627 """The directory was not an autoinstall, due to missing .scripts-version file."""
628 #: Directory in question
630 def __init__(self, dir):
635 ERROR: Could not find .scripts-version file. Are you sure
636 this is an autoinstalled application?
639 class NoTagError(Error):
640 """Deployment has a tag that does not have an equivalent in upstream repository."""
643 def __init__(self, tag):
648 ERROR: Could not find tag %s in repository.""" % self.tag
650 class NoLocalTagError(Error):
651 """Could not find tag in local repository."""
654 def __init__(self, tag):
659 ERROR: Could not find tag %s in local repository.""" % self.tag
661 class InconsistentPristineTagError(Error):
662 """Pristine tag commit ID does not match upstream pristine tag commit ID."""
665 def __init__(self, tag):
670 ERROR: Local pristine tag %s did not match repository's. This
671 probably means an upstream rebase occured.""" % self.tag
673 class InconsistentScriptsTagError(Error):
674 """Scripts tag commit ID does not match upstream scripts tag commit ID."""
677 def __init__(self, tag):
682 ERROR: Local scripts tag %s did not match repository's. This
683 probably means an upstream rebase occurred.""" % self.tag
685 class HeadNotDescendantError(Error):
686 """HEAD is not connected to tag."""
687 #: Tag that HEAD should have been descendant of.
689 def __init__(self, tag):
694 ERROR: HEAD is not a descendant of %s. This probably
695 means that an upstream rebase occurred, and new tags were
696 pulled, but local user commits were never rebased.""" % self.tag
698 class VersionDetectionError(Error):
699 """Could not detect real version of application."""
703 ERROR: Could not detect the real version of the application."""
705 class VersionMismatchError(Error):
706 """Git version of application does not match detected version."""
711 def __init__(self, real_version, git_version):
712 self.real_version = real_version
713 self.git_version = git_version
717 ERROR: The detected version %s did not match the Git
718 version %s.""" % (self.real_version, self.git_version)
720 class WebVerificationError(Error):
721 """Could not access the application on the web"""
722 #: Contents of web page access
724 def __init__(self, contents):
725 self.contents = contents
729 ERROR: We were not able to access the application on the
730 web. This may indicate that the website is behind
731 authentication on the htaccess level. The contents
734 %s""" % self.contents
736 class UnknownWebPath(Error):
737 """Could not determine application's web path."""
741 ERROR: We were not able to determine what the application's
742 host and path were in order to perform a web request
743 on the application. You can specify this manually using
744 the WIZARD_WEB_HOST and WIZARD_WEB_PATH environment
747 _application_list = [
748 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
749 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
750 # these are technically deprecated
751 "advancedpoll", "gallery",
756 """Hash table for looking up string application name to instance"""
758 if not _applications:
759 _applications = dict([(n,Application.make(n)) for n in _application_list ])