2 Object model for querying information and manipulating deployments
3 of autoinstalls. Every :class:`Deployment` has an :class:`ApplicationVersion`
4 which in turn has an :class:`Application`.
10 import distutils.version
15 from wizard import git, old_log, shell, util
17 ## -- Global Functions --
19 def get_install_lines(versions_store):
21 Low level function that retrieves a list of lines from the
22 :term:`versions store` that can be passed to :meth:`Deployment.parse`.
24 if os.path.isfile(versions_store):
25 return fileinput.input([versions_store])
26 return fileinput.input([versions_store + "/" + f for f in sorted(os.listdir(versions_store))])
28 def parse_install_lines(show, versions_store, yield_errors = False):
30 Generator function for iterating through all autoinstalls.
31 Each item is an instance of :class:`Deployment`, or possibly
32 a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can
33 filter out applications and versions by specifying ``app``
34 or ``app-1.2.3`` in ``show``. This function may generate
35 log output. Make sure that ``show`` is a list and not a string.
37 if not show: show = applications()
38 show = frozenset(show)
39 for line in get_install_lines(versions_store):
42 d = Deployment.parse(line)
43 name = d.application.name
44 except NoSuchApplication as e:
49 # we consider this a worse error
50 logging.warning("Error with '%s'" % line.rstrip())
53 if name + "-" + str(d.version) in show or name in show:
60 ## -- Model Objects --
62 class Deployment(object):
64 Represents a deployment of an autoinstall, e.g. directory in a user's
65 web_scripts that has ``.scripts`` directory or ``.scripts-version``
66 file in it. Supply ``version`` with an :class:`ApplicationVersion` only if
67 you were reading from the :term:`versions store` and care about
68 speed (data from there can be stale).
70 #: Absolute path to the deployment
72 def __init__(self, location, version=None):
73 self.location = os.path.abspath(location)
74 self._app_version = version
75 # some cache variables
78 def read(self, file, force = False):
80 Reads a file's contents, possibly from cache unless ``force``
83 if force or file not in self._read_cache:
84 f = open(os.path.join(self.location, file))
85 self._read_cache[file] = f.read()
87 return self._read_cache[file]
90 Extracts all the values of all variables from deployment.
91 These variables may be used for parametrizing generic parent
92 commits and include things such as database access credentials
93 and local configuration.
95 return self.application.extract(self)
96 def parametrize(self, dir):
98 Edits files in ``dir`` to replace WIZARD_* variables with literal
99 instances. This is used for constructing virtual merge bases, and
100 as such dir will generally not equal :attr:`location`.
102 return self.application.parametrize(self, dir)
103 def prepareConfig(self):
105 Edits files in the deployment such that any user-specific configuration
106 is replaced with generic WIZARD_* variables.
108 return self.application.prepareConfig(self)
109 def checkConfig(self, deployment):
111 Checks if the application is configured.
117 Checks if this is an autoinstall, throws an exception if there
120 with util.ChangeDirectory(self.location):
121 has_git = os.path.isdir(".git")
122 has_scripts = os.path.isdir(".scripts")
123 if not has_git and has_scripts:
124 raise CorruptedAutoinstallError(self.location)
125 elif has_git and not has_scripts:
126 raise AlreadyVersionedError(self.location)
127 elif not has_git and not has_scripts:
128 if os.path.isfile(".scripts-version"):
129 raise NotMigratedError(self.location)
131 def verifyTag(self, srv_path):
133 Checks if the purported version has a corresponding tag
134 in the upstream repository.
136 repo = self.application.repository(srv_path)
138 shell.Shell().eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--')
139 except shell.CallError:
140 raise NoTagError(self.app_version.scripts_tag)
142 def verifyGit(self, srv_path):
144 Checks if the autoinstall's Git repository makes sense,
145 checking if the tag is parseable and corresponds to
146 a real application, and if the tag in this repository
147 corresponds to the one in the remote repository.
149 with util.ChangeDirectory(self.location):
151 repo = self.application.repository(srv_path)
152 def repo_rev_parse(tag):
153 return sh.eval("git", "--git-dir", repo, "rev-parse", tag)
154 def self_rev_parse(tag):
156 return sh.safeCall("git", "rev-parse", tag, strip=True)
157 except shell.CallError:
158 raise NoLocalTagError(tag)
159 def compare_tags(tag):
160 return repo_rev_parse(tag) == self_rev_parse(tag)
161 if not compare_tags(self.app_version.pristine_tag):
162 raise InconsistentPristineTagError(self.app_version.pristine_tag)
163 if not compare_tags(self.app_version.scripts_tag):
164 raise InconsistentScriptsTagError(self.app_version.scripts_tag)
165 parent = repo_rev_parse(self.app_version.scripts_tag)
166 merge_base = sh.safeCall("git", "merge-base", parent, "HEAD", strip=True)
167 if merge_base != parent:
168 raise HeadNotDescendantError(self.app_version.scripts_tag)
170 def verifyConfigured(self):
172 Checks if the autoinstall is configured running.
174 if not self.configured:
175 raise NotConfiguredError(self.location)
177 def verifyVersion(self):
179 Checks if our version and the version number recorded in a file
182 real = self.application.detectVersion(self)
184 raise VersionDetectionError
185 elif not str(real) == self.app_version.pristine_tag.partition('-')[2]:
186 raise VersionMismatchError(real, self.version)
189 def configured(self):
190 """Whether or not an autoinstall has been configured/installed for use."""
191 return self.application.checkConfig(self)
194 """Whether or not the autoinstalls has been migrated."""
195 return os.path.isdir(self.scripts_dir)
197 def scripts_dir(self):
198 """The absolute path of the ``.scripts`` directory."""
199 return os.path.join(self.location, '.scripts')
201 def old_version_file(self):
203 The absolute path of either ``.scripts-version`` (for unmigrated
204 installs) or ``.scripts/version``.
208 Use of this is discouraged for migrated installs.
210 return os.path.join(self.location, '.scripts-version')
212 def version_file(self):
213 """The absolute path of the ``.scripts/version`` file."""
214 return os.path.join(self.scripts_dir, 'version')
216 def application(self):
217 """The :class:`Application` of this deployment."""
218 return self.app_version.application
222 The :class:`wizard.old_log.Log` of this deployment. This
223 is only applicable to un-migrated autoinstalls.
225 if not self._old_log:
226 self._old_log = old_log.DeployLog.load(self)
231 The :class:`distutils.version.LooseVersion` of this
234 return self.app_version.version
236 def app_version(self):
237 """The :class:`ApplicationVersion` of this deployment."""
238 if not self._app_version:
239 if os.path.isdir(os.path.join(self.location, ".git")):
241 with util.ChangeDirectory(self.location):
242 appname, _, version = git.describe().partition('-')
243 self._app_version = ApplicationVersion.make(appname, version)
244 except shell.CallError:
246 if not self._app_version:
247 self._app_version = self.old_log[-1].version
248 return self._app_version
252 Parses a line from the :term:`versions store`.
256 Use this method only when speed is of the utmost
257 importance. You should prefer to directly create a deployment
258 with only a ``location`` when possible.
262 location, deploydir = line.split(":")
264 return Deployment(line) # lazy loaded version
266 return Deployment(location, version=ApplicationVersion.parse(deploydir))
268 e.location = location
271 class Application(object):
272 """Represents an application, i.e. mediawiki or phpbb."""
273 #: String name of the application
275 #: Dictionary of version strings to :class:`ApplicationVersion`.
276 #: See also :meth:`makeVersion`.
278 #: List of files that need to be modified when parametrizing.
279 #: This is a class-wide constant, and should not normally be modified.
280 parametrized_files = []
281 def __init__(self, name):
285 self._extractors = {}
286 self._substitutions = {}
287 def repository(self, srv_path):
289 Returns the Git repository that would contain this application.
290 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
292 repo = os.path.join(srv_path, self.name + ".git")
293 if not os.path.isdir(repo):
294 repo = os.path.join(srv_path, self.name, ".git")
295 if not os.path.isdir(repo):
296 raise NoRepositoryError(self.name)
298 def makeVersion(self, version):
300 Creates or retrieves the :class:`ApplicationVersion` singleton for the
303 if version not in self.versions:
304 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
305 return self.versions[version]
306 def extract(self, deployment):
307 """Extracts wizard variables from a deployment."""
309 for k,extractor in self.extractors.items():
310 result[k] = extractor(deployment)
312 def parametrize(self, deployment, dir):
314 Takes a generic source checkout at dir and parametrizes
315 it according to the values of deployment.
317 variables = deployment.extract()
318 for file in self.parametrized_files:
319 fullpath = os.path.join(dir, file)
321 contents = open(fullpath, "r").read()
324 for key, value in variables.items():
325 if value is None: continue
326 contents = contents.replace(key, value)
327 tmp = tempfile.NamedTemporaryFile(delete=False)
329 os.rename(tmp.name, fullpath)
330 def prepareConfig(self, deployment):
332 Takes a deployment and replaces any explicit instances
333 of a configuration variable with generic WIZARD_* constants.
334 There is a sane default implementation built on substitutions;
335 you can override this method to provide arbitrary extra
338 for key, subst in self.substitutions.items():
339 subs = subst(deployment)
340 if not subs and key not in self.deprecated_keys:
341 logging.warning("No substitutions for %s" % key)
342 def install(self, options):
344 Run for 'wizard configure' (and, by proxy, 'wizard install')
345 to configure an application.
348 def upgrade(self, options):
350 Run for 'wizard upgrade' to upgrade database schemas and other
351 non-versioned data in an application.
354 def detectVersion(self, deployment):
356 Checks source files to determine the version manually.
360 def extractors(self):
362 Dictionary of variable names to extractor functions. These functions
363 take a :class:`Deployment` as an argument and return the value of
364 the variable, or ``None`` if it could not be found.
365 See also :func:`wizard.app.filename_regex_extractor`.
369 def substitutions(self):
371 Dictionary of variable names to substitution functions. These functions
372 take a :class:`Deployment` as an argument and modify the deployment such
373 that an explicit instance of the variable is released with the generic
374 WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`.
379 """Makes an application, but uses the correct subtype if available."""
381 __import__("wizard.app." + name)
382 return getattr(wizard.app, name).Application(name)
384 return Application(name)
386 class ApplicationVersion(object):
387 """Represents an abstract notion of a version for an application, where
388 ``version`` is a :class:`distutils.version.LooseVersion` and
389 ``application`` is a :class:`Application`."""
390 #: The :class:`distutils.version.LooseVersion` of this instance.
392 #: The :class:`Application` of this instance.
394 def __init__(self, version, application):
395 self.version = version
396 self.application = application
400 Returns the name of the git describe tag for the commit the user is
401 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
403 return "%s-%s" % (self.application, self.version)
405 def scripts_tag(self):
407 Returns the name of the Git tag for this version.
409 end = str(self.version).partition('-scripts')[2].partition('-')[0]
410 return "%s-scripts%s" % (self.pristine_tag, end)
412 def pristine_tag(self):
414 Returns the name of the Git tag for the pristine version corresponding
417 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
419 return cmp(x.version, y.version)
423 Parses a line from the :term:`versions store` and return
424 :class:`ApplicationVersion`.
426 Use this only for cases when speed is of primary importance;
427 the data in version is unreliable and when possible, you should
428 prefer directly instantiating a Deployment and having it query
429 the autoinstall itself for information.
431 The `value` to parse will vary. For old style installs, it
434 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
436 For new style installs, it will look like::
440 name = value.split("/")[-1]
442 if name.find("-") != -1:
443 app, _, version = name.partition("-")
445 # kind of poor, maybe should error. Generally this
446 # will actually result in a not found error
450 raise DeploymentParseError(deploydir)
451 return ApplicationVersion.make(app, version)
453 def make(app, version):
455 Makes/retrieves a singleton :class:`ApplicationVersion` from
456 a``app`` and ``version`` string.
459 # defer to the application for version creation to enforce
461 return applications()[app].makeVersion(version)
463 raise NoSuchApplication(app)
467 class Error(wizard.Error):
468 """Base error class for this module"""
471 class NoSuchApplication(Error):
473 You attempted to reference a :class:`Application` named
474 ``app``, which is not recognized by Wizard.
476 #: The name of the application that does not exist.
478 #: The location of the autoinstall that threw this variable.
479 #: This should be set by error handling code when it is availble.
481 def __init__(self, app):
484 class DeploymentParseError(Error):
486 Could not parse ``value`` from :term:`versions store`.
488 #: The value that failed to parse.
490 #: The location of the autoinstall that threw this variable.
491 #: This should be set by error handling code when it is available.
493 def __init__(self, value):
496 class NoRepositoryError(Error):
498 :class:`Application` does not appear to have a Git repository
499 in the normal location.
501 #: The name of the application that does not have a Git repository.
503 def __init__(self, app):
506 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
508 class NotMigratedError(Error):
510 The deployment contains a .scripts-version file, but no .git
511 or .scripts directory.
513 def __init__(self, dir):
516 return """This installation was not migrated"""
518 class AlreadyVersionedError(Error):
519 def __init__(self, dir):
524 ERROR: Directory contains a .git directory, but not
525 a .scripts directory. If this is not a corrupt
526 migration, this means that the user was versioning their
527 install using Git."""
529 class NotConfiguredError(Error):
530 def __init__(self, dir):
535 ERROR: The install was well-formed, but not configured
536 (essential configuration files were not found.)"""
538 class CorruptedAutoinstallError(Error):
539 def __init__(self, dir):
544 ERROR: Directory contains a .scripts directory,
545 but not a .git directory."""
547 class NotAutoinstallError(Error):
548 def __init__(self, dir):
553 ERROR: Could not find .scripts-version file. Are you sure
554 this is an autoinstalled application?
557 class NoTagError(Error):
558 def __init__(self, tag):
563 ERROR: Could not find tag %s in repository.""" % self.tag
565 class NoLocalTagError(Error):
566 def __init__(self, tag):
571 ERROR: Could not find tag %s in local repository.""" % self.tag
573 class InconsistentPristineTagError(Error):
574 def __init__(self, tag):
579 ERROR: Local pristine tag %s did not match repository's. This
580 probably means an upstream rebase occured.""" % self.tag
582 class InconsistentScriptsTagError(Error):
583 def __init__(self, tag):
588 ERROR: Local scripts tag %s did not match repository's. This
589 probably means an upstream rebase occurred.""" % self.tag
591 class HeadNotDescendantError(Error):
592 def __init__(self, tag):
597 ERROR: HEAD is not a descendant of %s. This probably
598 means that an upstream rebase occurred, and new tags were
599 pulled, but local user commits were never rebased.""" % self.tag
601 class VersionDetectionError(Error):
605 ERROR: Could not detect the real version of the application."""
607 class VersionMismatchError(Error):
608 def __init__(self, real_version, git_version):
609 self.real_version = real_version
610 self.git_version = git_version
614 ERROR: The detected version %s did not match the Git
615 version %s.""" % (self.real_version, self.git_version)
617 _application_list = [
618 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
619 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
620 # these are technically deprecated
621 "advancedpoll", "gallery",
626 """Hash table for looking up string application name to instance"""
628 if not _applications:
629 _applications = dict([(n,Application.make(n)) for n in _application_list ])