2 Object model for querying information and manipulating deployments
3 of autoinstalls. Every :class:`Deployment` has an :class:`ApplicationVersion`
4 which in turn has an :class:`Application`.
10 import distutils.version
16 from wizard import git, old_log, scripts, shell, util
18 ## -- Global Functions --
20 def get_install_lines(versions_store, user=None):
22 Low level function that retrieves a list of lines from the
23 :term:`versions store` that can be passed to :meth:`Deployment.parse`.
25 if os.path.isfile(versions_store):
26 return fileinput.input([versions_store])
28 return fileinput.input([versions_store + "/" + user])
29 return fileinput.input([versions_store + "/" + f for f in sorted(os.listdir(versions_store))])
31 def parse_install_lines(show, versions_store, yield_errors = False, user = None):
33 Generator function for iterating through all autoinstalls.
34 Each item is an instance of :class:`Deployment`, or possibly
35 a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can
36 filter out applications and versions by specifying ``app``
37 or ``app-1.2.3`` in ``show``. This function may generate
42 elif isinstance(show, str):
43 # otherwise, frozenset will treat string as an iterable
44 show = frozenset([show])
46 show = frozenset(show)
47 for line in get_install_lines(versions_store, user):
50 d = Deployment.parse(line)
51 name = d.application.name
52 except NoSuchApplication as e:
57 # we consider this a worse error
58 logging.warning("Error with '%s'" % line.rstrip())
61 if name + "-" + str(d.version) in show or name in show:
68 ## -- Model Objects --
70 class Deployment(object):
72 Represents a deployment of an autoinstall, e.g. directory in a user's
73 web_scripts that has ``.scripts`` directory or ``.scripts-version``
74 file in it. Supply ``version`` with an :class:`ApplicationVersion` only if
75 you were reading from the :term:`versions store` and care about
76 speed (data from there can be stale).
78 #: Absolute path to the deployment
80 def __init__(self, location, version=None):
81 self.location = os.path.abspath(location)
82 self._app_version = version
83 # some cache variables
86 def read(self, file, force = False):
88 Reads a file's contents, possibly from cache unless ``force``
91 if force or file not in self._read_cache:
92 f = open(os.path.join(self.location, file))
93 self._read_cache[file] = f.read()
95 return self._read_cache[file]
98 Extracts all the values of all variables from deployment.
99 These variables may be used for parametrizing generic parent
100 commits and include things such as database access credentials
101 and local configuration.
103 return self.application.extract(self)
104 def parametrize(self, dir):
106 Edits files in ``dir`` to replace WIZARD_* variables with literal
107 instances. This is used for constructing virtual merge bases, and
108 as such dir will generally not equal :attr:`location`.
110 return self.application.parametrize(self, dir)
111 def upgrade(self, version, options):
113 Performs an upgrae of database schemas and other non-versioned data.
115 with util.ChangeDirectory(self.location):
116 return self.application.upgrade(self, version, options)
117 def backup(self, options):
119 Performs a backup of database schemas and other non-versioned data.
121 with util.ChangeDirectory(self.location):
122 return self.application.backup(self, options)
123 def restore(self, backup, options):
125 Restores a backup. Destroys state, so be careful! Also, this does
126 NOT restore the file-level backup, which is what 'wizard restore'
127 does, so you probably do NOT want to call this elsewhere unless
128 you know what you're doing.
130 with util.ChangeDirectory(self.location):
131 return self.application.restore(self, backup, options)
132 def prepareConfig(self):
134 Edits files in the deployment such that any user-specific configuration
135 is replaced with generic WIZARD_* variables.
137 return self.application.prepareConfig(self)
138 def checkConfig(self, deployment):
140 Checks if the application is configured.
146 Checks if this is an autoinstall, throws an exception if there
149 with util.ChangeDirectory(self.location):
150 has_git = os.path.isdir(".git")
151 has_scripts = os.path.isdir(".scripts")
152 if not has_git and has_scripts:
153 raise CorruptedAutoinstallError(self.location)
154 elif has_git and not has_scripts:
155 raise AlreadyVersionedError(self.location)
156 elif not has_git and not has_scripts:
157 if os.path.isfile(".scripts-version"):
158 raise NotMigratedError(self.location)
160 def verifyTag(self, srv_path):
162 Checks if the purported version has a corresponding tag
163 in the upstream repository.
165 repo = self.application.repository(srv_path)
167 shell.Shell().eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--')
168 except shell.CallError:
169 raise NoTagError(self.app_version.scripts_tag)
171 def verifyGit(self, srv_path):
173 Checks if the autoinstall's Git repository makes sense,
174 checking if the tag is parseable and corresponds to
175 a real application, and if the tag in this repository
176 corresponds to the one in the remote repository.
178 with util.ChangeDirectory(self.location):
180 repo = self.application.repository(srv_path)
181 def repo_rev_parse(tag):
182 return sh.eval("git", "--git-dir", repo, "rev-parse", tag)
183 def self_rev_parse(tag):
185 return sh.safeCall("git", "rev-parse", tag, strip=True)
186 except shell.CallError:
187 raise NoLocalTagError(tag)
188 def compare_tags(tag):
189 return repo_rev_parse(tag) == self_rev_parse(tag)
190 if not compare_tags(self.app_version.pristine_tag):
191 raise InconsistentPristineTagError(self.app_version.pristine_tag)
192 if not compare_tags(self.app_version.scripts_tag):
193 raise InconsistentScriptsTagError(self.app_version.scripts_tag)
194 parent = repo_rev_parse(self.app_version.scripts_tag)
195 merge_base = sh.safeCall("git", "merge-base", parent, "HEAD", strip=True)
196 if merge_base != parent:
197 raise HeadNotDescendantError(self.app_version.scripts_tag)
199 def verifyConfigured(self):
201 Checks if the autoinstall is configured running.
203 if not self.configured:
204 raise NotConfiguredError(self.location)
206 def verifyVersion(self):
208 Checks if our version and the version number recorded in a file
211 real = self.application.detectVersion(self)
213 raise VersionDetectionError
214 elif not str(real) == self.app_version.pristine_tag.partition('-')[2]:
215 raise VersionMismatchError(real, self.version)
219 Checks if the autoinstall is viewable from the web.
222 if not self.application.checkWeb(self, out):
223 raise WebVerificationError(out[0])
225 def fetch(self, path, post=None):
227 Performs a HTTP request on the website.
230 host, basepath = scripts.get_web_host_and_path(self.location)
231 except (ValueError, TypeError):
233 return util.fetch(host, basepath, path, post)
236 def configured(self):
237 """Whether or not an autoinstall has been configured/installed for use."""
238 return self.application.checkConfig(self)
241 """Whether or not the autoinstalls has been migrated."""
242 return os.path.isdir(self.scripts_dir)
244 def scripts_dir(self):
245 """The absolute path of the ``.scripts`` directory."""
246 return os.path.join(self.location, '.scripts')
248 def old_version_file(self):
250 The absolute path of either ``.scripts-version`` (for unmigrated
251 installs) or ``.scripts/version``.
255 Use of this is discouraged for migrated installs.
257 return os.path.join(self.location, '.scripts-version')
259 def version_file(self):
260 """The absolute path of the ``.scripts/version`` file."""
261 return os.path.join(self.scripts_dir, 'version')
263 def application(self):
264 """The :class:`Application` of this deployment."""
265 return self.app_version.application
269 The :class:`wizard.old_log.Log` of this deployment. This
270 is only applicable to un-migrated autoinstalls.
272 if not self._old_log:
273 self._old_log = old_log.DeployLog.load(self)
278 The :class:`distutils.version.LooseVersion` of this
281 return self.app_version.version
283 def app_version(self):
284 """The :class:`ApplicationVersion` of this deployment."""
285 if not self._app_version:
286 if os.path.isdir(os.path.join(self.location, ".git")):
288 with util.ChangeDirectory(self.location):
289 appname, _, version = git.describe().partition('-')
290 self._app_version = ApplicationVersion.make(appname, version)
291 except shell.CallError:
293 if not self._app_version:
294 self._app_version = self.old_log[-1].version
295 return self._app_version
299 Parses a line from the :term:`versions store`.
303 Use this method only when speed is of the utmost
304 importance. You should prefer to directly create a deployment
305 with only a ``location`` when possible.
309 location, deploydir = line.split(":")
311 return Deployment(line) # lazy loaded version
313 return Deployment(location, version=ApplicationVersion.parse(deploydir))
315 e.location = location
318 class Application(object):
319 """Represents an application, i.e. mediawiki or phpbb."""
320 #: String name of the application
322 #: Dictionary of version strings to :class:`ApplicationVersion`.
323 #: See also :meth:`makeVersion`.
325 #: List of files that need to be modified when parametrizing.
326 #: This is a class-wide constant, and should not normally be modified.
327 parametrized_files = []
328 def __init__(self, name):
332 self._extractors = {}
333 self._substitutions = {}
334 def repository(self, srv_path):
336 Returns the Git repository that would contain this application.
337 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
339 repo = os.path.join(srv_path, self.name + ".git")
340 if not os.path.isdir(repo):
341 repo = os.path.join(srv_path, self.name, ".git")
342 if not os.path.isdir(repo):
343 raise NoRepositoryError(self.name)
345 def makeVersion(self, version):
347 Creates or retrieves the :class:`ApplicationVersion` singleton for the
350 if version not in self.versions:
351 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
352 return self.versions[version]
353 def extract(self, deployment):
354 """Extracts wizard variables from a deployment."""
356 for k,extractor in self.extractors.items():
357 result[k] = extractor(deployment)
359 def parametrize(self, deployment, dir):
361 Takes a generic source checkout at dir and parametrizes
362 it according to the values of deployment.
364 variables = deployment.extract()
365 for file in self.parametrized_files:
366 fullpath = os.path.join(dir, file)
368 contents = open(fullpath, "r").read()
371 for key, value in variables.items():
372 if value is None: continue
373 contents = contents.replace(key, value)
374 f = open(fullpath, "w")
376 def resolveConflicts(self, dir):
378 Takes a directory with conflicted files and attempts to
379 resolve them. Returns whether or not all conflicted
380 files were resolved or not. Fully resolved files are
381 added to the index, but no commit is made.
384 def prepareMerge(self, dir):
386 Takes a directory and performs various edits to files in
387 order to make a merge go more smoothly. This is usually
388 used to fix botched line-endings. If you add new files,
389 you have to 'git add' them; this is not necessary for edits.
392 def prepareConfig(self, deployment):
394 Takes a deployment and replaces any explicit instances
395 of a configuration variable with generic WIZARD_* constants.
396 There is a sane default implementation built on substitutions;
397 you can override this method to provide arbitrary extra
400 for key, subst in self.substitutions.items():
401 subs = subst(deployment)
402 if not subs and key not in self.deprecated_keys:
403 logging.warning("No substitutions for %s" % key)
404 def install(self, version, options):
406 Run for 'wizard configure' (and, by proxy, 'wizard install')
407 to configure an application. This assumes that the current
408 working directory is a deployment.
411 def upgrade(self, deployment, version, options):
413 Run for 'wizard upgrade' to upgrade database schemas and other
414 non-versioned data in an application. This assumes that
415 the current working directory is the deployment.
418 def backup(self, deployment, options):
420 Run for 'wizard backup' and upgrades to backup database schemas
421 and other non-versioned data in an application. This assumes
422 that the current working directory is the deployment.
425 def restore(self, deployment, backup, options):
427 Run for 'wizard restore' and failed upgrades to restore database
428 and other non-versioned data to a backed up version. This assumes
429 that the current working directory is the deployment.
432 def detectVersion(self, deployment):
434 Checks source files to determine the version manually.
437 def checkWeb(self, deployment, output=None):
439 Checks if the autoinstall is viewable from the web. Output
440 should be an empty list that will get mutated by this function.
444 def extractors(self):
446 Dictionary of variable names to extractor functions. These functions
447 take a :class:`Deployment` as an argument and return the value of
448 the variable, or ``None`` if it could not be found.
449 See also :func:`wizard.app.filename_regex_extractor`.
453 def substitutions(self):
455 Dictionary of variable names to substitution functions. These functions
456 take a :class:`Deployment` as an argument and modify the deployment such
457 that an explicit instance of the variable is released with the generic
458 WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`.
463 """Makes an application, but uses the correct subtype if available."""
465 __import__("wizard.app." + name)
466 return getattr(wizard.app, name).Application(name)
468 return Application(name)
470 class ApplicationVersion(object):
471 """Represents an abstract notion of a version for an application, where
472 ``version`` is a :class:`distutils.version.LooseVersion` and
473 ``application`` is a :class:`Application`."""
474 #: The :class:`distutils.version.LooseVersion` of this instance.
476 #: The :class:`Application` of this instance.
478 def __init__(self, version, application):
479 self.version = version
480 self.application = application
484 Returns the name of the git describe tag for the commit the user is
485 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
487 return "%s-%s" % (self.application, self.version)
489 def scripts_tag(self):
491 Returns the name of the Git tag for this version.
493 end = str(self.version).partition('-scripts')[2].partition('-')[0]
494 return "%s-scripts%s" % (self.pristine_tag, end)
496 def pristine_tag(self):
498 Returns the name of the Git tag for the pristine version corresponding
501 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
503 return cmp(x.version, y.version)
507 Parses a line from the :term:`versions store` and return
508 :class:`ApplicationVersion`.
510 Use this only for cases when speed is of primary importance;
511 the data in version is unreliable and when possible, you should
512 prefer directly instantiating a Deployment and having it query
513 the autoinstall itself for information.
515 The `value` to parse will vary. For old style installs, it
518 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
520 For new style installs, it will look like::
524 name = value.split("/")[-1]
526 if name.find("-") != -1:
527 app, _, version = name.partition("-")
529 # kind of poor, maybe should error. Generally this
530 # will actually result in a not found error
534 raise DeploymentParseError(deploydir)
535 return ApplicationVersion.make(app, version)
537 def make(app, version):
539 Makes/retrieves a singleton :class:`ApplicationVersion` from
540 a``app`` and ``version`` string.
543 # defer to the application for version creation to enforce
545 return applications()[app].makeVersion(version)
547 raise NoSuchApplication(app)
551 class Error(wizard.Error):
552 """Base error class for this module"""
555 class NoSuchApplication(Error):
557 You attempted to reference a :class:`Application` named
558 ``app``, which is not recognized by Wizard.
560 #: The name of the application that does not exist.
562 #: The location of the autoinstall that threw this variable.
563 #: This should be set by error handling code when it is availble.
565 def __init__(self, app):
568 class DeploymentParseError(Error):
570 Could not parse ``value`` from :term:`versions store`.
572 #: The value that failed to parse.
574 #: The location of the autoinstall that threw this variable.
575 #: This should be set by error handling code when it is available.
577 def __init__(self, value):
580 class NoRepositoryError(Error):
582 :class:`Application` does not appear to have a Git repository
583 in the normal location.
585 #: The name of the application that does not have a Git repository.
587 def __init__(self, app):
590 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
592 class NotMigratedError(Error):
594 The deployment contains a .scripts-version file, but no .git
595 or .scripts directory.
597 #: Directory of deployment
599 def __init__(self, dir):
602 return """This installation was not migrated"""
604 class AlreadyVersionedError(Error):
605 """The deployment contained a .git directory but no .scripts directory."""
606 #: Directory of deployment
608 def __init__(self, dir):
613 ERROR: Directory contains a .git directory, but not
614 a .scripts directory. If this is not a corrupt
615 migration, this means that the user was versioning their
616 install using Git."""
618 class NotConfiguredError(Error):
619 """The install was missing essential configuration."""
620 #: Directory of unconfigured install
622 def __init__(self, dir):
627 ERROR: The install was well-formed, but not configured
628 (essential configuration files were not found.)"""
630 class CorruptedAutoinstallError(Error):
631 """The install was missing a .git directory, but had a .scripts directory."""
632 #: Directory of the corrupted install
634 def __init__(self, dir):
639 ERROR: Directory contains a .scripts directory,
640 but not a .git directory."""
642 class NotAutoinstallError(Error):
643 """The directory was not an autoinstall, due to missing .scripts-version file."""
644 #: Directory in question
646 def __init__(self, dir):
651 ERROR: Could not find .scripts-version file. Are you sure
652 this is an autoinstalled application?
655 class NoTagError(Error):
656 """Deployment has a tag that does not have an equivalent in upstream repository."""
659 def __init__(self, tag):
664 ERROR: Could not find tag %s in repository.""" % self.tag
666 class NoLocalTagError(Error):
667 """Could not find tag in local repository."""
670 def __init__(self, tag):
675 ERROR: Could not find tag %s in local repository.""" % self.tag
677 class InconsistentPristineTagError(Error):
678 """Pristine tag commit ID does not match upstream pristine tag commit ID."""
681 def __init__(self, tag):
686 ERROR: Local pristine tag %s did not match repository's. This
687 probably means an upstream rebase occured.""" % self.tag
689 class InconsistentScriptsTagError(Error):
690 """Scripts tag commit ID does not match upstream scripts tag commit ID."""
693 def __init__(self, tag):
698 ERROR: Local scripts tag %s did not match repository's. This
699 probably means an upstream rebase occurred.""" % self.tag
701 class HeadNotDescendantError(Error):
702 """HEAD is not connected to tag."""
703 #: Tag that HEAD should have been descendant of.
705 def __init__(self, tag):
710 ERROR: HEAD is not a descendant of %s. This probably
711 means that an upstream rebase occurred, and new tags were
712 pulled, but local user commits were never rebased.""" % self.tag
714 class VersionDetectionError(Error):
715 """Could not detect real version of application."""
719 ERROR: Could not detect the real version of the application."""
721 class VersionMismatchError(Error):
722 """Git version of application does not match detected version."""
727 def __init__(self, real_version, git_version):
728 self.real_version = real_version
729 self.git_version = git_version
733 ERROR: The detected version %s did not match the Git
734 version %s.""" % (self.real_version, self.git_version)
736 class WebVerificationError(Error):
737 """Could not access the application on the web"""
738 #: Contents of web page access
740 def __init__(self, contents):
741 self.contents = contents
745 ERROR: We were not able to access the application on the
746 web. This may indicate that the website is behind
747 authentication on the htaccess level. The contents
750 %s""" % self.contents
752 class UnknownWebPath(Error):
753 """Could not determine application's web path."""
757 ERROR: We were not able to determine what the application's
758 host and path were in order to perform a web request
759 on the application. You can specify this manually using
760 the WIZARD_WEB_HOST and WIZARD_WEB_PATH environment
763 _application_list = [
764 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
765 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
766 # these are technically deprecated
767 "advancedpoll", "gallery",
772 """Hash table for looking up string application name to instance"""
774 if not _applications:
775 _applications = dict([(n,Application.make(n)) for n in _application_list ])