2 Object model for querying information and manipulating deployments
3 of autoinstalls. Every :class:`Deployment` has an :class:`ApplicationVersion`
4 which in turn has an :class:`Application`.
10 import distutils.version
15 from wizard import git, old_log, scripts, shell, util
17 ## -- Global Functions --
19 def get_install_lines(versions_store, user=None):
21 Low level function that retrieves a list of lines from the
22 :term:`versions store` that can be passed to :meth:`Deployment.parse`.
24 if os.path.isfile(versions_store):
25 return fileinput.input([versions_store])
27 return fileinput.input([versions_store + "/" + user])
28 return fileinput.input([versions_store + "/" + f for f in sorted(os.listdir(versions_store))])
30 def parse_install_lines(show, versions_store, yield_errors = False, user = None):
32 Generator function for iterating through all autoinstalls.
33 Each item is an instance of :class:`Deployment`, or possibly
34 a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can
35 filter out applications and versions by specifying ``app``
36 or ``app-1.2.3`` in ``show``. This function may generate
41 elif isinstance(show, str):
42 # otherwise, frozenset will treat string as an iterable
43 show = frozenset([show])
45 show = frozenset(show)
46 for line in get_install_lines(versions_store, user):
49 d = Deployment.parse(line)
50 name = d.application.name
51 except NoSuchApplication as e:
56 # we consider this a worse error
57 logging.warning("Error with '%s'" % line.rstrip())
60 if name + "-" + str(d.version) in show or name in show:
67 ## -- Model Objects --
69 class Deployment(object):
71 Represents a deployment of an autoinstall, e.g. directory in a user's
72 web_scripts that has ``.scripts`` directory or ``.scripts-version``
73 file in it. Supply ``version`` with an :class:`ApplicationVersion` only if
74 you were reading from the :term:`versions store` and care about
75 speed (data from there can be stale).
77 #: Absolute path to the deployment
79 def __init__(self, location, version=None):
80 self.location = os.path.abspath(location)
81 self._app_version = version
82 # some cache variables
85 def read(self, file, force = False):
87 Reads a file's contents, possibly from cache unless ``force``
90 if force or file not in self._read_cache:
91 f = open(os.path.join(self.location, file))
92 self._read_cache[file] = f.read()
94 return self._read_cache[file]
97 Extracts all the values of all variables from deployment.
98 These variables may be used for parametrizing generic parent
99 commits and include things such as database access credentials
100 and local configuration.
102 return self.application.extract(self)
103 def parametrize(self, dir):
105 Edits files in ``dir`` to replace WIZARD_* variables with literal
106 instances. This is used for constructing virtual merge bases, and
107 as such dir will generally not equal :attr:`location`.
109 return self.application.parametrize(self, dir)
110 def upgrade(self, version, options):
112 Performs an upgrae of database schemas and other non-versioned data.
114 with util.ChangeDirectory(self.location):
115 return self.application.upgrade(self, version, options)
116 def backup(self, options):
118 Performs a backup of database schemas and other non-versioned data.
120 with util.ChangeDirectory(self.location):
121 return self.application.backup(self, options)
122 def restore(self, backup, options):
124 Restores a backup. Destroys state, so be careful! Also, this does
125 NOT restore the file-level backup, which is what 'wizard restore'
126 does, so you probably do NOT want to call this elsewhere unless
127 you know what you're doing.
129 with util.ChangeDirectory(self.location):
130 return self.application.restore(self, backup, options)
131 def prepareConfig(self):
133 Edits files in the deployment such that any user-specific configuration
134 is replaced with generic WIZARD_* variables.
136 return self.application.prepareConfig(self)
137 def checkConfig(self, deployment):
139 Checks if the application is configured.
145 Checks if this is an autoinstall, throws an exception if there
148 with util.ChangeDirectory(self.location):
149 has_git = os.path.isdir(".git")
150 has_scripts = os.path.isdir(".scripts")
151 if not has_git and has_scripts:
152 raise CorruptedAutoinstallError(self.location)
153 elif has_git and not has_scripts:
154 raise AlreadyVersionedError(self.location)
155 elif not has_git and not has_scripts:
156 if os.path.isfile(".scripts-version"):
157 raise NotMigratedError(self.location)
159 def verifyTag(self, srv_path):
161 Checks if the purported version has a corresponding tag
162 in the upstream repository.
164 repo = self.application.repository(srv_path)
166 shell.Shell().eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--')
167 except shell.CallError:
168 raise NoTagError(self.app_version.scripts_tag)
170 def verifyGit(self, srv_path):
172 Checks if the autoinstall's Git repository makes sense,
173 checking if the tag is parseable and corresponds to
174 a real application, and if the tag in this repository
175 corresponds to the one in the remote repository.
177 with util.ChangeDirectory(self.location):
179 repo = self.application.repository(srv_path)
180 def repo_rev_parse(tag):
181 return sh.eval("git", "--git-dir", repo, "rev-parse", tag)
182 def self_rev_parse(tag):
184 return sh.safeCall("git", "rev-parse", tag, strip=True)
185 except shell.CallError:
186 raise NoLocalTagError(tag)
187 def compare_tags(tag):
188 return repo_rev_parse(tag) == self_rev_parse(tag)
189 if not compare_tags(self.app_version.pristine_tag):
190 raise InconsistentPristineTagError(self.app_version.pristine_tag)
191 if not compare_tags(self.app_version.scripts_tag):
192 raise InconsistentScriptsTagError(self.app_version.scripts_tag)
193 parent = repo_rev_parse(self.app_version.scripts_tag)
194 merge_base = sh.safeCall("git", "merge-base", parent, "HEAD", strip=True)
195 if merge_base != parent:
196 raise HeadNotDescendantError(self.app_version.scripts_tag)
198 def verifyConfigured(self):
200 Checks if the autoinstall is configured running.
202 if not self.configured:
203 raise NotConfiguredError(self.location)
205 def verifyVersion(self):
207 Checks if our version and the version number recorded in a file
210 real = self.application.detectVersion(self)
212 raise VersionDetectionError
213 elif not str(real) == self.app_version.pristine_tag.partition('-')[2]:
214 raise VersionMismatchError(real, self.version)
218 Checks if the autoinstall is viewable from the web.
220 if not self.application.checkWeb(self):
221 raise WebVerificationError
223 def fetch(self, path, post=None):
225 Performs a HTTP request on the website.
228 host, basepath = scripts.get_web_host_and_path(self.location)
231 return util.fetch(host, basepath, path, post)
234 def configured(self):
235 """Whether or not an autoinstall has been configured/installed for use."""
236 return self.application.checkConfig(self)
239 """Whether or not the autoinstalls has been migrated."""
240 return os.path.isdir(self.scripts_dir)
242 def scripts_dir(self):
243 """The absolute path of the ``.scripts`` directory."""
244 return os.path.join(self.location, '.scripts')
246 def old_version_file(self):
248 The absolute path of either ``.scripts-version`` (for unmigrated
249 installs) or ``.scripts/version``.
253 Use of this is discouraged for migrated installs.
255 return os.path.join(self.location, '.scripts-version')
257 def version_file(self):
258 """The absolute path of the ``.scripts/version`` file."""
259 return os.path.join(self.scripts_dir, 'version')
261 def application(self):
262 """The :class:`Application` of this deployment."""
263 return self.app_version.application
267 The :class:`wizard.old_log.Log` of this deployment. This
268 is only applicable to un-migrated autoinstalls.
270 if not self._old_log:
271 self._old_log = old_log.DeployLog.load(self)
276 The :class:`distutils.version.LooseVersion` of this
279 return self.app_version.version
281 def app_version(self):
282 """The :class:`ApplicationVersion` of this deployment."""
283 if not self._app_version:
284 if os.path.isdir(os.path.join(self.location, ".git")):
286 with util.ChangeDirectory(self.location):
287 appname, _, version = git.describe().partition('-')
288 self._app_version = ApplicationVersion.make(appname, version)
289 except shell.CallError:
291 if not self._app_version:
292 self._app_version = self.old_log[-1].version
293 return self._app_version
297 Parses a line from the :term:`versions store`.
301 Use this method only when speed is of the utmost
302 importance. You should prefer to directly create a deployment
303 with only a ``location`` when possible.
307 location, deploydir = line.split(":")
309 return Deployment(line) # lazy loaded version
311 return Deployment(location, version=ApplicationVersion.parse(deploydir))
313 e.location = location
316 class Application(object):
317 """Represents an application, i.e. mediawiki or phpbb."""
318 #: String name of the application
320 #: Dictionary of version strings to :class:`ApplicationVersion`.
321 #: See also :meth:`makeVersion`.
323 #: List of files that need to be modified when parametrizing.
324 #: This is a class-wide constant, and should not normally be modified.
325 parametrized_files = []
326 def __init__(self, name):
330 self._extractors = {}
331 self._substitutions = {}
332 def repository(self, srv_path):
334 Returns the Git repository that would contain this application.
335 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
337 repo = os.path.join(srv_path, self.name + ".git")
338 if not os.path.isdir(repo):
339 repo = os.path.join(srv_path, self.name, ".git")
340 if not os.path.isdir(repo):
341 raise NoRepositoryError(self.name)
343 def makeVersion(self, version):
345 Creates or retrieves the :class:`ApplicationVersion` singleton for the
348 if version not in self.versions:
349 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
350 return self.versions[version]
351 def extract(self, deployment):
352 """Extracts wizard variables from a deployment."""
354 for k,extractor in self.extractors.items():
355 result[k] = extractor(deployment)
357 def parametrize(self, deployment, dir):
359 Takes a generic source checkout at dir and parametrizes
360 it according to the values of deployment.
362 variables = deployment.extract()
363 for file in self.parametrized_files:
364 fullpath = os.path.join(dir, file)
366 contents = open(fullpath, "r").read()
369 for key, value in variables.items():
370 if value is None: continue
371 contents = contents.replace(key, value)
372 tmp = tempfile.NamedTemporaryFile(delete=False)
374 os.rename(tmp.name, fullpath)
375 def prepareConfig(self, deployment):
377 Takes a deployment and replaces any explicit instances
378 of a configuration variable with generic WIZARD_* constants.
379 There is a sane default implementation built on substitutions;
380 you can override this method to provide arbitrary extra
383 for key, subst in self.substitutions.items():
384 subs = subst(deployment)
385 if not subs and key not in self.deprecated_keys:
386 logging.warning("No substitutions for %s" % key)
387 def install(self, version, options):
389 Run for 'wizard configure' (and, by proxy, 'wizard install')
390 to configure an application. This assumes that the current
391 working directory is a deployment.
394 def upgrade(self, deployment, version, options):
396 Run for 'wizard upgrade' to upgrade database schemas and other
397 non-versioned data in an application. This assumes that
398 the current working directory is the deployment.
401 def backup(self, deployment, options):
403 Run for 'wizard backup' and upgrades to backup database schemas
404 and other non-versioned data in an application. This assumes
405 that the current working directory is the deployment.
408 def restore(self, deployment, backup, options):
410 Run for 'wizard restore' and failed upgrades to restore database
411 and other non-versioned data to a backed up version. This assumes
412 that the current working directory is the deployment.
415 def detectVersion(self, deployment):
417 Checks source files to determine the version manually.
420 def checkWeb(self, deployment):
422 Checks if the autoinstall is viewable from the web.
426 def extractors(self):
428 Dictionary of variable names to extractor functions. These functions
429 take a :class:`Deployment` as an argument and return the value of
430 the variable, or ``None`` if it could not be found.
431 See also :func:`wizard.app.filename_regex_extractor`.
435 def substitutions(self):
437 Dictionary of variable names to substitution functions. These functions
438 take a :class:`Deployment` as an argument and modify the deployment such
439 that an explicit instance of the variable is released with the generic
440 WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`.
445 """Makes an application, but uses the correct subtype if available."""
447 __import__("wizard.app." + name)
448 return getattr(wizard.app, name).Application(name)
450 return Application(name)
452 class ApplicationVersion(object):
453 """Represents an abstract notion of a version for an application, where
454 ``version`` is a :class:`distutils.version.LooseVersion` and
455 ``application`` is a :class:`Application`."""
456 #: The :class:`distutils.version.LooseVersion` of this instance.
458 #: The :class:`Application` of this instance.
460 def __init__(self, version, application):
461 self.version = version
462 self.application = application
466 Returns the name of the git describe tag for the commit the user is
467 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
469 return "%s-%s" % (self.application, self.version)
471 def scripts_tag(self):
473 Returns the name of the Git tag for this version.
475 end = str(self.version).partition('-scripts')[2].partition('-')[0]
476 return "%s-scripts%s" % (self.pristine_tag, end)
478 def pristine_tag(self):
480 Returns the name of the Git tag for the pristine version corresponding
483 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
485 return cmp(x.version, y.version)
489 Parses a line from the :term:`versions store` and return
490 :class:`ApplicationVersion`.
492 Use this only for cases when speed is of primary importance;
493 the data in version is unreliable and when possible, you should
494 prefer directly instantiating a Deployment and having it query
495 the autoinstall itself for information.
497 The `value` to parse will vary. For old style installs, it
500 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
502 For new style installs, it will look like::
506 name = value.split("/")[-1]
508 if name.find("-") != -1:
509 app, _, version = name.partition("-")
511 # kind of poor, maybe should error. Generally this
512 # will actually result in a not found error
516 raise DeploymentParseError(deploydir)
517 return ApplicationVersion.make(app, version)
519 def make(app, version):
521 Makes/retrieves a singleton :class:`ApplicationVersion` from
522 a``app`` and ``version`` string.
525 # defer to the application for version creation to enforce
527 return applications()[app].makeVersion(version)
529 raise NoSuchApplication(app)
533 class Error(wizard.Error):
534 """Base error class for this module"""
537 class NoSuchApplication(Error):
539 You attempted to reference a :class:`Application` named
540 ``app``, which is not recognized by Wizard.
542 #: The name of the application that does not exist.
544 #: The location of the autoinstall that threw this variable.
545 #: This should be set by error handling code when it is availble.
547 def __init__(self, app):
550 class DeploymentParseError(Error):
552 Could not parse ``value`` from :term:`versions store`.
554 #: The value that failed to parse.
556 #: The location of the autoinstall that threw this variable.
557 #: This should be set by error handling code when it is available.
559 def __init__(self, value):
562 class NoRepositoryError(Error):
564 :class:`Application` does not appear to have a Git repository
565 in the normal location.
567 #: The name of the application that does not have a Git repository.
569 def __init__(self, app):
572 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
574 class NotMigratedError(Error):
576 The deployment contains a .scripts-version file, but no .git
577 or .scripts directory.
579 #: Directory of deployment
581 def __init__(self, dir):
584 return """This installation was not migrated"""
586 class AlreadyVersionedError(Error):
587 """The deployment contained a .git directory but no .scripts directory."""
588 #: Directory of deployment
590 def __init__(self, dir):
595 ERROR: Directory contains a .git directory, but not
596 a .scripts directory. If this is not a corrupt
597 migration, this means that the user was versioning their
598 install using Git."""
600 class NotConfiguredError(Error):
601 """The install was missing essential configuration."""
602 #: Directory of unconfigured install
604 def __init__(self, dir):
609 ERROR: The install was well-formed, but not configured
610 (essential configuration files were not found.)"""
612 class CorruptedAutoinstallError(Error):
613 """The install was missing a .git directory, but had a .scripts directory."""
614 #: Directory of the corrupted install
616 def __init__(self, dir):
621 ERROR: Directory contains a .scripts directory,
622 but not a .git directory."""
624 class NotAutoinstallError(Error):
625 """The directory was not an autoinstall, due to missing .scripts-version file."""
626 #: Directory in question
628 def __init__(self, dir):
633 ERROR: Could not find .scripts-version file. Are you sure
634 this is an autoinstalled application?
637 class NoTagError(Error):
638 """Deployment has a tag that does not have an equivalent in upstream repository."""
641 def __init__(self, tag):
646 ERROR: Could not find tag %s in repository.""" % self.tag
648 class NoLocalTagError(Error):
649 """Could not find tag in local repository."""
652 def __init__(self, tag):
657 ERROR: Could not find tag %s in local repository.""" % self.tag
659 class InconsistentPristineTagError(Error):
660 """Pristine tag commit ID does not match upstream pristine tag commit ID."""
663 def __init__(self, tag):
668 ERROR: Local pristine tag %s did not match repository's. This
669 probably means an upstream rebase occured.""" % self.tag
671 class InconsistentScriptsTagError(Error):
672 """Scripts tag commit ID does not match upstream scripts tag commit ID."""
675 def __init__(self, tag):
680 ERROR: Local scripts tag %s did not match repository's. This
681 probably means an upstream rebase occurred.""" % self.tag
683 class HeadNotDescendantError(Error):
684 """HEAD is not connected to tag."""
685 #: Tag that HEAD should have been descendant of.
687 def __init__(self, tag):
692 ERROR: HEAD is not a descendant of %s. This probably
693 means that an upstream rebase occurred, and new tags were
694 pulled, but local user commits were never rebased.""" % self.tag
696 class VersionDetectionError(Error):
697 """Could not detect real version of application."""
701 ERROR: Could not detect the real version of the application."""
703 class VersionMismatchError(Error):
704 """Git version of application does not match detected version."""
709 def __init__(self, real_version, git_version):
710 self.real_version = real_version
711 self.git_version = git_version
715 ERROR: The detected version %s did not match the Git
716 version %s.""" % (self.real_version, self.git_version)
718 class WebVerificationError(Error):
719 """Could not access the application on the web"""
724 ERROR: We were not able to access the application on the
725 web. This may indicate that the website is behind
726 authentication on the htaccess level."""
728 class UnknownWebPath(Error):
729 """Could not determine application's web path."""
733 ERROR: We were not able to determine what the application's
734 host and path were in order to perform a web request
735 on the application. You can specify this manually using
736 the WIZARD_WEB_HOST and WIZARD_WEB_PATH environment
739 _application_list = [
740 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
741 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
742 # these are technically deprecated
743 "advancedpoll", "gallery",
748 """Hash table for looking up string application name to instance"""
750 if not _applications:
751 _applications = dict([(n,Application.make(n)) for n in _application_list ])