2 Object model for querying information and manipulating deployments
3 of autoinstalls. Every :class:`Deployment` has an :class:`ApplicationVersion`
4 which in turn has an :class:`Application`.
10 import distutils.version
18 from wizard import git, old_log, scripts, shell, util
20 ## -- Global Functions --
22 def get_install_lines(versions_store, user=None):
24 Low level function that retrieves a list of lines from the
25 :term:`versions store` that can be passed to :meth:`Deployment.parse`.
27 if os.path.isfile(versions_store):
28 return fileinput.input([versions_store])
30 return fileinput.input([versions_store + "/" + user])
31 return fileinput.input([versions_store + "/" + f for f in sorted(os.listdir(versions_store))])
33 def parse_install_lines(show, versions_store, yield_errors = False, user = None):
35 Generator function for iterating through all autoinstalls.
36 Each item is an instance of :class:`Deployment`, or possibly
37 a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can
38 filter out applications and versions by specifying ``app``
39 or ``app-1.2.3`` in ``show``. This function may generate
44 elif isinstance(show, str):
45 # otherwise, frozenset will treat string as an iterable
46 show = frozenset([show])
48 show = frozenset(show)
49 for line in get_install_lines(versions_store, user):
52 d = Deployment.parse(line)
53 name = d.application.name
54 except NoSuchApplication as e:
59 # we consider this a worse error
60 logging.warning("Error with '%s'" % line.rstrip())
63 if name + "-" + str(d.version) in show or name in show:
70 ## -- Model Objects --
73 def chdir_to_location(f, self, *args, **kwargs):
75 Decorator for making a function have working directory
76 :attr:`Deployment.location`.
78 with util.ChangeDirectory(self.location):
79 return f(self, *args, **kwargs)
81 class Deployment(object):
83 Represents a deployment of an autoinstall, e.g. directory
84 that has ``.scripts`` directory or ``.scripts-version``
85 file in it. Supply ``version`` with an :class:`ApplicationVersion` only if
86 you were reading from the :term:`versions store` and care about
87 speed (data from there can be stale).
89 The Deployment interface is somewhat neutered, so you may
90 want to use :class:`WorkingCopy` or :class:`ProductionCopy` for
91 more powerful operations.
93 #: Absolute path to the deployment
95 def __init__(self, location, version=None):
96 self.location = os.path.abspath(location)
97 self._app_version = version
98 # some cache variables
101 def invalidateCache(self):
103 Invalidates all cached variables. This currently applies to
104 :attr:`app_version`, :attr:`old_log` and :meth:`read`.
106 self._app_version = None
107 self._read_cache = {}
109 def read(self, file, force = False):
111 Reads a file's contents, possibly from cache unless ``force``
114 if force or file not in self._read_cache:
115 f = open(os.path.join(self.location, file))
116 self._read_cache[file] = f.read()
118 return self._read_cache[file]
121 Extracts all the values of all variables from deployment.
122 These variables may be used for parametrizing generic parent
123 commits and include things such as database access credentials
124 and local configuration.
126 return self.application.extract(self)
130 Checks if this is an autoinstall, throws an exception if there
133 with util.ChangeDirectory(self.location):
134 has_git = os.path.isdir(".git")
135 has_scripts = os.path.isdir(".scripts")
136 if not has_git and has_scripts:
137 raise CorruptedAutoinstallError(self.location)
138 elif has_git and not has_scripts:
139 raise AlreadyVersionedError(self.location)
140 elif not has_git and not has_scripts:
141 if os.path.isfile(".scripts-version"):
142 raise NotMigratedError(self.location)
144 def verifyTag(self, srv_path):
146 Checks if the purported version has a corresponding tag
147 in the upstream repository.
149 repo = self.application.repository(srv_path)
151 shell.Shell().eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--')
152 except shell.CallError:
153 raise NoTagError(self.app_version.scripts_tag)
155 def verifyGit(self, srv_path):
157 Checks if the autoinstall's Git repository makes sense,
158 checking if the tag is parseable and corresponds to
159 a real application, and if the tag in this repository
160 corresponds to the one in the remote repository.
162 with util.ChangeDirectory(self.location):
164 repo = self.application.repository(srv_path)
165 def repo_rev_parse(tag):
166 return sh.eval("git", "--git-dir", repo, "rev-parse", tag)
167 def self_rev_parse(tag):
169 return sh.safeCall("git", "rev-parse", tag, strip=True)
170 except shell.CallError:
171 raise NoLocalTagError(tag)
172 def compare_tags(tag):
173 return repo_rev_parse(tag) == self_rev_parse(tag)
174 if not compare_tags(self.app_version.pristine_tag):
175 raise InconsistentPristineTagError(self.app_version.pristine_tag)
176 if not compare_tags(self.app_version.scripts_tag):
177 raise InconsistentScriptsTagError(self.app_version.scripts_tag)
178 parent = repo_rev_parse(self.app_version.scripts_tag)
179 merge_base = sh.safeCall("git", "merge-base", parent, "HEAD", strip=True)
180 if merge_base != parent:
181 raise HeadNotDescendantError(self.app_version.scripts_tag)
183 def verifyConfigured(self):
185 Checks if the autoinstall is configured running.
187 if not self.configured:
188 raise NotConfiguredError(self.location)
191 def verifyVersion(self):
193 Checks if our version and the version number recorded in a file
196 real = self.application.detectVersion(self)
198 raise VersionDetectionError
199 elif not str(real) == self.app_version.pristine_tag.partition('-')[2]:
200 raise VersionMismatchError(real, self.version)
204 def configured(self):
205 """Whether or not an autoinstall has been configured/installed for use."""
206 return self.application.checkConfig(self)
209 """Whether or not the autoinstalls has been migrated."""
210 return os.path.isdir(self.scripts_dir)
212 def scripts_dir(self):
213 """The absolute path of the ``.scripts`` directory."""
214 return os.path.join(self.location, '.scripts')
216 def old_version_file(self):
218 The absolute path of either ``.scripts-version`` (for unmigrated
219 installs) or ``.scripts/version``.
223 Use of this is discouraged for migrated installs.
225 return os.path.join(self.location, '.scripts-version')
227 def version_file(self):
228 """The absolute path of the ``.scripts/version`` file."""
229 return os.path.join(self.scripts_dir, 'version')
231 def application(self):
232 """The :class:`Application` of this deployment."""
233 return self.app_version.application
237 The :class:`wizard.old_log.Log` of this deployment. This
238 is only applicable to un-migrated autoinstalls.
240 if not self._old_log:
241 self._old_log = old_log.DeployLog.load(self)
246 The :class:`distutils.version.LooseVersion` of this
249 return self.app_version.version
251 def app_version(self):
252 """The :class:`ApplicationVersion` of this deployment."""
253 if not self._app_version:
254 if os.path.isdir(os.path.join(self.location, ".git")):
256 with util.ChangeDirectory(self.location):
257 appname, _, version = git.describe().partition('-')
258 self._app_version = ApplicationVersion.make(appname, version)
259 except shell.CallError:
261 if not self._app_version:
262 self._app_version = self.old_log[-1].version
263 return self._app_version
267 Parses a line from the :term:`versions store`.
271 Use this method only when speed is of the utmost
272 importance. You should prefer to directly create a deployment
273 with only a ``location`` when possible.
277 location, deploydir = line.split(":")
279 return ProductionCopy(line) # lazy loaded version
281 return ProductionCopy(location, version=ApplicationVersion.parse(deploydir))
283 e.location = location
286 class ProductionCopy(Deployment):
288 Represents the production copy of a deployment. This copy
289 is canonical, and is the only one guaranteed to be accessible
290 via web, have a database, etc.
293 def upgrade(self, version, options):
295 Performs an upgrade of database schemas and other non-versioned data.
297 return self.application.upgrade(self, version, options)
299 def backup(self, options):
301 Performs a backup of database schemas and other non-versioned data.
303 return self.application.backup(self, options)
305 def restore(self, backup, options):
307 Restores a backup. Destroys state, so be careful! Also, this does
308 NOT restore the file-level backup, which is what 'wizard restore'
309 does, so you probably do NOT want to call this elsewhere unless
310 you know what you're doing (call 'wizard restore' instead).
312 return self.application.restore(self, backup, options)
315 Checks if the autoinstall is viewable from the web.
318 if not self.application.checkWeb(self, out):
319 raise WebVerificationError(out[0])
320 def fetch(self, path, post=None):
322 Performs a HTTP request on the website.
325 host, basepath = scripts.get_web_host_and_path(self.location)
326 except (ValueError, TypeError):
328 return util.fetch(host, basepath, path, post)
330 class WorkingCopy(Deployment):
332 Represents a temporary clone of a deployment that we can make
333 modifications to without fear of interfering with a production
334 deployment. More operations are permitted on these copies.
337 def parametrize(self):
339 Edits files in ``dir`` to replace WIZARD_* variables with literal
340 instances. This is used for constructing virtual merge bases, and
341 as such dir will generally not equal :attr:`location`.
343 return self.application.parametrize(self)
345 def prepareConfig(self):
347 Edits files in the deployment such that any user-specific configuration
348 is replaced with generic WIZARD_* variables.
350 return self.application.prepareConfig(self)
352 def resolveConflicts(self):
354 Resolves conflicted files in this working copy. Returns whether or
355 not all conflicted files were resolved or not. Fully resolved
356 files are added to the index, but no commit is made.
358 return self.application.resolveConflicts(self)
360 def prepareMerge(self):
362 Performs various edits to files in the current working directory in
363 order to make a merge go more smoothly. This is usually
364 used to fix botched line-endings.
366 return self.application.prepareMerge(self)
368 class Application(object):
370 Represents an application, i.e. mediawiki or phpbb.
373 Many of these methods assume a specific working
374 directory; prefer using the corresponding methods
375 in :class:`Deployment` and its subclasses.
377 #: String name of the application
379 #: Dictionary of version strings to :class:`ApplicationVersion`.
380 #: See also :meth:`makeVersion`.
382 #: List of files that need to be modified when parametrizing.
383 #: This is a class-wide constant, and should not normally be modified.
384 parametrized_files = []
385 #: Keys that are used in older versions of the application, but
386 #: not for the most recent version.
388 def __init__(self, name):
392 self._extractors = {}
393 self._substitutions = {}
394 def repository(self, srv_path):
396 Returns the Git repository that would contain this application.
397 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
399 repo = os.path.join(srv_path, self.name + ".git")
400 if not os.path.isdir(repo):
401 repo = os.path.join(srv_path, self.name, ".git")
402 if not os.path.isdir(repo):
403 raise NoRepositoryError(self.name)
405 def makeVersion(self, version):
407 Creates or retrieves the :class:`ApplicationVersion` singleton for the
410 if version not in self.versions:
411 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
412 return self.versions[version]
413 def extract(self, deployment):
414 """Extracts wizard variables from a deployment."""
416 for k,extractor in self.extractors.items():
417 result[k] = extractor(deployment)
419 def parametrize(self, deployment):
421 Takes a generic source checkout and parametrizes
422 it according to the values of deployment. This function
423 operates on the current working directory.
425 variables = deployment.extract()
426 for file in self.parametrized_files:
428 contents = open(file, "r").read()
431 for key, value in variables.items():
432 if value is None: continue
433 contents = contents.replace(key, value)
436 def resolveConflicts(self, deployment):
438 Resolves conflicted files in the current working
439 directory. Returns whether or not all conflicted
440 files were resolved or not. Fully resolved files are
441 added to the index, but no commit is made. By default
442 this is a no-op and returns ``False``.
445 def prepareMerge(self, deployment):
447 Performs various edits to files in the current working directory in
448 order to make a merge go more smoothly. This is usually
449 used to fix botched line-endings. If you add new files,
450 you have to 'git add' them; this is not necessary for edits.
451 By default this is a no-op.
454 def prepareConfig(self, deployment):
456 Takes a deployment and replaces any explicit instances
457 of a configuration variable with generic ``WIZARD_*`` constants.
458 The default implementation uses :attr:`substitutions`;
459 you can override this method to provide arbitrary extra
462 for key, subst in self.substitutions.items():
463 subs = subst(deployment)
464 if not subs and key not in self.deprecated_keys:
465 logging.warning("No substitutions for %s" % key)
466 def install(self, version, options):
468 Run for 'wizard configure' (and, by proxy, 'wizard install')
469 to configure an application. This assumes that the current
470 working directory is a deployment. (This function does not take
471 a :class:`Deployment` as a parameter, as those operations are
475 def upgrade(self, deployment, version, options):
477 Run for 'wizard upgrade' to upgrade database schemas and other
478 non-versioned data in an application. This assumes that
479 the current working directory is the deployment.
482 def backup(self, deployment, options):
484 Run for 'wizard backup' and upgrades to backup database schemas
485 and other non-versioned data in an application. This assumes
486 that the current working directory is the deployment.
489 def restore(self, deployment, backup, options):
491 Run for 'wizard restore' and failed upgrades to restore database
492 and other non-versioned data to a backed up version. This assumes
493 that the current working directory is the deployment.
496 def detectVersion(self, deployment):
498 Checks source files to determine the version manually. This assumes
499 that the current working directory is the deployment.
502 def checkWeb(self, deployment, output=None):
504 Checks if the autoinstall is viewable from the web. To get
505 the HTML source that was retrieved, pass a variable containing
506 an empty list to ``output``; it will be mutated to have its
507 first element be the output.
510 def checkConfig(self, deployment):
512 Checks whether or not an autoinstall has been configured/installed
513 for use. Assumes that the current working directory is the deployment.
517 def extractors(self):
519 Dictionary of variable names to extractor functions. These functions
520 take a :class:`Deployment` as an argument and return the value of
521 the variable, or ``None`` if it could not be found.
522 See also :func:`wizard.app.filename_regex_extractor`.
526 def substitutions(self):
528 Dictionary of variable names to substitution functions. These functions
529 take a :class:`Deployment` as an argument and modify the deployment such
530 that an explicit instance of the variable is released with the generic
531 WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`.
536 """Makes an application, but uses the correct subtype if available."""
538 __import__("wizard.app." + name)
539 return getattr(wizard.app, name).Application(name)
541 return Application(name)
543 class ApplicationVersion(object):
544 """Represents an abstract notion of a version for an application, where
545 ``version`` is a :class:`distutils.version.LooseVersion` and
546 ``application`` is a :class:`Application`."""
547 #: The :class:`distutils.version.LooseVersion` of this instance.
549 #: The :class:`Application` of this instance.
551 def __init__(self, version, application):
552 self.version = version
553 self.application = application
557 Returns the name of the git describe tag for the commit the user is
558 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
560 return "%s-%s" % (self.application, self.version)
562 def scripts_tag(self):
564 Returns the name of the Git tag for this version.
566 end = str(self.version).partition('-scripts')[2].partition('-')[0]
567 return "%s-scripts%s" % (self.pristine_tag, end)
569 def pristine_tag(self):
571 Returns the name of the Git tag for the pristine version corresponding
574 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
575 def __cmp__(self, y):
576 return cmp(self.version, y.version)
580 Parses a line from the :term:`versions store` and return
581 :class:`ApplicationVersion`.
583 Use this only for cases when speed is of primary importance;
584 the data in version is unreliable and when possible, you should
585 prefer directly instantiating a Deployment and having it query
586 the autoinstall itself for information.
588 The `value` to parse will vary. For old style installs, it
591 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
593 For new style installs, it will look like::
597 name = value.split("/")[-1]
599 if name.find("-") != -1:
600 app, _, version = name.partition("-")
602 # kind of poor, maybe should error. Generally this
603 # will actually result in a not found error
607 raise DeploymentParseError(value)
608 return ApplicationVersion.make(app, version)
610 def make(app, version):
612 Makes/retrieves a singleton :class:`ApplicationVersion` from
613 a``app`` and ``version`` string.
616 # defer to the application for version creation to enforce
618 return applications()[app].makeVersion(version)
620 raise NoSuchApplication(app)
624 class Error(wizard.Error):
625 """Base error class for this module"""
628 class NoSuchApplication(Error):
630 You attempted to reference a :class:`Application` named
631 ``app``, which is not recognized by Wizard.
633 #: The name of the application that does not exist.
635 #: The location of the autoinstall that threw this variable.
636 #: This should be set by error handling code when it is availble.
638 def __init__(self, app):
641 class DeploymentParseError(Error):
643 Could not parse ``value`` from :term:`versions store`.
645 #: The value that failed to parse.
647 #: The location of the autoinstall that threw this variable.
648 #: This should be set by error handling code when it is available.
650 def __init__(self, value):
653 class NoRepositoryError(Error):
655 :class:`Application` does not appear to have a Git repository
656 in the normal location.
658 #: The name of the application that does not have a Git repository.
660 def __init__(self, app):
663 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
665 class NotMigratedError(Error):
667 The deployment contains a .scripts-version file, but no .git
668 or .scripts directory.
670 #: Directory of deployment
672 def __init__(self, dir):
675 return """This installation was not migrated"""
677 class AlreadyVersionedError(Error):
678 """The deployment contained a .git directory but no .scripts directory."""
679 #: Directory of deployment
681 def __init__(self, dir):
686 ERROR: Directory contains a .git directory, but not
687 a .scripts directory. If this is not a corrupt
688 migration, this means that the user was versioning their
689 install using Git."""
691 class NotConfiguredError(Error):
692 """The install was missing essential configuration."""
693 #: Directory of unconfigured install
695 def __init__(self, dir):
700 ERROR: The install was well-formed, but not configured
701 (essential configuration files were not found.)"""
703 class CorruptedAutoinstallError(Error):
704 """The install was missing a .git directory, but had a .scripts directory."""
705 #: Directory of the corrupted install
707 def __init__(self, dir):
712 ERROR: Directory contains a .scripts directory,
713 but not a .git directory."""
715 class NotAutoinstallError(Error):
716 """The directory was not an autoinstall, due to missing .scripts-version file."""
717 #: Directory in question
719 def __init__(self, dir):
724 ERROR: Could not find .scripts-version file. Are you sure
725 this is an autoinstalled application?
728 class NoTagError(Error):
729 """Deployment has a tag that does not have an equivalent in upstream repository."""
732 def __init__(self, tag):
737 ERROR: Could not find tag %s in repository.""" % self.tag
739 class NoLocalTagError(Error):
740 """Could not find tag in local repository."""
743 def __init__(self, tag):
748 ERROR: Could not find tag %s in local repository.""" % self.tag
750 class InconsistentPristineTagError(Error):
751 """Pristine tag commit ID does not match upstream pristine tag commit ID."""
754 def __init__(self, tag):
759 ERROR: Local pristine tag %s did not match repository's. This
760 probably means an upstream rebase occured.""" % self.tag
762 class InconsistentScriptsTagError(Error):
763 """Scripts tag commit ID does not match upstream scripts tag commit ID."""
766 def __init__(self, tag):
771 ERROR: Local scripts tag %s did not match repository's. This
772 probably means an upstream rebase occurred.""" % self.tag
774 class HeadNotDescendantError(Error):
775 """HEAD is not connected to tag."""
776 #: Tag that HEAD should have been descendant of.
778 def __init__(self, tag):
783 ERROR: HEAD is not a descendant of %s. This probably
784 means that an upstream rebase occurred, and new tags were
785 pulled, but local user commits were never rebased.""" % self.tag
787 class VersionDetectionError(Error):
788 """Could not detect real version of application."""
792 ERROR: Could not detect the real version of the application."""
794 class VersionMismatchError(Error):
795 """Git version of application does not match detected version."""
800 def __init__(self, real_version, git_version):
801 self.real_version = real_version
802 self.git_version = git_version
806 ERROR: The detected version %s did not match the Git
807 version %s.""" % (self.real_version, self.git_version)
809 class WebVerificationError(Error):
810 """Could not access the application on the web"""
811 #: Contents of web page access
813 def __init__(self, contents):
814 self.contents = contents
818 ERROR: We were not able to access the application on the
819 web. This may indicate that the website is behind
820 authentication on the htaccess level. The contents
823 %s""" % self.contents
825 class UnknownWebPath(Error):
826 """Could not determine application's web path."""
830 ERROR: We were not able to determine what the application's
831 host and path were in order to perform a web request
832 on the application. You can specify this manually using
833 the WIZARD_WEB_HOST and WIZARD_WEB_PATH environment
836 _application_list = [
837 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
838 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
839 # these are technically deprecated
840 "advancedpoll", "gallery",
845 """Hash table for looking up string application name to instance"""
847 if not _applications:
848 _applications = dict([(n,Application.make(n)) for n in _application_list ])