""" Object model for querying information and manipulating deployments of autoinstalls. Every :class:`Deployment` has an :class:`ApplicationVersion` which in turn has an :class:`Application`. """ import os.path import fileinput import dateutil.parser import distutils.version import tempfile import logging import wizard from wizard import git, old_log, shell, util ## -- Global Functions -- def get_install_lines(versions_store): """ Low level function that retrieves a list of lines from the :term:`versions store` that can be passed to :meth:`Deployment.parse`. """ if os.path.isfile(versions_store): return fileinput.input([versions_store]) return fileinput.input([versions_store + "/" + f for f in sorted(os.listdir(versions_store))]) def parse_install_lines(show, versions_store, yield_errors = False): """ Generator function for iterating through all autoinstalls. Each item is an instance of :class:`Deployment`, or possibly a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can filter out applications and versions by specifying ``app`` or ``app-1.2.3`` in ``show``. This function may generate log output. """ if not show: show = applications() elif isinstance(show, str): # otherwise, frozenset will treat string as an iterable show = frozenset([show]) else: show = frozenset(show) for line in get_install_lines(versions_store): # construction try: d = Deployment.parse(line) name = d.application.name except NoSuchApplication as e: if yield_errors: yield e continue except Error: # we consider this a worse error logging.warning("Error with '%s'" % line.rstrip()) continue # filter if name + "-" + str(d.version) in show or name in show: pass else: continue # yield yield d ## -- Model Objects -- class Deployment(object): """ Represents a deployment of an autoinstall, e.g. directory in a user's web_scripts that has ``.scripts`` directory or ``.scripts-version`` file in it. Supply ``version`` with an :class:`ApplicationVersion` only if you were reading from the :term:`versions store` and care about speed (data from there can be stale). """ #: Absolute path to the deployment location = None def __init__(self, location, version=None): self.location = os.path.abspath(location) self._app_version = version # some cache variables self._read_cache = {} self._old_log = None def read(self, file, force = False): """ Reads a file's contents, possibly from cache unless ``force`` is ``True``. """ if force or file not in self._read_cache: f = open(os.path.join(self.location, file)) self._read_cache[file] = f.read() f.close() return self._read_cache[file] def extract(self): """ Extracts all the values of all variables from deployment. These variables may be used for parametrizing generic parent commits and include things such as database access credentials and local configuration. """ return self.application.extract(self) def parametrize(self, dir): """ Edits files in ``dir`` to replace WIZARD_* variables with literal instances. This is used for constructing virtual merge bases, and as such dir will generally not equal :attr:`location`. """ return self.application.parametrize(self, dir) def prepareConfig(self): """ Edits files in the deployment such that any user-specific configuration is replaced with generic WIZARD_* variables. """ return self.application.prepareConfig(self) def checkConfig(self, deployment): """ Checks if the application is configured. """ raise NotImplemented def verify(self): """ Checks if this is an autoinstall, throws an exception if there are problems. """ with util.ChangeDirectory(self.location): has_git = os.path.isdir(".git") has_scripts = os.path.isdir(".scripts") if not has_git and has_scripts: raise CorruptedAutoinstallError(self.location) elif has_git and not has_scripts: raise AlreadyVersionedError(self.location) elif not has_git and not has_scripts: if os.path.isfile(".scripts-version"): raise NotMigratedError(self.location) def verifyTag(self, srv_path): """ Checks if the purported version has a corresponding tag in the upstream repository. """ repo = self.application.repository(srv_path) try: shell.Shell().eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--') except shell.CallError: raise NoTagError(self.app_version.scripts_tag) def verifyGit(self, srv_path): """ Checks if the autoinstall's Git repository makes sense, checking if the tag is parseable and corresponds to a real application, and if the tag in this repository corresponds to the one in the remote repository. """ with util.ChangeDirectory(self.location): sh = shell.Shell() repo = self.application.repository(srv_path) def repo_rev_parse(tag): return sh.eval("git", "--git-dir", repo, "rev-parse", tag) def self_rev_parse(tag): try: return sh.safeCall("git", "rev-parse", tag, strip=True) except shell.CallError: raise NoLocalTagError(tag) def compare_tags(tag): return repo_rev_parse(tag) == self_rev_parse(tag) if not compare_tags(self.app_version.pristine_tag): raise InconsistentPristineTagError(self.app_version.pristine_tag) if not compare_tags(self.app_version.scripts_tag): raise InconsistentScriptsTagError(self.app_version.scripts_tag) parent = repo_rev_parse(self.app_version.scripts_tag) merge_base = sh.safeCall("git", "merge-base", parent, "HEAD", strip=True) if merge_base != parent: raise HeadNotDescendantError(self.app_version.scripts_tag) def verifyConfigured(self): """ Checks if the autoinstall is configured running. """ if not self.configured: raise NotConfiguredError(self.location) def verifyVersion(self): """ Checks if our version and the version number recorded in a file are consistent. """ real = self.application.detectVersion(self) if not real: raise VersionDetectionError elif not str(real) == self.app_version.pristine_tag.partition('-')[2]: raise VersionMismatchError(real, self.version) @property def configured(self): """Whether or not an autoinstall has been configured/installed for use.""" return self.application.checkConfig(self) @property def migrated(self): """Whether or not the autoinstalls has been migrated.""" return os.path.isdir(self.scripts_dir) @property def scripts_dir(self): """The absolute path of the ``.scripts`` directory.""" return os.path.join(self.location, '.scripts') @property def old_version_file(self): """ The absolute path of either ``.scripts-version`` (for unmigrated installs) or ``.scripts/version``. .. note:: Use of this is discouraged for migrated installs. """ return os.path.join(self.location, '.scripts-version') @property def version_file(self): """The absolute path of the ``.scripts/version`` file.""" return os.path.join(self.scripts_dir, 'version') @property def application(self): """The :class:`Application` of this deployment.""" return self.app_version.application @property def old_log(self): """ The :class:`wizard.old_log.Log` of this deployment. This is only applicable to un-migrated autoinstalls. """ if not self._old_log: self._old_log = old_log.DeployLog.load(self) return self._old_log @property def version(self): """ The :class:`distutils.version.LooseVersion` of this deployment. """ return self.app_version.version @property def app_version(self): """The :class:`ApplicationVersion` of this deployment.""" if not self._app_version: if os.path.isdir(os.path.join(self.location, ".git")): try: with util.ChangeDirectory(self.location): appname, _, version = git.describe().partition('-') self._app_version = ApplicationVersion.make(appname, version) except shell.CallError: pass if not self._app_version: self._app_version = self.old_log[-1].version return self._app_version @staticmethod def parse(line): """ Parses a line from the :term:`versions store`. .. note:: Use this method only when speed is of the utmost importance. You should prefer to directly create a deployment with only a ``location`` when possible. """ line = line.rstrip() try: location, deploydir = line.split(":") except ValueError: return Deployment(line) # lazy loaded version try: return Deployment(location, version=ApplicationVersion.parse(deploydir)) except Error as e: e.location = location raise e class Application(object): """Represents an application, i.e. mediawiki or phpbb.""" #: String name of the application name = None #: Dictionary of version strings to :class:`ApplicationVersion`. #: See also :meth:`makeVersion`. versions = None #: List of files that need to be modified when parametrizing. #: This is a class-wide constant, and should not normally be modified. parametrized_files = [] def __init__(self, name): self.name = name self.versions = {} # cache variables self._extractors = {} self._substitutions = {} def repository(self, srv_path): """ Returns the Git repository that would contain this application. ``srv_path`` corresponds to ``options.srv_path`` from the global baton. """ repo = os.path.join(srv_path, self.name + ".git") if not os.path.isdir(repo): repo = os.path.join(srv_path, self.name, ".git") if not os.path.isdir(repo): raise NoRepositoryError(self.name) return repo def makeVersion(self, version): """ Creates or retrieves the :class:`ApplicationVersion` singleton for the specified version. """ if version not in self.versions: self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self) return self.versions[version] def extract(self, deployment): """Extracts wizard variables from a deployment.""" result = {} for k,extractor in self.extractors.items(): result[k] = extractor(deployment) return result def parametrize(self, deployment, dir): """ Takes a generic source checkout at dir and parametrizes it according to the values of deployment. """ variables = deployment.extract() for file in self.parametrized_files: fullpath = os.path.join(dir, file) try: contents = open(fullpath, "r").read() except IOError: continue for key, value in variables.items(): if value is None: continue contents = contents.replace(key, value) tmp = tempfile.NamedTemporaryFile(delete=False) tmp.write(contents) os.rename(tmp.name, fullpath) def prepareConfig(self, deployment): """ Takes a deployment and replaces any explicit instances of a configuration variable with generic WIZARD_* constants. There is a sane default implementation built on substitutions; you can override this method to provide arbitrary extra behavior. """ for key, subst in self.substitutions.items(): subs = subst(deployment) if not subs and key not in self.deprecated_keys: logging.warning("No substitutions for %s" % key) def install(self, version, options): """ Run for 'wizard configure' (and, by proxy, 'wizard install') to configure an application. """ raise NotImplemented def upgrade(self, version, options): """ Run for 'wizard upgrade' to upgrade database schemas and other non-versioned data in an application. """ raise NotImplemented def detectVersion(self, deployment): """ Checks source files to determine the version manually. """ return None @property def extractors(self): """ Dictionary of variable names to extractor functions. These functions take a :class:`Deployment` as an argument and return the value of the variable, or ``None`` if it could not be found. See also :func:`wizard.app.filename_regex_extractor`. """ return {} @property def substitutions(self): """ Dictionary of variable names to substitution functions. These functions take a :class:`Deployment` as an argument and modify the deployment such that an explicit instance of the variable is released with the generic WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`. """ return {} @staticmethod def make(name): """Makes an application, but uses the correct subtype if available.""" try: __import__("wizard.app." + name) return getattr(wizard.app, name).Application(name) except ImportError: return Application(name) class ApplicationVersion(object): """Represents an abstract notion of a version for an application, where ``version`` is a :class:`distutils.version.LooseVersion` and ``application`` is a :class:`Application`.""" #: The :class:`distutils.version.LooseVersion` of this instance. version = None #: The :class:`Application` of this instance. application = None def __init__(self, version, application): self.version = version self.application = application @property def tag(self): """ Returns the name of the git describe tag for the commit the user is presently on, something like mediawiki-1.2.3-scripts-4-g123abcd """ return "%s-%s" % (self.application, self.version) @property def scripts_tag(self): """ Returns the name of the Git tag for this version. """ end = str(self.version).partition('-scripts')[2].partition('-')[0] return "%s-scripts%s" % (self.pristine_tag, end) @property def pristine_tag(self): """ Returns the name of the Git tag for the pristine version corresponding to this version. """ return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0]) def __cmp__(x, y): return cmp(x.version, y.version) @staticmethod def parse(value): """ Parses a line from the :term:`versions store` and return :class:`ApplicationVersion`. Use this only for cases when speed is of primary importance; the data in version is unreliable and when possible, you should prefer directly instantiating a Deployment and having it query the autoinstall itself for information. The `value` to parse will vary. For old style installs, it will look like:: /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z For new style installs, it will look like:: APP-x.y.z-scripts """ name = value.split("/")[-1] try: if name.find("-") != -1: app, _, version = name.partition("-") else: # kind of poor, maybe should error. Generally this # will actually result in a not found error app = name version = "trunk" except ValueError: raise DeploymentParseError(deploydir) return ApplicationVersion.make(app, version) @staticmethod def make(app, version): """ Makes/retrieves a singleton :class:`ApplicationVersion` from a``app`` and ``version`` string. """ try: # defer to the application for version creation to enforce # singletons return applications()[app].makeVersion(version) except KeyError: raise NoSuchApplication(app) ## -- Exceptions -- class Error(wizard.Error): """Base error class for this module""" pass class NoSuchApplication(Error): """ You attempted to reference a :class:`Application` named ``app``, which is not recognized by Wizard. """ #: The name of the application that does not exist. app = None #: The location of the autoinstall that threw this variable. #: This should be set by error handling code when it is availble. location = None def __init__(self, app): self.app = app class DeploymentParseError(Error): """ Could not parse ``value`` from :term:`versions store`. """ #: The value that failed to parse. value = None #: The location of the autoinstall that threw this variable. #: This should be set by error handling code when it is available. location = None def __init__(self, value): self.value = value class NoRepositoryError(Error): """ :class:`Application` does not appear to have a Git repository in the normal location. """ #: The name of the application that does not have a Git repository. app = None def __init__(self, app): self.app = app def __str__(self): return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app class NotMigratedError(Error): """ The deployment contains a .scripts-version file, but no .git or .scripts directory. """ #: Directory of deployment dir = None def __init__(self, dir): self.dir = dir def __str__(self): return """This installation was not migrated""" class AlreadyVersionedError(Error): """The deployment contained a .git directory but no .scripts directory.""" #: Directory of deployment dir = None def __init__(self, dir): self.dir = dir def __str__(self): return """ ERROR: Directory contains a .git directory, but not a .scripts directory. If this is not a corrupt migration, this means that the user was versioning their install using Git.""" class NotConfiguredError(Error): """The install was missing essential configuration.""" #: Directory of unconfigured install dir = None def __init__(self, dir): self.dir = dir def __str__(self): return """ ERROR: The install was well-formed, but not configured (essential configuration files were not found.)""" class CorruptedAutoinstallError(Error): """The install was missing a .git directory, but had a .scripts directory.""" #: Directory of the corrupted install dir = None def __init__(self, dir): self.dir = dir def __str__(self): return """ ERROR: Directory contains a .scripts directory, but not a .git directory.""" class NotAutoinstallError(Error): """The directory was not an autoinstall, due to missing .scripts-version file.""" #: Directory in question dir = None def __init__(self, dir): self.dir = dir def __str__(self): return """ ERROR: Could not find .scripts-version file. Are you sure this is an autoinstalled application? """ class NoTagError(Error): """Deployment has a tag that does not have an equivalent in upstream repository.""" #: Missing tag tag = None def __init__(self, tag): self.tag = tag def __str__(self): return """ ERROR: Could not find tag %s in repository.""" % self.tag class NoLocalTagError(Error): """Could not find tag in local repository.""" #: Missing tag tag = None def __init__(self, tag): self.tag = tag def __str__(self): return """ ERROR: Could not find tag %s in local repository.""" % self.tag class InconsistentPristineTagError(Error): """Pristine tag commit ID does not match upstream pristine tag commit ID.""" #: Inconsistent tag tag = None def __init__(self, tag): self.tag = tag def __str__(self): return """ ERROR: Local pristine tag %s did not match repository's. This probably means an upstream rebase occured.""" % self.tag class InconsistentScriptsTagError(Error): """Scripts tag commit ID does not match upstream scripts tag commit ID.""" #: Inconsistent tag tag = None def __init__(self, tag): self.tag = tag def __str__(self): return """ ERROR: Local scripts tag %s did not match repository's. This probably means an upstream rebase occurred.""" % self.tag class HeadNotDescendantError(Error): """HEAD is not connected to tag.""" #: Tag that HEAD should have been descendant of. tag = None def __init__(self, tag): self.tag = tag def __str__(self): return """ ERROR: HEAD is not a descendant of %s. This probably means that an upstream rebase occurred, and new tags were pulled, but local user commits were never rebased.""" % self.tag class VersionDetectionError(Error): """Could not detect real version of application.""" def __str__(self): return """ ERROR: Could not detect the real version of the application.""" class VersionMismatchError(Error): """Git version of application does not match detected version.""" #: Detected version real_version = None #: Version from Git git_version = None def __init__(self, real_version, git_version): self.real_version = real_version self.git_version = git_version def __str__(self): return """ ERROR: The detected version %s did not match the Git version %s.""" % (self.real_version, self.git_version) _application_list = [ "mediawiki", "wordpress", "joomla", "e107", "gallery2", "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django", # these are technically deprecated "advancedpoll", "gallery", ] _applications = None def applications(): """Hash table for looking up string application name to instance""" global _applications if not _applications: _applications = dict([(n,Application.make(n)) for n in _application_list ]) return _applications