""" Object model for querying information and manipulating deployments of autoinstalls. Every :class:`Deployment` has an :class:`ApplicationVersion` which in turn has an :class:`Application`. """ import os.path import fileinput import dateutil.parser import distutils.version import tempfile import logging import wizard from wizard import git, old_log, util ## -- Global Functions -- def get_install_lines(versions_store): """ Low level function that retrieves a list of lines from the :term:`versions store` that can be passed to :meth:`Deployment.parse`. """ if os.path.isfile(versions_store): return fileinput.input([versions_store]) return fileinput.input([versions_store + "/" + f for f in os.listdir(versions_store)]) def parse_install_lines(show, versions_store, yield_errors = False): """ Generator function for iterating through all autoinstalls. Each item is an instance of :class:`Deployment`, or possibly a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can filter out applications and versions by specifying ``app`` or ``app-1.2.3`` in ``show``. This function may generate log output. """ if not show: show = applications() show = frozenset(show) for line in get_install_lines(versions_store): # construction try: d = Deployment.parse(line) name = d.application.name except NoSuchApplication as e: if yield_errors: yield e continue except Error: # we consider this a worse error logging.warning("Error with '%s'" % line.rstrip()) continue # filter if name + "-" + str(d.version) in show or name in show: pass else: continue # yield yield d ## -- Model Objects -- class Deployment(object): """ Represents a deployment of an autoinstall, e.g. directory in a user's web_scripts that has ``.scripts`` directory or ``.scripts-version`` file in it. Supply ``version`` with an :class:`ApplicationVersion` only if you were reading from the :term:`versions store` and care about speed (data from there can be stale). """ #: Absolute path to the deployment location = None def __init__(self, location, version=None): self.location = os.path.abspath(location) self._app_version = version # some cache variables self._read_cache = {} self._old_log = None def read(self, file, force = False): """ Reads a file's contents, possibly from cache unless ``force`` is ``True``. """ if force or file not in self._read_cache: f = open(os.path.join(self.location, file)) self._read_cache[file] = f.read() f.close() return self._read_cache[file] def extract(self): """ Extracts all the values of all variables from deployment. These variables may be used for parametrizing generic parent commits and include things such as database access credentials and local configuration. """ return self.application.extract(self) def parametrize(self, dir): """ Edits files in ``dir`` to replace WIZARD_* variables with literal instances. This is used for constructing virtual merge bases, and as such dir will generally not equal :attr:`location`. """ return self.application.parametrize(self, dir) def prepareConfig(self): """ Edits files in the deployment such that any user-specific configuration is replaced with generic WIZARD_* variables. """ return self.application.prepareConfig(self) def updateVersion(self, version): """ Update the version of this deployment. This method will update the version of this deployment in memory and on disk. It doesn't actually do an upgrade. The version string you pass here should have ``-scripts`` as a suffix. """ self._app_version = self.application.makeVersion(version) f = open(os.path.join(self.scripts_dir, 'version'), 'w') f.write(self.application.name + '-' + version + "\n") f.close() def scriptsifyVersion(self): """ Converts from ``v1.0`` to ``v1.0-scripts``; use at end of migration. .. note:: This makes the assumption that a migration will be to a ``-scripts`` tag and not a ``-scripts2`` tag. If you botch migration, blow away the tag and try again. """ self.updateVersion(self.app_version.scripts_tag) @property def migrated(self): """Whether or not the autoinstalls has been migrated.""" return os.path.isdir(self.scripts_dir) @property def scripts_dir(self): """The absolute path of the ``.scripts`` directory.""" return os.path.join(self.location, '.scripts') @property def old_version_file(self): """ The absolute path of either ``.scripts-version`` (for unmigrated installs) or ``.scripts/version``. .. note:: Use of this is discouraged for migrated installs. """ if self.migrated: return os.path.join(self.scripts_dir, 'old-version') else: return os.path.join(self.location, '.scripts-version') @property def version_file(self): """The absolute path of the ``.scripts/version`` file.""" return os.path.join(self.scripts_dir, 'version') @property def application(self): """The :class:`Application` of this deployment.""" return self.app_version.application @property def old_log(self): """ The :class:`wizard.old_log.Log` of this deployment. This is only applicable to un-migrated autoinstalls. """ if not self._old_log: self._old_log = old_log.DeployLog.load(self) return self._old_log @property def version(self): """ The :class:`distutils.version.LooseVersion` of this deployment. """ return self.app_version.version @property def app_version(self): """The :class:`ApplicationVersion` of this deployment.""" if not self._app_version: if os.path.isdir(os.path.join(self.location, ".git")): with util.ChangeDirectory(self.location): appname, _, version = git.describe().partition('-') self._app_version = ApplicationVersion.make(appname, version) else: self._app_version = self.old_log[-1].version return self._app_version @staticmethod def parse(line): """ Parses a line from the :term:`versions store`. .. note:: Use this method only when speed is of the utmost importance. You should prefer to directly create a deployment with only a ``location`` when possible. """ line = line.rstrip() try: location, deploydir = line.split(":") except ValueError: return Deployment(line) # lazy loaded version try: return Deployment(location, version=ApplicationVersion.parse(deploydir)) except Error as e: e.location = location raise e class Application(object): """Represents an application, i.e. mediawiki or phpbb.""" #: String name of the application name = None #: Dictionary of version strings to :class:`ApplicationVersion`. #: See also :meth:`makeVersion`. versions = None #: List of files that need to be modified when parametrizing. #: This is a class-wide constant, and should not normally be modified. parametrized_files = [] def __init__(self, name): self.name = name self.versions = {} # cache variables self._extractors = {} self._substitutions = {} def repository(self, srv_path): """ Returns the Git repository that would contain this application. ``srv_path`` corresponds to ``options.srv_path`` from the global baton. Throws :exc:`NoRepositoryError` if the calculated path does not exist. """ repo = os.path.join(srv_path, self.name + ".git") if not os.path.isdir(repo): repo = os.path.join(srv_path, self.name, ".git") if not os.path.isdir(repo): raise NoRepositoryError(self.name) return repo def makeVersion(self, version): """ Creates or retrieves the :class:`ApplicationVersion` singleton for the specified version. """ if version not in self.versions: self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self) return self.versions[version] def extract(self, deployment): """Extracts wizard variables from a deployment.""" result = {} for k,extractor in self.extractors.items(): result[k] = extractor(deployment) return result def parametrize(self, deployment, dir): """ Takes a generic source checkout at dir and parametrizes it according to the values of deployment. """ variables = deployment.extract() for file in self.parametrized_files: fullpath = os.path.join(dir, file) f = open(fullpath, "r") contents = f.read() f.close() for key, value in variables.items(): if value is None: continue contents = contents.replace(key, value) tmp = tempfile.NamedTemporaryFile(delete=False) tmp.write(contents) tmp.close() os.rename(tmp.name, fullpath) def prepareConfig(self, deployment): """ Takes a deployment and replaces any explicit instances of a configuration variable with generic WIZARD_* constants. There is a sane default implementation built on substitutions; you can override this method to provide arbitrary extra behavior. """ for key, subst in self.substitutions.items(): subs = subst(deployment) if not subs and key not in self.deprecated_keys: logging.warning("No substitutions for %s" % key) @property def extractors(self): """ Dictionary of variable names to extractor functions. These functions take a :class:`Deployment` as an argument and return the value of the variable, or ``None`` if it could not be found. See also :func:`wizard.app.filename_regex_extractor`. """ return {} @property def substitutions(self): """ Dictionary of variable names to substitution functions. These functions take a :class:`Deployment` as an argument and modify the deployment such that an explicit instance of the variable is released with the generic WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`. """ return {} @staticmethod def make(name): """Makes an application, but uses the correct subtype if available.""" try: __import__("wizard.app." + name) return getattr(wizard.app, name).Application(name) except ImportError: return Application(name) class ApplicationVersion(object): """Represents an abstract notion of a version for an application, where ``version`` is a :class:`distutils.version.LooseVersion` and ``application`` is a :class:`Application`.""" #: The :class:`distutils.version.LooseVersion` of this instance. version = None #: The :class:`Application` of this instance. application = None def __init__(self, version, application): self.version = version self.application = application @property def scripts_tag(self): """ Returns the name of the Git tag for this version. .. note:: Use this function only during migration, as it does not account for the existence of ``-scripts2``. """ return "%s-scripts" % self.pristine_tag @property def pristine_tag(self): """ Returns the name of the Git tag for the pristine version corresponding to this version. """ return "%s-%s" % (self.application.name, self.version) def __cmp__(x, y): return cmp(x.version, y.version) @staticmethod def parse(value): """ Parses a line from the :term:`versions store` and return :class:`ApplicationVersion`. Use this only for cases when speed is of primary importance; the data in version is unreliable and when possible, you should prefer directly instantiating a Deployment and having it query the autoinstall itself for information. The `value` to parse will vary. For old style installs, it will look like:: /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z For new style installs, it will look like:: APP-x.y.z-scripts """ name = value.split("/")[-1] try: if name.find("-") != -1: app, _, version = name.partition("-") else: # kind of poor, maybe should error. Generally this # will actually result in a not found error app = name version = "trunk" except ValueError: raise DeploymentParseError(deploydir) return ApplicationVersion.make(app, version) @staticmethod def make(app, version): """ Makes/retrieves a singleton :class:`ApplicationVersion` from a``app`` and ``version`` string. """ try: # defer to the application for version creation to enforce # singletons return applications()[app].makeVersion(version) except KeyError: raise NoSuchApplication(app) ## -- Exceptions -- class Error(wizard.Error): """Base error class for this module""" pass class NoSuchApplication(Error): """ You attempted to reference a :class:`Application` named ``app``, which is not recognized by Wizard. """ #: The name of the application that does not exist. app = None #: The location of the autoinstall that threw this variable. #: This should be set by error handling code when it is availble. location = None def __init__(self, app): self.app = app class DeploymentParseError(Error): """ Could not parse ``value`` from :term:`versions store`. """ #: The value that failed to parse. value = None #: The location of the autoinstall that threw this variable. #: This should be set by error handling code when it is available. location = None def __init__(self, value): self.value = value class NoRepositoryError(Error): """ :class:`Application` does not appear to have a Git repository in the normal location. """ #: The name of the application that does not have a Git repository. app = None def __init__(self, app): self.app = app def __str__(self): return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app # If you want, you can wrap this up into a registry and access things # through that, but it's not really necessary _application_list = [ "mediawiki", "wordpress", "joomla", "e107", "gallery2", "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django", # these are technically deprecated "advancedpoll", "gallery", ] _applications = None def applications(): """Hash table for looking up string application name to instance""" global _applications if not _applications: _applications = dict([(n,Application.make(n)) for n in _application_list ]) return _applications