+"""
+Object model for querying information and manipulating deployments
+of autoinstalls. Every :class:`Deployment` has an :class:`ApplicationVersion`
+which in turn has an :class:`Application`.
+"""
+
import os.path
import fileinput
import dateutil.parser
import distutils.version
+import tempfile
+import logging
import wizard
-from wizard import log
+from wizard import git, log, util
## -- Global Functions --
-def getInstallLines(vs):
- """Retrieves a list of lines from the version directory that
- can be passed to Deployment.parse()"""
- if os.path.isfile(vs):
- return fileinput.input([vs])
- return fileinput.input([vs + "/" + f for f in os.listdir(vs)])
+def get_install_lines(versions_store):
+ """
+ Low level function that retrieves a list of lines from the
+ :term:`versions store` that can be passed to :meth:`Deployment.parse`.
+ """
+ if os.path.isfile(versions_store):
+ return fileinput.input([versions_store])
+ return fileinput.input([versions_store + "/" + f for f in os.listdir(versions_store)])
+
+def parse_install_lines(show, versions_store, yield_errors = False):
+ """
+ Generator function for iterating through all autoinstalls.
+ Each item is an instance of :class:`Deployment`, or possibly
+ a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can
+ filter out applications and versions by specifying ``app``
+ or ``app-1.2.3`` in ``show``. This function may generate
+ log output.
+ """
+ if not show: show = applications()
+ show = frozenset(show)
+ for line in get_install_lines(versions_store):
+ # construction
+ try:
+ d = Deployment.parse(line)
+ name = d.application.name
+ except NoSuchApplication as e:
+ if yield_errors:
+ yield e
+ continue
+ except Error:
+ # we consider this a worse error
+ logging.warning("Error with '%s'" % line.rstrip())
+ continue
+ # filter
+ if name + "-" + str(d.version) in show or name in show:
+ pass
+ else:
+ continue
+ # yield
+ yield d
## -- Model Objects --
class Deployment(object):
- """Represents a deployment of an autoinstall; i.e. a concrete
- directory in web_scripts that has .scripts-version in it."""
- def __init__(self, location, log=None, version=None):
- """ `location` Location of the deployment
- `version` ApplicationVersion of the app (this is cached info)
- `log` DeployLog of the app"""
- self.location = location
+ """
+ Represents a deployment of an autoinstall, e.g. directory in a user's
+ web_scripts that has ``.scripts`` directory or ``.scripts-version``
+ file in it. Supply ``version`` with an :class:`ApplicationVersion` only if
+ you were reading from the :term:`versions store` and care about
+ speed (data from there can be stale).
+ """
+ #: Absolute path to the deployment
+ location = None
+ def __init__(self, location, version=None):
+ self.location = os.path.abspath(location)
self._app_version = version
- self._log = log
+ # some cache variables
self._read_cache = {}
+ self._log = None
def read(self, file, force = False):
- """Reads a file's contents and stuffs it in a cache"""
+ """
+ Reads a file's contents, possibly from cache unless ``force``
+ is ``True``.
+ """
if force or file not in self._read_cache:
f = open(os.path.join(self.location, file))
self._read_cache[file] = f.read()
f.close()
return self._read_cache[file]
def extract(self):
+ """
+ Extracts all the values of all variables from deployment.
+ These variables may be used for parametrizing generic parent
+ commits and include things such as database access credentials
+ and local configuration.
+ """
return self.application.extract(self)
- def updateVersion(self, version=None):
- """`version` Version string to update to, or leave out to simply
- force the creation of .scripts/version file"""
- if not version:
- version = str(self.version)
- else:
- self._app_version = self.application.makeVersion(version)
+ def parametrize(self, dir):
+ """
+ Edits files in ``dir`` to replace WIZARD_* variables with literal
+ instances. This is used for constructing virtual merge bases, and
+ as such dir will generally not equal :attr:`location`.
+ """
+ return self.application.parametrize(self, dir)
+ def prepareConfig(self):
+ """
+ Edits files in the deployment such that any user-specific configuration
+ is replaced with generic WIZARD_* variables.
+ """
+ return self.application.prepareConfig(self)
+ def updateVersion(self, version):
+ """
+ Update the version of this deployment.
+
+ This method will update the version of this deployment in memory
+ and on disk. It doesn't actually do an upgrade. The version
+ string you pass here should have ``-scripts`` as a suffix.
+ """
+ self._app_version = self.application.makeVersion(version)
f = open(os.path.join(self.scripts_dir, 'version'), 'w')
f.write(self.application.name + '-' + version + "\n")
f.close()
+ def scriptsifyVersion(self):
+ """
+ Converts from ``v1.0`` to ``v1.0-scripts``; use at end of migration.
+
+ .. note::
+
+ This makes the assumption that a migration will be to
+ a ``-scripts`` tag and not a ``-scripts2`` tag. If you botch
+ migration, blow away the tag and try again.
+ """
+ self.updateVersion(self.app_version.scripts_tag)
+ @property
+ def migrated(self):
+ """Whether or not the autoinstalls has been migrated."""
+ return os.path.isdir(self.scripts_dir)
@property
def scripts_dir(self):
+ """The absolute path of the ``.scripts`` directory."""
return os.path.join(self.location, '.scripts')
@property
+ def old_version_file(self):
+ """
+ The absolute path of either ``.scripts-version`` (for unmigrated
+ installs) or ``.scripts/version``.
+
+ .. note::
+
+ Use of this is discouraged for migrated installs.
+ """
+ if self.migrated:
+ return os.path.join(self.scripts_dir, 'old-version')
+ else:
+ return os.path.join(self.location, '.scripts-version')
+ @property
def version_file(self):
- return os.path.join(self.location, '.scripts-version')
+ """The absolute path of the ``.scripts/version`` file."""
+ return os.path.join(self.scripts_dir, 'version')
@property
def application(self):
+ """The :class:`Application` of this deployment."""
return self.app_version.application
@property
def log(self):
+ """The :class:`wizard.deploy.Log` of this deployment."""
if not self._log:
- self._log = log.DeployLog.load(self.version_file)
+ self._log = log.DeployLog.load(self)
return self._log
@property
def version(self):
- """Returns the distutils Version of the deployment"""
+ """
+ The :class:`distutils.version.LooseVersion` of this
+ deployment.
+ """
return self.app_version.version
@property
- def app_version(self, force = False):
- """Returns the ApplicationVersion of the deployment"""
- if self._app_version and not force: return self._app_version
- else: return self.log[-1].version
+ def app_version(self):
+ """The :class:`ApplicationVersion` of this deployment."""
+ if not self._app_version:
+ if os.path.isdir(os.path.join(self.location, ".git")):
+ with util.ChangeDirectory(self.location):
+ appname, _, version = git.describe().partition('-')
+ self._app_version = ApplicationVersion.make(appname, version)
+ else:
+ self._app_version = self.log[-1].version
+ return self._app_version
@staticmethod
def parse(line):
- """Parses a line from the results of parallel-find.pl.
- This will work out of the box with fileinput, see
- getInstallLines()"""
+ """
+ Parses a line from the :term:`versions store`.
+
+ .. note::
+
+ Use this method only when speed is of the utmost
+ importance. You should prefer to directly create a deployment
+ with only a ``location`` when possible.
+ """
line = line.rstrip()
try:
location, deploydir = line.split(":")
except ValueError:
return Deployment(line) # lazy loaded version
- return Deployment(location, version=ApplicationVersion.parse(deploydir, location))
+ try:
+ return Deployment(location, version=ApplicationVersion.parse(deploydir))
+ except Error as e:
+ e.location = location
+ raise e
class Application(object):
- """Represents the generic notion of an application, i.e.
- mediawiki or phpbb."""
+ """Represents an application, i.e. mediawiki or phpbb."""
+ #: String name of the application
+ name = None
+ #: Dictionary of version strings to :class:`ApplicationVersion`.
+ #: See also :meth:`makeVersion`.
+ versions = None
+ #: List of files that need to be modified when parametrizing.
+ #: This is a class-wide constant, and should not normally be modified.
+ parametrized_files = []
def __init__(self, name):
self.name = name
self.versions = {}
+ # cache variables
self._extractors = {}
- @property
- def repository(self):
- """Returns the Git repository that would contain this application."""
- repo = os.path.join("/afs/athena.mit.edu/contrib/scripts/git/autoinstalls", self.name + ".git")
+ self._substitutions = {}
+ def repository(self, srv_path):
+ """
+ Returns the Git repository that would contain this application.
+ ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
+ Throws :exc:`NoRepositoryError` if the calculated path does not
+ exist.
+ """
+ repo = os.path.join(srv_path, self.name + ".git")
if not os.path.isdir(repo):
- raise NoRepositoryError(app)
+ repo = os.path.join(srv_path, self.name, ".git")
+ if not os.path.isdir(repo):
+ raise NoRepositoryError(self.name)
return repo
def makeVersion(self, version):
+ """
+ Creates or retrieves the :class:`ApplicationVersion` singleton for the
+ specified version.
+ """
if version not in self.versions:
self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
return self.versions[version]
for k,extractor in self.extractors.items():
result[k] = extractor(deployment)
return result
+ def parametrize(self, deployment, dir):
+ """
+ Takes a generic source checkout at dir and parametrizes
+ it according to the values of deployment.
+ """
+ variables = deployment.extract()
+ for file in self.parametrized_files:
+ fullpath = os.path.join(dir, file)
+ f = open(fullpath, "r")
+ contents = f.read()
+ f.close()
+ for key, value in variables.items():
+ if value is None: continue
+ contents = contents.replace(key, value)
+ tmp = tempfile.NamedTemporaryFile(delete=False)
+ tmp.write(contents)
+ tmp.close()
+ os.rename(tmp.name, fullpath)
+ def prepareConfig(self, deployment):
+ """
+ Takes a deployment and replaces any explicit instances
+ of a configuration variable with generic WIZARD_* constants.
+ There is a sane default implementation built on substitutions;
+ you can override this method to provide arbitrary extra
+ behavior.
+ """
+ for key, subst in self.substitutions.items():
+ subs = subst(deployment)
+ if not subs and key not in self.deprecated_keys:
+ logging.warning("No substitutions for %s" % key)
@property
def extractors(self):
+ """
+ Dictionary of variable names to extractor functions. These functions
+ take a :class:`Deployment` as an argument and return the value of
+ the variable, or ``None`` if it could not be found.
+ See also :func:`wizard.app.filename_regex_extractor`.
+ """
+ return {}
+ @property
+ def substitutions(self):
+ """
+ Dictionary of variable names to substitution functions. These functions
+ take a :class:`Deployment` as an argument and modify the deployment such
+ that an explicit instance of the variable is released with the generic
+ WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`.
+ """
return {}
@staticmethod
def make(name):
return Application(name)
class ApplicationVersion(object):
- """Represents an abstract notion of a version for an application"""
+ """Represents an abstract notion of a version for an application, where
+ ``version`` is a :class:`distutils.version.LooseVersion` and
+ ``application`` is a :class:`Application`."""
+ #: The :class:`distutils.version.LooseVersion` of this instance.
+ version = None
+ #: The :class:`Application` of this instance.
+ application = None
def __init__(self, version, application):
- """ `version` Instance of distutils.LooseVersion
- `application` Instance of Application
- WARNING: Please don't call this directly; instead, use getVersion()
- on the application you want, so that this version gets registered."""
self.version = version
self.application = application
@property
def scripts_tag(self):
- """Returns the name of the Git tag for this version"""
- # XXX: This assumes that there's only a -scripts version
- # which will not be true in the future. Unfortunately, finding
- # the "true" latest version is computationally expensive
- return "v%s-scripts" % self.version
+ """
+ Returns the name of the Git tag for this version.
+
+ .. note::
+
+ Use this function only during migration, as it does
+ not account for the existence of ``-scripts2``.
+ """
+ return "%s-scripts" % self.pristine_tag
+ @property
+ def pristine_tag(self):
+ """
+ Returns the name of the Git tag for the pristine version corresponding
+ to this version.
+ """
+ return "%s-%s" % (self.application.name, self.version)
def __cmp__(x, y):
return cmp(x.version, y.version)
@staticmethod
- def parse(deploydir,location,applookup=None):
- # The version of the deployment, will be:
- # /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z for old style installs
- name = deploydir.split("/")[-1]
+ def parse(value):
+ """
+ Parses a line from the :term:`versions store` and return
+ :class:`ApplicationVersion`.
+
+ Use this only for cases when speed is of primary importance;
+ the data in version is unreliable and when possible, you should
+ prefer directly instantiating a Deployment and having it query
+ the autoinstall itself for information.
+
+ The `value` to parse will vary. For old style installs, it
+ will look like::
+
+ /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
+
+ For new style installs, it will look like::
+
+ APP-x.y.z-scripts
+ """
+ name = value.split("/")[-1]
try:
- if name.find(" ") != -1:
- raw_app, raw_version = name.split(" ")
- version = raw_version[1:] # remove leading v
- app, _ = raw_app.split(".") # remove trailing .git
- elif name.find("-") != -1:
+ if name.find("-") != -1:
app, _, version = name.partition("-")
else:
+ # kind of poor, maybe should error. Generally this
+ # will actually result in a not found error
app = name
version = "trunk"
- except ValueError: # mostly from the a, b = foo.split(' ')
- raise DeploymentParseError(deploydir, location)
- if not applookup: applookup = applications()
+ except ValueError:
+ raise DeploymentParseError(deploydir)
+ return ApplicationVersion.make(app, version)
+ @staticmethod
+ def make(app, version):
+ """
+ Makes/retrieves a singleton :class:`ApplicationVersion` from
+ a``app`` and ``version`` string.
+ """
try:
- # defer to the application for version creation
- return applookup[app].makeVersion(version)
+ # defer to the application for version creation to enforce
+ # singletons
+ return applications()[app].makeVersion(version)
except KeyError:
- raise NoSuchApplication(app, location)
+ raise NoSuchApplication(app)
## -- Exceptions --
-class Error(Exception):
+class Error(wizard.Error):
"""Base error class for this module"""
pass
class NoSuchApplication(Error):
- def __init__(self, name, location):
- self.name = name
- self.location = location
- def __str__(self):
- return "ERROR: Unrecognized app '%s' at %s" % (self.name, self.location)
+ """
+ You attempted to reference a :class:`Application` named
+ ``app``, which is not recognized by Wizard.
+ """
+ #: The name of the application that does not exist.
+ app = None
+ #: The location of the autoinstall that threw this variable.
+ #: This should be set by error handling code when it is availble.
+ location = None
+ def __init__(self, app):
+ self.app = app
class DeploymentParseError(Error):
- def __init__(self, malformed, location):
- self.malformed = malformed
- self.location = location
- def __str__(self):
- return """ERROR: Unparseable '%s' at %s""" % (self.malformed, self.location)
+ """
+ Could not parse ``value`` from :term:`versions store`.
+ """
+ #: The value that failed to parse.
+ value = None
+ #: The location of the autoinstall that threw this variable.
+ #: This should be set by error handling code when it is available.
+ location = None
+ def __init__(self, value):
+ self.value = value
class NoRepositoryError(Error):
+ """
+ :class:`Application` does not appear to have a Git repository
+ in the normal location.
+ """
+ #: The name of the application that does not have a Git repository.
+ app = None
def __init__(self, app):
self.app = app
- self.location = "unknown"
def __str__(self):
- return """
-
-ERROR: Could not find repository for this application. Have
-you converted the repository over? Is the name %s
-the same as the name of the .git folder?
-""" % self.app
+ return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
# If you want, you can wrap this up into a registry and access things
# through that, but it's not really necessary
-application_list = [
+_application_list = [
"mediawiki", "wordpress", "joomla", "e107", "gallery2",
"phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
# these are technically deprecated
"""Hash table for looking up string application name to instance"""
global _applications
if not _applications:
- _applications = dict([(n,Application.make(n)) for n in application_list ])
+ _applications = dict([(n,Application.make(n)) for n in _application_list ])
return _applications
scripts.mit.edu / wizard.git / blobdiff