+"""
+Object model for querying information and manipulating deployments
+of autoinstalls. Every :class:`Deployment` has an :class:`ApplicationVersion`
+which in turn has an :class:`Application`.
+"""
+
import os.path
-import math
import fileinput
import dateutil.parser
import distutils.version
+import tempfile
+import logging
import wizard
+from wizard import git, log, util
-class Error(wizard.Error):
- """Base error class for deploy errors"""
- pass
-
-class NoSuchApplication(Error):
- pass
-
-class DeploymentParseError(Error):
- def __init__(self, malformed):
- self.malformed = malformed
- def __str__(self):
- return """
+## -- Global Functions --
-ERROR: Could not parse deployment string:
-%s
-""" % self.malformed
+def get_install_lines(versions_store):
+ """
+ Low level function that retrieves a list of lines from the
+ :term:`versions store` that can be passed to :meth:`Deployment.parse`.
+ """
+ if os.path.isfile(versions_store):
+ return fileinput.input([versions_store])
+ return fileinput.input([versions_store + "/" + f for f in os.listdir(versions_store)])
-class ScriptsVersionTooManyFieldsError(Error):
- def __str__(self):
- return """
+def parse_install_lines(show, versions_store, yield_errors = False):
+ """
+ Generator function for iterating through all autoinstalls.
+ Each item is an instance of :class:`Deployment`, or possibly
+ a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can
+ filter out applications and versions by specifying ``app``
+ or ``app-1.2.3`` in ``show``. This function may generate
+ log output.
+ """
+ if not show: show = applications()
+ show = frozenset(show)
+ for line in get_install_lines(versions_store):
+ # construction
+ try:
+ d = Deployment.parse(line)
+ name = d.application.name
+ except NoSuchApplication as e:
+ if yield_errors:
+ yield e
+ continue
+ except Error:
+ # we consider this a worse error
+ logging.warning("Error with '%s'" % line.rstrip())
+ continue
+ # filter
+ if name + "-" + str(d.version) in show or name in show:
+ pass
+ else:
+ continue
+ # yield
+ yield d
-ERROR: Could not parse .scripts-version file. It
-contained too many fields.
-"""
+## -- Model Objects --
-class ScriptsVersionNotEnoughFieldsError(Error):
- def __str__(self):
- return """
+class Deployment(object):
+ """
+ Represents a deployment of an autoinstall, e.g. directory in a user's
+ web_scripts that has ``.scripts`` directory or ``.scripts-version``
+ file in it. Supply ``version`` with an :class:`ApplicationVersion` only if
+ you were reading from the :term:`versions store` and care about
+ speed (data from there can be stale).
+ """
+ #: Absolute path to the deployment
+ location = None
+ def __init__(self, location, version=None):
+ self.location = os.path.abspath(location)
+ self._app_version = version
+ # some cache variables
+ self._read_cache = {}
+ self._log = None
+ def read(self, file, force = False):
+ """
+ Reads a file's contents, possibly from cache unless ``force``
+ is ``True``.
+ """
+ if force or file not in self._read_cache:
+ f = open(os.path.join(self.location, file))
+ self._read_cache[file] = f.read()
+ f.close()
+ return self._read_cache[file]
+ def extract(self):
+ """
+ Extracts all the values of all variables from deployment.
+ These variables may be used for parametrizing generic parent
+ commits and include things such as database access credentials
+ and local configuration.
+ """
+ return self.application.extract(self)
+ def parametrize(self, dir):
+ """
+ Edits files in ``dir`` to replace WIZARD_* variables with literal
+ instances. This is used for constructing virtual merge bases, and
+ as such dir will generally not equal :attr:`location`.
+ """
+ return self.application.parametrize(self, dir)
+ def prepareConfig(self):
+ """
+ Edits files in the deployment such that any user-specific configuration
+ is replaced with generic WIZARD_* variables.
+ """
+ return self.application.prepareConfig(self)
+ def updateVersion(self, version):
+ """
+ Update the version of this deployment.
-ERROR: Could not parse .scripts-version file. It
-didn't contain enough fields.
-"""
+ This method will update the version of this deployment in memory
+ and on disk. It doesn't actually do an upgrade. The version
+ string you pass here should have ``-scripts`` as a suffix.
+ """
+ self._app_version = self.application.makeVersion(version)
+ f = open(os.path.join(self.scripts_dir, 'version'), 'w')
+ f.write(self.application.name + '-' + version + "\n")
+ f.close()
+ def scriptsifyVersion(self):
+ """
+ Converts from ``v1.0`` to ``v1.0-scripts``; use at end of migration.
-class ScriptsVersionNoSuchFile(Error):
- def __init__(self, file):
- self.file = file
- def __str__(self):
- return """
+ .. note::
-ERROR: File %s didn't exist.
-""" % self.file
+ This makes the assumption that a migration will be to
+ a ``-scripts`` tag and not a ``-scripts2`` tag. If you botch
+ migration, blow away the tag and try again.
+ """
+ self.updateVersion(self.app_version.scripts_tag)
+ @property
+ def migrated(self):
+ """Whether or not the autoinstalls has been migrated."""
+ return os.path.isdir(self.scripts_dir)
+ @property
+ def scripts_dir(self):
+ """The absolute path of the ``.scripts`` directory."""
+ return os.path.join(self.location, '.scripts')
+ @property
+ def old_version_file(self):
+ """
+ The absolute path of either ``.scripts-version`` (for unmigrated
+ installs) or ``.scripts/version``.
-def getInstallLines(global_options):
- """Retrieves a list of lines from the version directory that
- can be passed to Deployment.parse()"""
- vs = global_options.versions
- if os.path.isfile(vs):
- return fileinput.input([vs])
- return fileinput.input([vs + "/" + f for f in os.listdir(vs)])
+ .. note::
-class Deployment(object):
- """Represents a deployment of an autoinstall; i.e. a concrete
- directory in web_scripts that has .scripts-version in it."""
- def __init__(self, location, log=None, version=None):
- """ `location` Location of the deployment
- `version` ApplicationVersion of the app (this is cached info)
- `log` DeployLog of the app"""
- self.location = location
- self._version = version
- self._log = log
+ Use of this is discouraged for migrated installs.
+ """
+ if self.migrated:
+ return os.path.join(self.scripts_dir, 'old-version')
+ else:
+ return os.path.join(self.location, '.scripts-version')
+ @property
+ def version_file(self):
+ """The absolute path of the ``.scripts/version`` file."""
+ return os.path.join(self.scripts_dir, 'version')
+ @property
+ def application(self):
+ """The :class:`Application` of this deployment."""
+ return self.app_version.application
+ @property
+ def log(self):
+ """The :class:`wizard.deploy.Log` of this deployment."""
+ if not self._log:
+ self._log = log.DeployLog.load(self)
+ return self._log
+ @property
+ def version(self):
+ """
+ The :class:`distutils.version.LooseVersion` of this
+ deployment.
+ """
+ return self.app_version.version
+ @property
+ def app_version(self):
+ """The :class:`ApplicationVersion` of this deployment."""
+ if not self._app_version:
+ if os.path.isdir(os.path.join(self.location, ".git")):
+ with util.ChangeDirectory(self.location):
+ appname, _, version = git.describe().partition('-')
+ self._app_version = ApplicationVersion.make(appname, version)
+ else:
+ self._app_version = self.log[-1].version
+ return self._app_version
@staticmethod
def parse(line):
- """Parses a line from the results of parallel-find.pl.
- This will work out of the box with fileinput, see
- getInstallLines()"""
+ """
+ Parses a line from the :term:`versions store`.
+
+ .. note::
+
+ Use this method only when speed is of the utmost
+ importance. You should prefer to directly create a deployment
+ with only a ``location`` when possible.
+ """
line = line.rstrip()
try:
location, deploydir = line.split(":")
except ValueError:
return Deployment(line) # lazy loaded version
- return Deployment(location, version=ApplicationVersion.parse(deploydir))
- @staticmethod
- def fromDir(dir):
- """Lazily creates a deployment from a directory"""
- return Deployment(dir)
- def getVersionFile(self):
- return os.path.join(self.location, '.scripts-version')
- def getApplication(self):
- return self.getAppVersion().application
- def getLog(self):
- if not self._log:
- self._log = DeployLog.load(self.getVersionFile())
- return self._log
- def getVersion(self):
- """Returns the distutils Version of the deployment"""
- return self.getAppVersion().version
- def getAppVersion(self, force = False):
- """Returns the ApplicationVersion of the deployment"""
- if self._version and not force: return self._version
- else: return self.getLog()[-1].version
- def count(self):
- """Simple method which registers the deployment as a +1 on the
- appropriate version. No further inspection is done."""
- self.getAppVersion().count(self)
- return True
- def count_exists(self, file):
- """Checks if the codebase has a certain file/directory in it."""
- if os.path.exists(self.location + "/" + file):
- self.getAppVersion().count_exists(self, file)
- return True
- return False
+ try:
+ return Deployment(location, version=ApplicationVersion.parse(deploydir))
+ except Error as e:
+ e.location = location
+ raise e
class Application(object):
- """Represents the generic notion of an application, i.e.
- mediawiki or phpbb."""
+ """Represents an application, i.e. mediawiki or phpbb."""
+ #: String name of the application
+ name = None
+ #: Dictionary of version strings to :class:`ApplicationVersion`.
+ #: See also :meth:`makeVersion`.
+ versions = None
+ #: List of files that need to be modified when parametrizing.
+ #: This is a class-wide constant, and should not normally be modified.
+ parametrized_files = []
def __init__(self, name):
self.name = name
self.versions = {}
- # Some cache variables for fast access of calculated data
- self._total = 0
- self._max = 0
- self._c_exists = {}
- def getVersion(self, version):
+ # cache variables
+ self._extractors = {}
+ self._substitutions = {}
+ def repository(self, srv_path):
+ """
+ Returns the Git repository that would contain this application.
+ ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
+ Throws :exc:`NoRepositoryError` if the calculated path does not
+ exist.
+ """
+ repo = os.path.join(srv_path, self.name + ".git")
+ if not os.path.isdir(repo):
+ repo = os.path.join(srv_path, self.name, ".git")
+ if not os.path.isdir(repo):
+ raise NoRepositoryError(self.name)
+ return repo
+ def makeVersion(self, version):
+ """
+ Creates or retrieves the :class:`ApplicationVersion` singleton for the
+ specified version.
+ """
if version not in self.versions:
self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
return self.versions[version]
- # XXX: This code should go in summary.py; maybe as a mixin, maybe as
- # a visitor acceptor
- HISTOGRAM_WIDTH = 30
- def _graph(self, v):
- return '+' * int(math.ceil(float(v)/self._max * self.HISTOGRAM_WIDTH))
- def report(self):
- if not self.versions: return "%-11s no installs" % self.name
- ret = \
- ["%-16s %3d installs" % (self.name, self._total)] + \
- [v.report() for v in sorted(self.versions.values())]
- for f,c in self._c_exists.items():
- ret.append("%d users have %s" % (c,f))
- return "\n".join(ret)
-
-class DeployLog(list):
- # As per #python; if you decide to start overloading magic methods,
- # we should remove this subclass
- """Equivalent to .scripts-version: a series of DeployRevisions."""
- def __init__(self, revs = []):
- """`revs` List of DeployRevision objects"""
- list.__init__(self, revs) # pass to list
+ def extract(self, deployment):
+ """Extracts wizard variables from a deployment."""
+ result = {}
+ for k,extractor in self.extractors.items():
+ result[k] = extractor(deployment)
+ return result
+ def parametrize(self, deployment, dir):
+ """
+ Takes a generic source checkout at dir and parametrizes
+ it according to the values of deployment.
+ """
+ variables = deployment.extract()
+ for file in self.parametrized_files:
+ fullpath = os.path.join(dir, file)
+ f = open(fullpath, "r")
+ contents = f.read()
+ f.close()
+ for key, value in variables.items():
+ if value is None: continue
+ contents = contents.replace(key, value)
+ tmp = tempfile.NamedTemporaryFile(delete=False)
+ tmp.write(contents)
+ tmp.close()
+ os.rename(tmp.name, fullpath)
+ def prepareConfig(self, deployment):
+ """
+ Takes a deployment and replaces any explicit instances
+ of a configuration variable with generic WIZARD_* constants.
+ There is a sane default implementation built on substitutions;
+ you can override this method to provide arbitrary extra
+ behavior.
+ """
+ for key, subst in self.substitutions.items():
+ subs = subst(deployment)
+ if not subs and key not in self.deprecated_keys:
+ logging.warning("No substitutions for %s" % key)
+ @property
+ def extractors(self):
+ """
+ Dictionary of variable names to extractor functions. These functions
+ take a :class:`Deployment` as an argument and return the value of
+ the variable, or ``None`` if it could not be found.
+ See also :func:`wizard.app.filename_regex_extractor`.
+ """
+ return {}
+ @property
+ def substitutions(self):
+ """
+ Dictionary of variable names to substitution functions. These functions
+ take a :class:`Deployment` as an argument and modify the deployment such
+ that an explicit instance of the variable is released with the generic
+ WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`.
+ """
+ return {}
@staticmethod
- def load(file):
- """Loads a scripts version file and parses it into
- DeployLog and DeployRevision objects"""
- i = 0
- rev = DeployRevision()
- revs = []
- def append(rev):
- if i:
- if i != 4:
- raise ScriptsVersionNotEnoughFieldsError()
- revs.append(rev)
+ def make(name):
+ """Makes an application, but uses the correct subtype if available."""
try:
- fh = open(file)
- except IOError:
- raise ScriptsVersionNoSuchFile(file)
- for line in fh:
- line = line.rstrip()
- if not line:
- append(rev)
- i = 0
- rev = DeployRevision()
- continue
- if i == 0:
- rev.datetime = dateutil.parser.parse(line)
- elif i == 1:
- rev.user = line
- elif i == 2:
- rev.source = DeploySource.parse(line)
- elif i == 3:
- rev.version = ApplicationVersion.parse(line)
- else:
- # ruh oh
- raise ScriptsVersionTooManyFieldsError()
- i += 1
- append(rev)
- return DeployLog(revs)
- def __repr__(self):
- return '<DeployLog %s>' % list.__repr__(self)
-
-class DeployRevision(object):
- """A single entry in the .scripts-version file. Contains who deployed
- this revision, what application version this is, etc."""
- def __init__(self, datetime=None, user=None, source=None, version=None):
- """ `datetime` Time this revision was deployed
- `user` Person who deployed this revision, in user@host format.
- `source` Instance of DeploySource
- `version` Instance of ApplicationVersion
- Note: This object is typically built incrementally."""
- self.datetime = datetime
- self.user = user
- self.source = source
- self.version = version
-
-class DeploySource(object):
- """Source of the deployment; see subclasses for examples"""
- def __init__(self):
- raise NotImplementedError # abstract class
- @staticmethod
- def parse(line):
- # munge out common prefix
- rel = os.path.relpath(line, "/afs/athena.mit.edu/contrib/scripts/")
- parts = rel.split("/")
- if parts[0] == "wizard":
- return WizardUpdate()
- elif parts[0] == "deploy" or parts[0] == "deploydev":
- isDev = ( parts[0] == "deploydev" )
- try:
- if parts[1] == "updates":
- return OldUpdate(isDev)
- else:
- return TarballInstall(line, isDev)
- except IndexError:
- pass
- return UnknownDeploySource(line)
-
-class TarballInstall(DeploySource):
- """Original installation from tarball, characterized by
- /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z.tar.gz
- """
- def __init__(self, location, isDev):
- self.location = location
- self.isDev = isDev
-
-class OldUpdate(DeploySource):
- """Upgrade using old upgrade infrastructure, characterized by
- /afs/athena.mit.edu/contrib/scripts/deploydev/updates/update-scripts-version.pl
- """
- def __init__(self, isDev):
- self.isDev = isDev
-
-class WizardUpdate(DeploySource):
- """Upgrade using wizard infrastructure, characterized by
- /afs/athena.mit.edu/contrib/scripts/wizard/bin/wizard HASHGOBBLEDYGOOK
- """
- def __init__(self):
- pass
-
-class UnknownDeploySource(DeploySource):
- """Deployment that we don't know the meaning of. Wot!"""
- def __init__(self, line):
- self.line = line
+ __import__("wizard.app." + name)
+ return getattr(wizard.app, name).Application(name)
+ except ImportError:
+ return Application(name)
class ApplicationVersion(object):
- """Represents an abstract notion of a version for an application"""
+ """Represents an abstract notion of a version for an application, where
+ ``version`` is a :class:`distutils.version.LooseVersion` and
+ ``application`` is a :class:`Application`."""
+ #: The :class:`distutils.version.LooseVersion` of this instance.
+ version = None
+ #: The :class:`Application` of this instance.
+ application = None
def __init__(self, version, application):
- """ `version` Instance of distutils.LooseVersion
- `application` Instance of Application
- WARNING: Please don't call this directly; instead, use getVersion()
- on the application you want, so that this version gets registered."""
self.version = version
self.application = application
- self.c = 0
- self.c_exists = {}
+ @property
+ def scripts_tag(self):
+ """
+ Returns the name of the Git tag for this version.
+
+ .. note::
+
+ Use this function only during migration, as it does
+ not account for the existence of ``-scripts2``.
+ """
+ return "%s-scripts" % self.pristine_tag
+ @property
+ def pristine_tag(self):
+ """
+ Returns the name of the Git tag for the pristine version corresponding
+ to this version.
+ """
+ return "%s-%s" % (self.application.name, self.version)
def __cmp__(x, y):
return cmp(x.version, y.version)
@staticmethod
- def parse(deploydir,applookup=None):
- # The version of the deployment, will be:
- # /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z for old style installs
- # /afs/athena.mit.edu/contrib/scripts/wizard/srv/APP.git vx.y.z-scripts for new style installs
- name = deploydir.split("/")[-1]
+ def parse(value):
+ """
+ Parses a line from the :term:`versions store` and return
+ :class:`ApplicationVersion`.
+
+ Use this only for cases when speed is of primary importance;
+ the data in version is unreliable and when possible, you should
+ prefer directly instantiating a Deployment and having it query
+ the autoinstall itself for information.
+
+ The `value` to parse will vary. For old style installs, it
+ will look like::
+
+ /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
+
+ For new style installs, it will look like::
+
+ APP-x.y.z-scripts
+ """
+ name = value.split("/")[-1]
try:
- if name.find(" ") != -1:
- raw_app, raw_version = name.split(" ")
- version = raw_version[1:] # remove leading v
- app, _ = raw_app.split(".") # remove trailing .git
- elif name.find("-") != -1:
- app, version = name.split("-")
- elif name == "deploy":
- # Assume that it's django, since those were botched
- app = "django"
- version = "0.1-scripts"
+ if name.find("-") != -1:
+ app, _, version = name.partition("-")
else:
- raise DeploymentParseError(deploydir)
- except ValueError: # mostly from the a, b = foo.split(' ')
+ # kind of poor, maybe should error. Generally this
+ # will actually result in a not found error
+ app = name
+ version = "trunk"
+ except ValueError:
raise DeploymentParseError(deploydir)
- if not applookup: applookup = applications
+ return ApplicationVersion.make(app, version)
+ @staticmethod
+ def make(app, version):
+ """
+ Makes/retrieves a singleton :class:`ApplicationVersion` from
+ a``app`` and ``version`` string.
+ """
try:
- # defer to the application for version creation
- return applookup[app].getVersion(version)
+ # defer to the application for version creation to enforce
+ # singletons
+ return applications()[app].makeVersion(version)
except KeyError:
- raise NoSuchApplication
- # This is summary specific code
- def count(self, deployment):
- self.c += 1
- self.application._total += 1
- if self.c > self.application._max:
- self.application._max = self.c
- def count_exists(self, deployment, n):
- if n in self.c_exists: self.c_exists[n] += 1
- else: self.c_exists[n] = 1
- if n in self.application._c_exists: self.application._c_exists[n] += 1
- else: self.application._c_exists[n] = 1
- def report(self):
- return " %-12s %3d %s" \
- % (self.version, self.c, self.application._graph(self.c))
+ raise NoSuchApplication(app)
+
+## -- Exceptions --
+
+class Error(wizard.Error):
+ """Base error class for this module"""
+ pass
+
+class NoSuchApplication(Error):
+ """
+ You attempted to reference a :class:`Application` named
+ ``app``, which is not recognized by Wizard.
+ """
+ #: The name of the application that does not exist.
+ app = None
+ #: The location of the autoinstall that threw this variable.
+ #: This should be set by error handling code when it is availble.
+ location = None
+ def __init__(self, app):
+ self.app = app
+
+class DeploymentParseError(Error):
+ """
+ Could not parse ``value`` from :term:`versions store`.
+ """
+ #: The value that failed to parse.
+ value = None
+ #: The location of the autoinstall that threw this variable.
+ #: This should be set by error handling code when it is available.
+ location = None
+ def __init__(self, value):
+ self.value = value
+
+class NoRepositoryError(Error):
+ """
+ :class:`Application` does not appear to have a Git repository
+ in the normal location.
+ """
+ #: The name of the application that does not have a Git repository.
+ app = None
+ def __init__(self, app):
+ self.app = app
+ def __str__(self):
+ return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
# If you want, you can wrap this up into a registry and access things
# through that, but it's not really necessary
-application_list = [
+_application_list = [
"mediawiki", "wordpress", "joomla", "e107", "gallery2",
"phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
# these are technically deprecated
"advancedpoll", "gallery",
]
+_applications = None
-"""Hash table for looking up string application name to instance"""
-applications = dict([(n,Application(n)) for n in application_list ])
+def applications():
+ """Hash table for looking up string application name to instance"""
+ global _applications
+ if not _applications:
+ _applications = dict([(n,Application.make(n)) for n in _application_list ])
+ return _applications
scripts.mit.edu / wizard.git / blobdiff