X-Git-Url: https://scripts.mit.edu/gitweb/wizard.git/blobdiff_plain/3f56e5237e400234761729ddae0d0b2a9ca74085..7d857aba9c7d324084127f7ba4430b7607dd7824:/wizard/deploy.py

diff --git a/wizard/deploy.py b/wizard/deploy.py
index 39d1503..8c694ef 100644
--- a/wizard/deploy.py
+++ b/wizard/deploy.py
@@ -1,169 +1,444 @@
+"""
+Object model for querying information and manipulating deployments
+of autoinstalls.  Every :class:`Deployment` has an :class:`ApplicationVersion`
+which in turn has an :class:`Application`.
+"""
+
 import os.path
 import fileinput
 import dateutil.parser
 import distutils.version
+import tempfile
+import logging
 
 import wizard
-from wizard import log
+from wizard import git, log, util
 
 ## -- Global Functions --
 
-def getInstallLines(vs):
-    """Retrieves a list of lines from the version directory that
-    can be passed to Deployment.parse()"""
-    if os.path.isfile(vs):
-        return fileinput.input([vs])
-    return fileinput.input([vs + "/" + f for f in os.listdir(vs)])
+def get_install_lines(versions_store):
+    """
+    Low level function that retrieves a list of lines from the
+    :term:`versions store` that can be passed to :meth:`Deployment.parse`.
+    """
+    if os.path.isfile(versions_store):
+        return fileinput.input([versions_store])
+    return fileinput.input([versions_store + "/" + f for f in os.listdir(versions_store)])
+
+def parse_install_lines(show, versions_store, yield_errors = False):
+    """
+    Generator function for iterating through all autoinstalls.
+    Each item is an instance of :class:`Deployment`, or possibly
+    a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``.  You can
+    filter out applications and versions by specifying ``app``
+    or ``app-1.2.3`` in ``show``.  This function may generate
+    log output.
+    """
+    if not show: show = applications()
+    show = frozenset(show)
+    for line in get_install_lines(versions_store):
+        # construction
+        try:
+            d = Deployment.parse(line)
+            name = d.application.name
+        except NoSuchApplication as e:
+            if yield_errors:
+                yield e
+            continue
+        except Error:
+            # we consider this a worse error
+            logging.warning("Error with '%s'" % line.rstrip())
+            continue
+        # filter
+        if name + "-" + str(d.version) in show or name in show:
+            pass
+        else:
+            continue
+        # yield
+        yield d
 
 ## -- Model Objects --
 
 class Deployment(object):
-    """Represents a deployment of an autoinstall; i.e. a concrete
-    directory in web_scripts that has .scripts-version in it."""
-    def __init__(self, location, log=None, version=None):
-        """ `location`  Location of the deployment
-            `version`   ApplicationVersion of the app (this is cached info)
-            `log`       DeployLog of the app"""
-        self.location = location
-        self._version = version
-        self._log = log
+    """
+    Represents a deployment of an autoinstall, e.g. directory in a user's
+    web_scripts that has ``.scripts`` directory or ``.scripts-version``
+    file in it.  Supply ``version`` with an :class:`ApplicationVersion` only if
+    you were reading from the :term:`versions store` and care about
+    speed (data from there can be stale).
+    """
+    #: Absolute path to the deployment
+    location = None
+    def __init__(self, location, version=None):
+        self.location = os.path.abspath(location)
+        self._app_version = version
+        # some cache variables
+        self._read_cache = {}
+        self._log = None
+    def read(self, file, force = False):
+        """
+        Reads a file's contents, possibly from cache unless ``force``
+        is ``True``.
+        """
+        if force or file not in self._read_cache:
+            f = open(os.path.join(self.location, file))
+            self._read_cache[file] = f.read()
+            f.close()
+        return self._read_cache[file]
+    def extract(self):
+        """
+        Extracts all the values of all variables from deployment.
+        These variables may be used for parametrizing generic parent
+        commits and include things such as database access credentials
+        and local configuration.
+        """
+        return self.application.extract(self)
+    def parametrize(self, dir):
+        """
+        Edits files in ``dir`` to replace WIZARD_* variables with literal
+        instances.  This is used for constructing virtual merge bases, and
+        as such dir will generally not equal :attr:`location`.
+        """
+        return self.application.parametrize(self, dir)
+    def prepareConfig(self):
+        """
+        Edits files in the deployment such that any user-specific configuration
+        is replaced with generic WIZARD_* variables.
+        """
+        return self.application.prepareConfig(self)
+    def updateVersion(self, version):
+        """
+        Update the version of this deployment.
+
+        This method will update the version of this deployment in memory
+        and on disk.  It doesn't actually do an upgrade.  The version
+        string you pass here should have ``-scripts`` as a suffix.
+        """
+        self._app_version = self.application.makeVersion(version)
+        f = open(os.path.join(self.scripts_dir, 'version'), 'w')
+        f.write(self.application.name + '-' + version + "\n")
+        f.close()
+    def scriptsifyVersion(self):
+        """
+        Converts from ``v1.0`` to ``v1.0-scripts``; use at end of migration.
+
+        .. note::
+
+            This makes the assumption that a migration will be to
+            a ``-scripts`` tag and not a ``-scripts2`` tag.  If you botch
+            migration, blow away the tag and try again.
+        """
+        self.updateVersion(self.app_version.scripts_tag)
+    @property
+    def migrated(self):
+        """Whether or not the autoinstalls has been migrated."""
+        return os.path.isdir(self.scripts_dir)
+    @property
+    def scripts_dir(self):
+        """The absolute path of the ``.scripts`` directory."""
+        return os.path.join(self.location, '.scripts')
+    @property
+    def old_version_file(self):
+        """
+        The absolute path of either ``.scripts-version`` (for unmigrated
+        installs) or ``.scripts/version``.
+
+        .. note::
+
+            Use of this is discouraged for migrated installs.
+        """
+        if self.migrated:
+            return os.path.join(self.scripts_dir, 'old-version')
+        else:
+            return os.path.join(self.location, '.scripts-version')
     @property
     def version_file(self):
-        return os.path.join(self.location, '.scripts-version')
+        """The absolute path of the ``.scripts/version`` file."""
+        return os.path.join(self.scripts_dir, 'version')
     @property
     def application(self):
+        """The :class:`Application` of this deployment."""
         return self.app_version.application
     @property
     def log(self):
+        """The :class:`wizard.deploy.Log` of this deployment."""
         if not self._log:
-            self._log = log.DeployLog.load(self.version_file)
+            self._log = log.DeployLog.load(self)
         return self._log
     @property
     def version(self):
-        """Returns the distutils Version of the deployment"""
+        """
+        The :class:`distutils.version.LooseVersion` of this
+        deployment.
+        """
         return self.app_version.version
     @property
-    def app_version(self, force = False):
-        """Returns the ApplicationVersion of the deployment"""
-        if self._version and not force: return self._version
-        else: return self.log[-1].version
+    def app_version(self):
+        """The :class:`ApplicationVersion` of this deployment."""
+        if not self._app_version:
+            if os.path.isdir(os.path.join(self.location, ".git")):
+                with util.ChangeDirectory(self.location):
+                    appname, _, version = git.describe().partition('-')
+                self._app_version = ApplicationVersion.make(appname, version)
+            else:
+                self._app_version = self.log[-1].version
+        return self._app_version
     @staticmethod
     def parse(line):
-        """Parses a line from the results of parallel-find.pl.
-        This will work out of the box with fileinput, see
-        getInstallLines()"""
+        """
+        Parses a line from the :term:`versions store`.
+
+        .. note::
+
+            Use this method only when speed is of the utmost
+            importance.  You should prefer to directly create a deployment
+            with only a ``location`` when possible.
+        """
         line = line.rstrip()
         try:
             location, deploydir = line.split(":")
         except ValueError:
             return Deployment(line) # lazy loaded version
-        return Deployment(location, version=ApplicationVersion.parse(deploydir, location))
+        try:
+            return Deployment(location, version=ApplicationVersion.parse(deploydir))
+        except Error as e:
+            e.location = location
+            raise e
 
 class Application(object):
-    """Represents the generic notion of an application, i.e.
-    mediawiki or phpbb."""
+    """Represents an application, i.e. mediawiki or phpbb."""
+    #: String name of the application
+    name = None
+    #: Dictionary of version strings to :class:`ApplicationVersion`.
+    #: See also :meth:`makeVersion`.
+    versions = None
+    #: List of files that need to be modified when parametrizing.
+    #: This is a class-wide constant, and should not normally be modified.
+    parametrized_files = []
     def __init__(self, name):
         self.name = name
         self.versions = {}
-    @property
-    def repository(self):
-        """Returns the Git repository that would contain this application."""
-        repo = os.path.join("/afs/athena.mit.edu/contrib/scripts/git/autoinstalls", self.name + ".git")
+        # cache variables
+        self._extractors = {}
+        self._substitutions = {}
+    def repository(self, srv_path):
+        """
+        Returns the Git repository that would contain this application.
+        ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
+        Throws :exc:`NoRepositoryError` if the calculated path does not
+        exist.
+        """
+        repo = os.path.join(srv_path, self.name + ".git")
         if not os.path.isdir(repo):
-            raise NoRepositoryError(app)
+            repo = os.path.join(srv_path, self.name, ".git")
+            if not os.path.isdir(repo):
+                raise NoRepositoryError(self.name)
         return repo
     def makeVersion(self, version):
+        """
+        Creates or retrieves the :class:`ApplicationVersion` singleton for the
+        specified version.
+        """
         if version not in self.versions:
             self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
         return self.versions[version]
+    def extract(self, deployment):
+        """Extracts wizard variables from a deployment."""
+        result = {}
+        for k,extractor in self.extractors.items():
+            result[k] = extractor(deployment)
+        return result
+    def parametrize(self, deployment, dir):
+        """
+        Takes a generic source checkout at dir and parametrizes
+        it according to the values of deployment.
+        """
+        variables = deployment.extract()
+        for file in self.parametrized_files:
+            fullpath = os.path.join(dir, file)
+            f = open(fullpath, "r")
+            contents = f.read()
+            f.close()
+            for key, value in variables.items():
+                if value is None: continue
+                contents = contents.replace(key, value)
+            tmp = tempfile.NamedTemporaryFile(delete=False)
+            tmp.write(contents)
+            tmp.close()
+            os.rename(tmp.name, fullpath)
+    def prepareConfig(self, deployment):
+        """
+        Takes a deployment and replaces any explicit instances
+        of a configuration variable with generic WIZARD_* constants.
+        There is a sane default implementation built on substitutions;
+        you can override this method to provide arbitrary extra
+        behavior.
+        """
+        for key, subst in self.substitutions.items():
+            subs = subst(deployment)
+            if not subs and key not in self.deprecated_keys:
+                logging.warning("No substitutions for %s" % key)
+    @property
+    def extractors(self):
+        """
+        Dictionary of variable names to extractor functions.  These functions
+        take a :class:`Deployment` as an argument and return the value of
+        the variable, or ``None`` if it could not be found.
+        See also :func:`wizard.app.filename_regex_extractor`.
+        """
+        return {}
+    @property
+    def substitutions(self):
+        """
+        Dictionary of variable names to substitution functions.  These functions
+        take a :class:`Deployment` as an argument and modify the deployment such
+        that an explicit instance of the variable is released with the generic
+        WIZARD_* constant.  See also :func:`wizard.app.filename_regex_substitution`.
+        """
+        return {}
     @staticmethod
-    def make(self, name):
-        pass
+    def make(name):
+        """Makes an application, but uses the correct subtype if available."""
+        try:
+            __import__("wizard.app." + name)
+            return getattr(wizard.app, name).Application(name)
+        except ImportError:
+            return Application(name)
 
 class ApplicationVersion(object):
-    """Represents an abstract notion of a version for an application"""
+    """Represents an abstract notion of a version for an application, where
+    ``version`` is a :class:`distutils.version.LooseVersion` and
+    ``application`` is a :class:`Application`."""
+    #: The :class:`distutils.version.LooseVersion` of this instance.
+    version = None
+    #: The :class:`Application` of this instance.
+    application = None
     def __init__(self, version, application):
-        """ `version`       Instance of distutils.LooseVersion
-            `application`   Instance of Application
-        WARNING: Please don't call this directly; instead, use getVersion()
-        on the application you want, so that this version gets registered."""
         self.version = version
         self.application = application
     @property
     def scripts_tag(self):
-        """Returns the name of the Git tag for this version"""
-        # XXX: This assumes that there's only a -scripts version
-        # which will not be true in the future.  Unfortunately, finding
-        # the "true" latest version is computationally expensive
-        return "v%s-scripts" % self.version
+        """
+        Returns the name of the Git tag for this version.
+
+        .. note::
+
+            Use this function only during migration, as it does
+            not account for the existence of ``-scripts2``.
+        """
+        return "%s-scripts" % self.pristine_tag
+    @property
+    def pristine_tag(self):
+        """
+        Returns the name of the Git tag for the pristine version corresponding
+        to this version.
+        """
+        return "%s-%s" % (self.application.name, self.version)
     def __cmp__(x, y):
         return cmp(x.version, y.version)
     @staticmethod
-    def parse(deploydir,location,applookup=None):
-        # The version of the deployment, will be:
-        #   /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z for old style installs
-        name = deploydir.split("/")[-1]
+    def parse(value):
+        """
+        Parses a line from the :term:`versions store` and return
+        :class:`ApplicationVersion`.
+
+        Use this only for cases when speed is of primary importance;
+        the data in version is unreliable and when possible, you should
+        prefer directly instantiating a Deployment and having it query
+        the autoinstall itself for information.
+
+        The `value` to parse will vary.  For old style installs, it
+        will look like::
+
+           /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
+
+        For new style installs, it will look like::
+
+           APP-x.y.z-scripts
+        """
+        name = value.split("/")[-1]
         try:
-            if name.find(" ") != -1:
-                raw_app, raw_version = name.split(" ")
-                version = raw_version[1:] # remove leading v
-                app, _ = raw_app.split(".") # remove trailing .git
-            elif name.find("-") != -1:
+            if name.find("-") != -1:
                 app, _, version = name.partition("-")
             else:
+                # kind of poor, maybe should error.  Generally this
+                # will actually result in a not found error
                 app = name
                 version = "trunk"
-        except ValueError: # mostly from the a, b = foo.split(' ')
-            raise DeploymentParseError(deploydir, location)
-        if not applookup: applookup = applications
+        except ValueError:
+            raise DeploymentParseError(deploydir)
+        return ApplicationVersion.make(app, version)
+    @staticmethod
+    def make(app, version):
+        """
+        Makes/retrieves a singleton :class:`ApplicationVersion` from
+        a``app`` and ``version`` string.
+        """
         try:
-            # defer to the application for version creation
-            return applookup[app].makeVersion(version)
+            # defer to the application for version creation to enforce
+            # singletons
+            return applications()[app].makeVersion(version)
         except KeyError:
-            raise NoSuchApplication(app, location)
+            raise NoSuchApplication(app)
 
 ## -- Exceptions --
 
-class Error(Exception):
+class Error(wizard.Error):
     """Base error class for this module"""
     pass
 
 class NoSuchApplication(Error):
-    def __init__(self, name, location):
-        self.name = name
-        self.location = location
-    def __str__(self):
-        return "ERROR: Unrecognized app '%s' at %s" % (self.name, self.location)
+    """
+    You attempted to reference a :class:`Application` named
+    ``app``, which is not recognized by Wizard.
+    """
+    #: The name of the application that does not exist.
+    app = None
+    #: The location of the autoinstall that threw this variable.
+    #: This should be set by error handling code when it is availble.
+    location = None
+    def __init__(self, app):
+        self.app = app
 
 class DeploymentParseError(Error):
-    def __init__(self, malformed, location):
-        self.malformed = malformed
-        self.location = location
-    def __str__(self):
-        return """ERROR: Unparseable '%s' at %s""" % (self.malformed, self.location)
+    """
+    Could not parse ``value`` from :term:`versions store`.
+    """
+    #: The value that failed to parse.
+    value = None
+    #: The location of the autoinstall that threw this variable.
+    #: This should be set by error handling code when it is available.
+    location = None
+    def __init__(self, value):
+        self.value = value
 
 class NoRepositoryError(Error):
+    """
+    :class:`Application` does not appear to have a Git repository
+    in the normal location.
+    """
+    #: The name of the application that does not have a Git repository.
+    app = None
     def __init__(self, app):
         self.app = app
-        self.location = "unknown"
     def __str__(self):
-        return """
-
-ERROR: Could not find repository for this application. Have
-you converted the repository over? Is the name %s
-the same as the name of the .git folder?
-""" % self.app
+        return """Could not find Git repository for '%s'.  If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
 
 # If you want, you can wrap this up into a registry and access things
 # through that, but it's not really necessary
 
-application_list = [
+_application_list = [
     "mediawiki", "wordpress", "joomla", "e107", "gallery2",
     "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
     # these are technically deprecated
     "advancedpoll", "gallery",
 ]
+_applications = None
 
-"""Hash table for looking up string application name to instance"""
-applications = dict([(n,Application(n)) for n in application_list ])
+def applications():
+    """Hash table for looking up string application name to instance"""
+    global _applications
+    if not _applications:
+        _applications = dict([(n,Application.make(n)) for n in _application_list ])
+    return _applications