X-Git-Url: https://scripts.mit.edu/gitweb/wizard.git/blobdiff_plain/e6525070de9987c2f9cf87443d474b8dd8a69713..7b2533e1821daeb667b60c00dda4fd7d4ad3c41e:/wizard/app/__init__.py

diff --git a/wizard/app/__init__.py b/wizard/app/__init__.py
index ac35f40..929cacf 100644
--- a/wizard/app/__init__.py
+++ b/wizard/app/__init__.py
@@ -1,82 +1,628 @@
+"""
+Plumbing object model for representing applications we want to
+install.  This module does the heavy lifting, but you probably
+want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
+You'll need to know how to overload the :class:`Application` class
+and use some of the functions in this module in order to specify
+new applications.
+
+.. testsetup:: *
+
+    import re
+    import shutil
+    import os
+    from wizard import deploy, util
+    from wizard.app import *
+"""
+
 import os.path
 import re
+import distutils.version
+import decorator
+import shlex
+import logging
+import shutil
+
+import wizard
+from wizard import resolve, scripts, shell, util
+
+_application_list = [
+    "mediawiki", "wordpress", "joomla", "e107", "gallery2",
+    "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
+    # these are technically deprecated
+    "advancedpoll", "gallery",
+]
+_applications = None
+
+def applications():
+    """Hash table for looking up string application name to instance"""
+    global _applications
+    if not _applications:
+        _applications = dict([(n,Application.make(n)) for n in _application_list ])
+    return _applications
+
+
+class Application(object):
+    """
+    Represents an application, i.e. mediawiki or phpbb.
+
+    .. note::
+        Many of these methods assume a specific working
+        directory; prefer using the corresponding methods
+        in :class:`wizard.deploy.Deployment` and its subclasses.
+    """
+    #: String name of the application
+    name = None
+    #: Dictionary of version strings to :class:`ApplicationVersion`.
+    #: See also :meth:`makeVersion`.
+    versions = None
+    #: List of files that need to be modified when parametrizing.
+    #: This is a class-wide constant, and should not normally be modified.
+    parametrized_files = []
+    #: Keys that are used in older versions of the application, but
+    #: not for the most recent version.
+    deprecated_keys = []
+    #: Dictionary of variable names to extractor functions.  These functions
+    #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
+    #: the variable, or ``None`` if it could not be found.
+    #: See also :func:`filename_regex_extractor`.
+    extractors = {}
+    #: Dictionary of variable names to substitution functions.  These functions
+    #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
+    #: that an explicit instance of the variable is released with the generic
+    #: ``WIZARD_*`` constant.  See also :func:`filename_regex_substitution`.
+    substitutions = {}
+    #: Dictionary of file names to a list of resolutions, which are tuples of
+    #: a conflict marker string and a result list.  See :mod:`wizard.resolve`
+    #: for more information.
+    resolutions = {}
+    #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
+    #: this application requires.
+    install_schema = None
+    def __init__(self, name):
+        self.name = name
+        self.versions = {}
+        # cache variables
+        self._extractors = {}
+        self._substitutions = {}
+    def repository(self, srv_path):
+        """
+        Returns the Git repository that would contain this application.
+        ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
+        """
+        repo = os.path.join(srv_path, self.name + ".git")
+        if not os.path.isdir(repo):
+            repo = os.path.join(srv_path, self.name, ".git")
+            if not os.path.isdir(repo):
+                raise NoRepositoryError(self.name)
+        return repo
+    def makeVersion(self, version):
+        """
+        Creates or retrieves the :class:`ApplicationVersion` singleton for the
+        specified version.
+        """
+        if version not in self.versions:
+            self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
+        return self.versions[version]
+    def extract(self, deployment):
+        """
+        Extracts wizard variables from a deployment.  Default implementation
+        uses :attr:`extractors`.
+        """
+        result = {}
+        for k,extractor in self.extractors.items():
+            result[k] = extractor(deployment)
+        return result
+    def parametrize(self, deployment, ref_deployment):
+        """
+        Takes a generic source checkout and parametrizes it according to the
+        values of ``deployment``.  This function operates on the current
+        working directory.  ``deployment`` should **not** be the same as the
+        current working directory.  Default implementation uses
+        :attr:`parametrized_files` and a simple search and replace on those
+        files.
+        """
+        variables = ref_deployment.extract()
+        for file in self.parametrized_files:
+            try:
+                contents = open(file, "r").read()
+            except IOError:
+                continue
+            for key, value in variables.items():
+                if value is None: continue
+                contents = contents.replace(key, value)
+            f = open(file, "w")
+            f.write(contents)
+    def resolveConflicts(self, deployment):
+        """
+        Resolves conflicted files in the current working directory.  Returns
+        whether or not all conflicted files were resolved or not.  Fully
+        resolved files are added to the index, but no commit is made.  The
+        default implementation uses :attr:`resolutions`.
+        """
+        resolved = True
+        sh = shell.Shell()
+        for status in sh.eval("git", "ls-files", "--unmerged").splitlines():
+            file = status.split()[-1]
+            if file in self.resolutions:
+                contents = open(file, "r").read()
+                for spec, result in self.resolutions[file]:
+                    old_contents = contents
+                    contents = resolve.resolve(contents, spec, result)
+                    if old_contents != contents:
+                        logging.info("Did resolution with spec:\n" + spec)
+                open(file, "w").write(contents)
+                if not resolve.is_conflict(contents):
+                    sh.call("git", "add", file)
+                else:
+                    resolved = False
+            else:
+                resolved = False
+        return resolved
+    def prepareMerge(self, deployment):
+        """
+        Performs various edits to files in the current working directory in
+        order to make a merge go more smoothly.  This is usually
+        used to fix botched line-endings.  If you add new files,
+        you have to 'git add' them; this is not necessary for edits.
+        By default this is a no-op; subclasses should replace this
+        with useful behavior.
+        """
+        pass
+    def prepareConfig(self, deployment):
+        """
+        Takes a deployment and replaces any explicit instances
+        of a configuration variable with generic ``WIZARD_*`` constants.
+        The default implementation uses :attr:`substitutions`, and
+        emits warnings when it encounters keys in :attr:`deprecated_keys`.
+        """
+        for key, subst in self.substitutions.items():
+            subs = subst(deployment)
+            if not subs and key not in self.deprecated_keys:
+                logging.warning("No substitutions for %s" % key)
+    def install(self, version, options):
+        """
+        Run for 'wizard configure' (and, by proxy, 'wizard install') to
+        configure an application.  This assumes that the current working
+        directory is a deployment.  (Unlike its kin, this function does not
+        take a :class:`wizard.deploy.Deployment` as a parameter.)  Subclasses should
+        provide an implementation.
+        """
+        raise NotImplementedError
+    def upgrade(self, deployment, version, options):
+        """
+        Run for 'wizard upgrade' to upgrade database schemas and other
+        non-versioned data in an application after the filesystem has been
+        upgraded.  This assumes that the current working directory is the
+        deployment.  Subclasses should provide an implementation.
+        """
+        raise NotImplementedError
+    def backup(self, deployment, outdir, options):
+        """
+        Run for 'wizard backup' and upgrades to backup database schemas
+        and other non-versioned data in an application.  ``outdir`` is
+        the directory that backup files should be placed.  This assumes
+        that the current working directory is the deployment.  Subclasses
+        should provide an implementation, even if it is a no-op.
+
+        .. note::
+            Static user files may not need to be backed up, since in
+            many applications upgrades do not modify static files.
+        """
+        raise NotImplementedError
+    def restore(self, deployment, backup_dir, options):
+        """
+        Run for 'wizard restore' and failed upgrades to restore database
+        and other non-versioned data to a backed up version.  This assumes
+        that the current working directory is the deployment.  Subclasses
+        should provide an implementation.
+        """
+        raise NotImplementedError
+    def detectVersion(self, deployment):
+        """
+        Checks source files to determine the version manually.  This assumes
+        that the current working directory is the deployment.  Subclasses
+        should provide an implementation.
+        """
+        raise NotImplementedError
+    def download(self, version):
+        """
+        Returns a URL that can be used to download a tarball of ``version`` of
+        this application.
+        """
+        raise NotImplementedError
+    def checkWeb(self, deployment, output=None):
+        """
+        Checks if the autoinstall is viewable from the web.  To get
+        the HTML source that was retrieved, pass a variable containing
+        an empty list to ``output``; it will be mutated to have its
+        first element be the output.  Subclasses should provide an
+        implementation.
+
+        .. note::
+            Finding a reasonable heuristic that works across skinning
+            choices can be difficult.  We've had reasonable success
+            searching for metadata.  Be sure that the standard error
+            page does not contain the features you search for.  Try
+            not to depend on pages that are not the main page.
+        """
+        raise NotImplementedError
+    def checkConfig(self, deployment):
+        """
+        Checks whether or not an autoinstall has been configured/installed
+        for use.  Assumes that the current working directory is the deployment.
+        Subclasses should provide an implementation.
+        """
+        # XXX: Unfortunately, this doesn't quite work because we package
+        # bogus config files in the -scripts versions of installs.  Maybe
+        # we should check a hash or something?
+        raise NotImplementedError
+    @staticmethod
+    def make(name):
+        """Makes an application, but uses the correct subtype if available."""
+        try:
+            __import__("wizard.app." + name)
+            return getattr(wizard.app, name).Application(name)
+        except ImportError:
+            return Application(name)
+
+class ApplicationVersion(object):
+    """Represents an abstract notion of a version for an application, where
+    ``version`` is a :class:`distutils.version.LooseVersion` and
+    ``application`` is a :class:`Application`."""
+    #: The :class:`distutils.version.LooseVersion` of this instance.
+    version = None
+    #: The :class:`Application` of this instance.
+    application = None
+    def __init__(self, version, application):
+        self.version = version
+        self.application = application
+    @property
+    def tag(self):
+        """
+        Returns the name of the git describe tag for the commit the user is
+        presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
+        """
+        return "%s-%s" % (self.application, self.version)
+    @property
+    def scripts_tag(self):
+        """
+        Returns the name of the Git tag for this version.
+        """
+        end = str(self.version).partition('-scripts')[2].partition('-')[0]
+        return "%s-scripts%s" % (self.pristine_tag, end)
+    @property
+    def pristine_tag(self):
+        """
+        Returns the name of the Git tag for the pristine version corresponding
+        to this version.
+        """
+        return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
+    def __cmp__(self, y):
+        return cmp(self.version, y.version)
+    @staticmethod
+    def parse(value):
+        """
+        Parses a line from the :term:`versions store` and return
+        :class:`ApplicationVersion`.
+
+        Use this only for cases when speed is of primary importance;
+        the data in version is unreliable and when possible, you should
+        prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
+        the autoinstall itself for information.
+
+        The `value` to parse will vary.  For old style installs, it
+        will look like::
+
+           /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
+
+        For new style installs, it will look like::
+
+           APP-x.y.z-scripts
+        """
+        name = value.split("/")[-1]
+        try:
+            if name.find("-") != -1:
+                app, _, version = name.partition("-")
+            else:
+                # kind of poor, maybe should error.  Generally this
+                # will actually result in a not found error
+                app = name
+                version = "trunk"
+        except ValueError:
+            raise DeploymentParseError(value)
+        return ApplicationVersion.make(app, version)
+    @staticmethod
+    def make(app, version):
+        """
+        Makes/retrieves a singleton :class:`ApplicationVersion` from
+        a``app`` and ``version`` string.
+        """
+        try:
+            # defer to the application for version creation to enforce
+            # singletons
+            return applications()[app].makeVersion(version)
+        except KeyError:
+            raise NoSuchApplication(app)
 
 def expand_re(val):
+    """
+    Takes a tree of values (implement using nested lists) and
+    transforms them into regular expressions.
+
+        >>> expand_re('*')
+        '\\\\*'
+        >>> expand_re(['a', 'b'])
+        '(?:a|b)'
+        >>> expand_re(['*', ['b', 'c']])
+        '(?:\\\\*|(?:b|c))'
+    """
     if isinstance(val, str):
         return re.escape(val)
     else:
         return '(?:' + '|'.join(map(expand_re, val)) + ')'
 
-def filename_regex_extractor(f):
+def make_extractors(seed):
+    """
+    Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into
+    extractor functions (which take a :class:`wizard.deploy.Deployment`
+    and return the value of the second subpattern of ``regex`` when matched
+    with the contents of ``file``).
+    """
+    return util.dictmap(lambda a: filename_regex_extractor(*a), seed)
+
+def make_substitutions(seed):
+    """
+    Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into substitution
+    functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
+    of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
     """
-    This is a decorator to apply to functions that take a name and return
-    (filename, RegexObject) tuples.  It converts it into a function
-    that takes a name and returns another function (the actual extractor)
-    which takes a deployment and returns the value of the extracted variable.
+    return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)
 
-    The regular expression requires a very specific form, essentially ()()()
-    (with the second subgroup being the value we care about), so that we can
-    reuse the regex for other things.
+# The following two functions are *highly* functional, and I recommend
+# not touching them unless you know what you're doing.
+
+def filename_regex_extractor(file, regex):
+    """
+    .. highlight:: haskell
+
+    Given a relative file name ``file``, a regular expression ``regex``, and a
+    :class:`wizard.deploy.Deployment` extracts a value out of the file in that
+    deployment.  This function is curried, so you pass just ``file`` and
+    ``regex``, and then pass ``deployment`` to the resulting function.
 
     Its Haskell-style type signature would be::
 
-        (String -> (Filename, Regex)) -> (String -> (Deployment -> String))
+        Filename -> Regex -> (Deployment -> String)
+
+    The regular expression requires a very specific form, essentially ``()()()``
+    (with the second subgroup being the value to extract).  These enables
+    the regular expression to be used equivalently with filename
+
+    .. highlight:: python
 
-    For convenience purposes, we also accept [Filename], in which case
+    For convenience purposes, we also accept ``[Filename]``, in which case
     we use the first entry (index 0).  Passing an empty list is invalid.
+
+        >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
+        >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
+        >>> f(deploy.Deployment("."))
+        '3'
+        >>> os.unlink("test-settings.extractor.ini")
+
+    .. note::
+        The first application of ``regex`` and ``file`` is normally performed
+        at compile-time inside a submodule; the second application is
+        performed at runtime.
     """
-    def g(var):
-        file, regex = f(var)
-        if not isinstance(file, str):
-            file = file[0]
-        def h(deployment):
-            try:
-                contents = deployment.read(file) # cached
-            except IOError:
-                return None
-            match = regex.search(contents)
-            if not match: return None
-            # assumes that the second match is the one we want.
-            return match.group(2)
-        return h
-    return g
-
-def filename_regex_substitution(f):
-    """
-    This is a decorator to apply to functions that take a name and return
-    (filename, RegexObject) tuples.  It converts it into a function
-    that takes a name and returns another function (that does substitution)
-    which takes a deployment and modifies its files to replace explicit
-    values with their generic WIZARD_* equivalents.  The final function returns
-    the number of replacements made.
-
-    The regular expression requires a very specific form, essentially ()()()
-    (with the second subgroup being the value to be replaced).
+    if not isinstance(file, str):
+        file = file[0]
+    def h(deployment):
+        try:
+            contents = deployment.read(file) # cached
+        except IOError:
+            return None
+        match = regex.search(contents)
+        if not match: return None
+        # assumes that the second match is the one we want.
+        return match.group(2)
+    return h
+
+def filename_regex_substitution(key, files, regex):
+    """
+    .. highlight:: haskell
+
+    Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
+    regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
+    performs a substitution of the second subpattern of ``regex``
+    with ``key``.  Returns the number of replacements made.  This function
+    is curried, so you pass just ``key``, ``files`` and ``regex``, and
+    then pass ``deployment`` to the resulting function.
 
     Its Haskell-style type signature would be::
 
-        (String -> ([Filename], Regex)) -> (String -> (Deployment -> IO Int))
+        Key -> ([File], Regex) -> (Deployment -> IO Int)
+
+    .. highlight:: python
 
-    For convenience purposes, we also accept Filename, in which case it is treated
+    For convenience purposes, we also accept ``Filename``, in which case it is treated
     as a single item list.
+
+        >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
+        >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
+        >>> f(deploy.Deployment("."))
+        1
+        >>> print open("test-settings.substitution.ini", "r").read()
+        config_var = WIZARD_KEY
+        >>> os.unlink("test-settings.substitution.ini")
+    """
+    if isinstance(files, str):
+        files = (files,)
+    def h(deployment):
+        base = deployment.location
+        subs = 0
+        for file in files:
+            file = os.path.join(base, file)
+            try:
+                contents = open(file, "r").read()
+                contents, n = regex.subn("\\1" + key + "\\3", contents)
+                subs += n
+                open(file, "w").write(contents)
+            except IOError:
+                pass
+        return subs
+    return h
+
+# XXX: rename to show that it's mysql specific
+def backup_database(outdir, deployment):
+    """
+    Generic database backup function for MySQL.  Assumes that ``WIZARD_DBNAME``
+    is extractable, and that :func:`wizard.scripts.get_sql_credentials`
+    works.
+    """
+    sh = shell.Shell()
+    outfile = os.path.join(outdir, "db.sql")
+    try:
+        sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment))
+        sh.call("gzip", "--best", outfile)
+    except shell.CallError as e:
+        shutil.rmtree(outdir)
+        raise BackupFailure(e.stderr)
+
+def restore_database(backup_dir, deployment):
+    """
+    Generic database restoration function for MySQL.  See :func:`backup_database`
+    for the assumptions that we make.
+    """
+    sh = shell.Shell()
+    if not os.path.exists(backup_dir):
+        raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
+    sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
+    sh.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
+    sql.seek(0)
+    sh.call("mysql", *get_mysql_args(deployment), stdin=sql)
+    sql.close()
+
+def get_mysql_args(d):
+    """
+    Extracts arguments that would be passed to the command line mysql utility
+    from a deployment.
+    """
+    # XXX: add support for getting these out of options
+    vars = d.extract()
+    if 'WIZARD_DBNAME' not in vars:
+        raise app.BackupFailure("Could not determine database name")
+    triplet = scripts.get_sql_credentials(vars)
+    args = []
+    if triplet is not None:
+        server, user, password = triplet
+        args += ["-h", server, "-u", user, "-p" + password]
+    name = shlex.split(vars['WIZARD_DBNAME'])[0]
+    args.append(name)
+    return args
+
+class Error(wizard.Error):
+    """Generic error class for this module."""
+    pass
+
+class RecoverableFailure(Error):
+    """
+    The installer failed, but we were able to determine what the
+    error was, and should give the user a second chance if we were
+    running interactively.
+    """
+    #: List of the errors that were found.
+    errors = None
+    def __init__(self, errors):
+        self.errors = errors
+    def __str__(self):
+        return """Installation failed due to the following errors: %s""" % ", ".join(self.errors)
+
+class NoRepositoryError(Error):
+    """
+    :class:`Application` does not appear to have a Git repository
+    in the normal location.
+    """
+    #: The name of the application that does not have a Git repository.
+    app = None
+    def __init__(self, app):
+        self.app = app
+    def __str__(self):
+        return """Could not find Git repository for '%s'.  If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
+
+class DeploymentParseError(Error):
+    """
+    Could not parse ``value`` from :term:`versions store`.
     """
-    def g(key, var):
-        files, regex = f(var)
-        if isinstance(files, str):
-            files = (files,)
-        def h(deployment):
-            base = deployment.location
-            subs = 0
-            for file in files:
-                file = os.path.join(base, file)
-                try:
-                    contents = open(file, "r").read()
-                    contents, n = regex.subn("\\1" + key + "\\3", contents)
-                    subs += n
-                    open(file, "w").write(contents)
-                except IOError:
-                    pass
-            return subs
-        return h
-    return g
+    #: The value that failed to parse.
+    value = None
+    #: The location of the autoinstall that threw this variable.
+    #: This should be set by error handling code when it is available.
+    location = None
+    def __init__(self, value):
+        self.value = value
+
+class NoSuchApplication(Error):
+    """
+    You attempted to reference a :class:`Application` named
+    ``app``, which is not recognized by Wizard.
+    """
+    #: The name of the application that does not exist.
+    app = None
+    #: The location of the autoinstall that threw this variable.
+    #: This should be set by error handling code when it is availble.
+    location = None
+    def __init__(self, app):
+        self.app = app
+
+class UpgradeFailure(Error):
+    """Upgrade script failed."""
+    #: String details of failure (possibly stdout or stderr output)
+    details = None
+    def __init__(self, details):
+        self.details = details
+    def __str__(self):
+        return """
+
+ERROR: Upgrade script failed, details:
+
+%s""" % self.details
+
+class UpgradeVerificationFailure(Error):
+    """Upgrade script passed, but website wasn't accessible afterwards"""
+    #: String details of failure (possibly stdout or stderr output)
+    details = None
+    def __init__(self, details):
+        self.details = details
+    def __str__(self):
+        return """
+
+ERROR: Upgrade script passed, but website wasn't accessible afterwards. Details:
+
+%s""" % self.details
+
+class BackupFailure(Error):
+    """Backup script failed."""
+    #: String details of failure
+    details = None
+    def __init__(self, details):
+        self.details = details
+    def __str__(self):
+        return """
+
+ERROR: Backup script failed, details:
+
+%s""" % self.details
+
+class RestoreFailure(Error):
+    """Restore script failed."""
+    #: String details of failure
+    details = None
+    def __init__(self, details):
+        self.details = details
+    def __str__(self):
+        return """
+
+ERROR: Restore script failed, details:
 
+%s""" % self.details