Reduce duplication in test scripts, more logging.

[wizard.git] / wizard / app / __init__.py

"""
Plumbing object model for representing applications we want to
install.  This module does the heavy lifting, but you probably
want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
You'll need to know how to overload the :class:`Application` class
and use some of the functions in this module in order to specify
new applications.

There are some submodules for programming languages that define common
functions and data that may be used by applications in that language.  See:

* :mod:`wizard.app.php`

.. testsetup:: *

    import re
    import shutil
    import os
    from wizard import deploy, util
    from wizard.app import *
"""

import os.path
import re
import distutils.version
import decorator
import shlex
import logging
import shutil

import wizard
from wizard import resolve, scripts, shell, util

_application_list = [
    "mediawiki", "wordpress", "joomla", "e107", "gallery2",
    "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
    # these are technically deprecated
    "advancedpoll", "gallery",
]
_applications = None

def applications():
    """Hash table for looking up string application name to instance"""
    global _applications
    if not _applications:
        _applications = dict([(n,Application.make(n)) for n in _application_list ])
    return _applications


class Application(object):
    """
    Represents an application, i.e. mediawiki or phpbb.

    .. note::
        Many of these methods assume a specific working
        directory; prefer using the corresponding methods
        in :class:`wizard.deploy.Deployment` and its subclasses.
    """
    #: String name of the application
    name = None
    #: Dictionary of version strings to :class:`ApplicationVersion`.
    #: See also :meth:`makeVersion`.
    versions = None
    #: List of files that need to be modified when parametrizing.
    #: This is a class-wide constant, and should not normally be modified.
    parametrized_files = []
    #: Keys that are used in older versions of the application, but
    #: not for the most recent version.
    deprecated_keys = []
    #: Dictionary of variable names to extractor functions.  These functions
    #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
    #: the variable, or ``None`` if it could not be found.
    #: See also :func:`filename_regex_extractor`.
    extractors = {}
    #: Dictionary of variable names to substitution functions.  These functions
    #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
    #: that an explicit instance of the variable is released with the generic
    #: ``WIZARD_*`` constant.  See also :func:`filename_regex_substitution`.
    substitutions = {}
    #: Dictionary of file names to a list of resolutions, which are tuples of
    #: a conflict marker string and a result list.  See :mod:`wizard.resolve`
    #: for more information.
    resolutions = {}
    #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
    #: this application requires.
    install_schema = None
    def __init__(self, name):
        self.name = name
        self.versions = {}
        # cache variables
        self._extractors = {}
        self._substitutions = {}
    def repository(self, srv_path):
        """
        Returns the Git repository that would contain this application.
        ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
        """
        repo = os.path.join(srv_path, self.name + ".git")
        if not os.path.isdir(repo):
            repo = os.path.join(srv_path, self.name, ".git")
            if not os.path.isdir(repo):
                raise NoRepositoryError(self.name)
        return repo
    def makeVersion(self, version):
        """
        Creates or retrieves the :class:`ApplicationVersion` singleton for the
        specified version.
        """
        if version not in self.versions:
            self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
        return self.versions[version]
    def extract(self, deployment):
        """
        Extracts wizard variables from a deployment.  Default implementation
        uses :attr:`extractors`.
        """
        result = {}
        for k,extractor in self.extractors.items():
            result[k] = extractor(deployment)
        return result
    def parametrize(self, deployment, ref_deployment):
        """
        Takes a generic source checkout and parametrizes it according to the
        values of ``deployment``.  This function operates on the current
        working directory.  ``deployment`` should **not** be the same as the
        current working directory.  Default implementation uses
        :attr:`parametrized_files` and a simple search and replace on those
        files.
        """
        variables = ref_deployment.extract()
        for file in self.parametrized_files:
            try:
                contents = open(file, "r").read()
            except IOError:
                continue
            for key, value in variables.items():
                if value is None: continue
                contents = contents.replace(key, value)
            f = open(file, "w")
            f.write(contents)
    def resolveConflicts(self, deployment):
        """
        Resolves conflicted files in the current working directory.  Returns
        whether or not all conflicted files were resolved or not.  Fully
        resolved files are added to the index, but no commit is made.  The
        default implementation uses :attr:`resolutions`.
        """
        resolved = True
        sh = shell.Shell()
        for status in sh.eval("git", "ls-files", "--unmerged").splitlines():
            file = status.split()[-1]
            if file in self.resolutions:
                contents = open(file, "r").read()
                for spec, result in self.resolutions[file]:
                    old_contents = contents
                    contents = resolve.resolve(contents, spec, result)
                    if old_contents != contents:
                        logging.info("Did resolution with spec:\n" + spec)
                open(file, "w").write(contents)
                if not resolve.is_conflict(contents):
                    sh.call("git", "add", file)
                else:
                    resolved = False
            else:
                resolved = False
        return resolved
    def prepareMerge(self, deployment):
        """
        Performs various edits to files in the current working directory in
        order to make a merge go more smoothly.  This is usually
        used to fix botched line-endings.  If you add new files,
        you have to 'git add' them; this is not necessary for edits.
        By default this is a no-op; subclasses should replace this
        with useful behavior.
        """
        pass
    def prepareConfig(self, deployment):
        """
        Takes a deployment and replaces any explicit instances
        of a configuration variable with generic ``WIZARD_*`` constants.
        The default implementation uses :attr:`substitutions`, and
        emits warnings when it encounters keys in :attr:`deprecated_keys`.
        """
        for key, subst in self.substitutions.items():
            subs = subst(deployment)
            if not subs and key not in self.deprecated_keys:
                logging.warning("No substitutions for %s" % key)
    def install(self, version, options):
        """
        Run for 'wizard configure' (and, by proxy, 'wizard install') to
        configure an application.  This assumes that the current working
        directory is a deployment.  (Unlike its kin, this function does not
        take a :class:`wizard.deploy.Deployment` as a parameter.)  Subclasses should
        provide an implementation.
        """
        raise NotImplementedError
    def upgrade(self, deployment, version, options):
        """
        Run for 'wizard upgrade' to upgrade database schemas and other
        non-versioned data in an application after the filesystem has been
        upgraded.  This assumes that the current working directory is the
        deployment.  Subclasses should provide an implementation.
        """
        raise NotImplementedError
    def backup(self, deployment, outdir, options):
        """
        Run for 'wizard backup' and upgrades to backup database schemas
        and other non-versioned data in an application.  ``outdir`` is
        the directory that backup files should be placed.  This assumes
        that the current working directory is the deployment.  Subclasses
        should provide an implementation, even if it is a no-op.

        .. note::
            Static user files may not need to be backed up, since in
            many applications upgrades do not modify static files.
        """
        raise NotImplementedError
    def restore(self, deployment, backup_dir, options):
        """
        Run for 'wizard restore' and failed upgrades to restore database
        and other non-versioned data to a backed up version.  This assumes
        that the current working directory is the deployment.  Subclasses
        should provide an implementation.
        """
        raise NotImplementedError
    def detectVersion(self, deployment):
        """
        Checks source files to determine the version manually.  This assumes
        that the current working directory is the deployment.  Subclasses
        should provide an implementation.
        """
        raise NotImplementedError
    def detectVersionFromFile(self, filename, regex):
        """
        Helper method that detects a version by using a regular expression
        from a file.  The regexed value is passed through :mod:`shlex`.
        This assumes that the current working directory is the deployment.
        """
        contents = open(filename).read()
        match = regex.search(contents)
        if not match: return None
        return distutils.version.LooseVersion(shlex.split(match.group(2))[0])
    def download(self, version):
        """
        Returns a URL that can be used to download a tarball of ``version`` of
        this application.
        """
        raise NotImplementedError
    def checkWeb(self, deployment):
        """
        Checks if the autoinstall is viewable from the web.  Subclasses should
        provide an implementation.

        .. note::
            Finding a reasonable heuristic that works across skinning
            choices can be difficult.  We've had reasonable success
            searching for metadata.  Be sure that the standard error
            page does not contain the features you search for.  Try
            not to depend on pages that are not the main page.
        """
        raise NotImplementedError
    def checkWebPage(self, deployment, page, output):
        """
        Checks if a given page of an autoinstall contains a particular string.
        """
        page = deployment.fetch(page)
        result = page.find(output) != -1
        if result:
            logging.debug("checkWebPage (passed):\n\n" + page)
        else:
            logging.info("checkWebPage (failed):\n\n" + page)
        return result
    def checkConfig(self, deployment):
        """
        Checks whether or not an autoinstall has been configured/installed
        for use.  Assumes that the current working directory is the deployment.
        Subclasses should provide an implementation.
        """
        # XXX: Unfortunately, this doesn't quite work because we package
        # bogus config files in the -scripts versions of installs.  Maybe
        # we should check a hash or something?
        raise NotImplementedError
    @staticmethod
    def make(name):
        """Makes an application, but uses the correct subtype if available."""
        try:
            __import__("wizard.app." + name)
            return getattr(wizard.app, name).Application(name)
        except ImportError:
            return Application(name)

class ApplicationVersion(object):
    """Represents an abstract notion of a version for an application, where
    ``version`` is a :class:`distutils.version.LooseVersion` and
    ``application`` is a :class:`Application`."""
    #: The :class:`distutils.version.LooseVersion` of this instance.
    version = None
    #: The :class:`Application` of this instance.
    application = None
    def __init__(self, version, application):
        self.version = version
        self.application = application
    @property
    def tag(self):
        """
        Returns the name of the git describe tag for the commit the user is
        presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
        """
        return "%s-%s" % (self.application, self.version)
    @property
    def scripts_tag(self):
        """
        Returns the name of the Git tag for this version.
        """
        end = str(self.version).partition('-scripts')[2].partition('-')[0]
        return "%s-scripts%s" % (self.pristine_tag, end)
    @property
    def pristine_tag(self):
        """
        Returns the name of the Git tag for the pristine version corresponding
        to this version.
        """
        return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
    def __cmp__(self, y):
        return cmp(self.version, y.version)
    @staticmethod
    def parse(value):
        """
        Parses a line from the :term:`versions store` and return
        :class:`ApplicationVersion`.

        Use this only for cases when speed is of primary importance;
        the data in version is unreliable and when possible, you should
        prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
        the autoinstall itself for information.

        The `value` to parse will vary.  For old style installs, it
        will look like::

           /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z

        For new style installs, it will look like::

           APP-x.y.z-scripts
        """
        name = value.split("/")[-1]
        try:
            if name.find("-") != -1:
                app, _, version = name.partition("-")
            else:
                # kind of poor, maybe should error.  Generally this
                # will actually result in a not found error
                app = name
                version = "trunk"
        except ValueError:
            raise DeploymentParseError(value)
        return ApplicationVersion.make(app, version)
    @staticmethod
    def make(app, version):
        """
        Makes/retrieves a singleton :class:`ApplicationVersion` from
        a``app`` and ``version`` string.
        """
        try:
            # defer to the application for version creation to enforce
            # singletons
            return applications()[app].makeVersion(version)
        except KeyError:
            raise NoSuchApplication(app)

def expand_re(val):
    """
    Takes a tree of values (implement using nested lists) and
    transforms them into regular expressions.

        >>> expand_re('*')
        '\\\\*'
        >>> expand_re(['a', 'b'])
        '(?:a|b)'
        >>> expand_re(['*', ['b', 'c']])
        '(?:\\\\*|(?:b|c))'
    """
    if isinstance(val, str):
        return re.escape(val)
    else:
        return '(?:' + '|'.join(map(expand_re, val)) + ')'

def make_extractors(seed):
    """
    Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into
    extractor functions (which take a :class:`wizard.deploy.Deployment`
    and return the value of the second subpattern of ``regex`` when matched
    with the contents of ``file``).
    """
    return util.dictmap(lambda a: filename_regex_extractor(*a), seed)

def make_substitutions(seed):
    """
    Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into substitution
    functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
    of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
    """
    return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)

# The following two functions are *highly* functional, and I recommend
# not touching them unless you know what you're doing.

def filename_regex_extractor(file, regex):
    """
    .. highlight:: haskell

    Given a relative file name ``file``, a regular expression ``regex``, and a
    :class:`wizard.deploy.Deployment` extracts a value out of the file in that
    deployment.  This function is curried, so you pass just ``file`` and
    ``regex``, and then pass ``deployment`` to the resulting function.

    Its Haskell-style type signature would be::

        Filename -> Regex -> (Deployment -> String)

    The regular expression requires a very specific form, essentially ``()()()``
    (with the second subgroup being the value to extract).  These enables
    the regular expression to be used equivalently with filename

    .. highlight:: python

    For convenience purposes, we also accept ``[Filename]``, in which case
    we use the first entry (index 0).  Passing an empty list is invalid.

        >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
        >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
        >>> f(deploy.Deployment("."))
        '3'
        >>> os.unlink("test-settings.extractor.ini")

    .. note::
        The first application of ``regex`` and ``file`` is normally performed
        at compile-time inside a submodule; the second application is
        performed at runtime.
    """
    if not isinstance(file, str):
        file = file[0]
    def h(deployment):
        try:
            contents = deployment.read(file) # cached
        except IOError:
            return None
        match = regex.search(contents)
        if not match: return None
        # assumes that the second match is the one we want.
        return match.group(2)
    return h

def filename_regex_substitution(key, files, regex):
    """
    .. highlight:: haskell

    Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
    regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
    performs a substitution of the second subpattern of ``regex``
    with ``key``.  Returns the number of replacements made.  This function
    is curried, so you pass just ``key``, ``files`` and ``regex``, and
    then pass ``deployment`` to the resulting function.

    Its Haskell-style type signature would be::

        Key -> ([File], Regex) -> (Deployment -> IO Int)

    .. highlight:: python

    For convenience purposes, we also accept ``Filename``, in which case it is treated
    as a single item list.

        >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
        >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
        >>> f(deploy.Deployment("."))
        1
        >>> print open("test-settings.substitution.ini", "r").read()
        config_var = WIZARD_KEY
        >>> os.unlink("test-settings.substitution.ini")
    """
    if isinstance(files, str):
        files = (files,)
    def h(deployment):
        base = deployment.location
        subs = 0
        for file in files:
            file = os.path.join(base, file)
            try:
                contents = open(file, "r").read()
                contents, n = regex.subn("\\1" + key + "\\3", contents)
                subs += n
                open(file, "w").write(contents)
            except IOError:
                pass
        return subs
    return h

# XXX: rename to show that it's mysql specific
def backup_database(outdir, deployment):
    """
    Generic database backup function for MySQL.  Assumes that ``WIZARD_DBNAME``
    is extractable, and that :func:`wizard.scripts.get_sql_credentials`
    works.
    """
    sh = shell.Shell()
    outfile = os.path.join(outdir, "db.sql")
    try:
        sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment))
        sh.call("gzip", "--best", outfile)
    except shell.CallError as e:
        shutil.rmtree(outdir)
        raise BackupFailure(e.stderr)

def restore_database(backup_dir, deployment):
    """
    Generic database restoration function for MySQL.  See :func:`backup_database`
    for the assumptions that we make.
    """
    sh = shell.Shell()
    if not os.path.exists(backup_dir):
        raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
    sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
    sh.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
    sql.seek(0)
    sh.call("mysql", *get_mysql_args(deployment), stdin=sql)
    sql.close()

def get_mysql_args(d):
    """
    Extracts arguments that would be passed to the command line mysql utility
    from a deployment.
    """
    # XXX: add support for getting these out of options
    vars = d.extract()
    if 'WIZARD_DBNAME' not in vars:
        raise BackupFailure("Could not determine database name")
    triplet = scripts.get_sql_credentials(vars)
    args = []
    if triplet is not None:
        server, user, password = triplet
        args += ["-h", server, "-u", user, "-p" + password]
    name = shlex.split(vars['WIZARD_DBNAME'])[0]
    args.append(name)
    return args

class Error(wizard.Error):
    """Generic error class for this module."""
    pass

class NoRepositoryError(Error):
    """
    :class:`Application` does not appear to have a Git repository
    in the normal location.
    """
    #: The name of the application that does not have a Git repository.
    app = None
    def __init__(self, app):
        self.app = app
    def __str__(self):
        return """Could not find Git repository for '%s'.  If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app

class DeploymentParseError(Error):
    """
    Could not parse ``value`` from :term:`versions store`.
    """
    #: The value that failed to parse.
    value = None
    #: The location of the autoinstall that threw this variable.
    #: This should be set by error handling code when it is available.
    location = None
    def __init__(self, value):
        self.value = value

class NoSuchApplication(Error):
    """
    You attempted to reference a :class:`Application` named
    ``app``, which is not recognized by Wizard.
    """
    #: The name of the application that does not exist.
    app = None
    #: The location of the autoinstall that threw this variable.
    #: This should be set by error handling code when it is availble.
    location = None
    def __init__(self, app):
        self.app = app

class Failure(Error):
    """
    Represents a failure when performing some double-dispatched operation
    such as an installation or an upgrade.  Failure classes are postfixed
    with Failure, not Error.
    """
    pass

class InstallFailure(Error):
    """Installation failed for unknown reason."""
    def __str__(self):
        return """Installation failed for unknown reason."""

class RecoverableInstallFailure(InstallFailure):
    """
    Installation failed, but we were able to determine what the
    error was, and should give the user a second chance if we were
    running interactively.
    """
    #: List of the errors that were found.
    errors = None
    def __init__(self, errors):
        self.errors = errors
    def __str__(self):
        return """Installation failed due to the following errors: %s""" % ", ".join(self.errors)

class UpgradeFailure(Failure):
    """Upgrade script failed."""
    #: String details of failure (possibly stdout or stderr output)
    details = None
    def __init__(self, details):
        self.details = details
    def __str__(self):
        return """

ERROR: Upgrade script failed, details:

%s""" % self.details

class UpgradeVerificationFailure(Failure):
    """Upgrade script passed, but website wasn't accessible afterwards"""
    def __str__(self):
        return """

ERROR: Upgrade script passed, but website wasn't accessible afterwards.  Check
the debug logs for the contents of the page."""

class BackupFailure(Failure):
    """Backup script failed."""
    #: String details of failure
    details = None
    def __init__(self, details):
        self.details = details
    def __str__(self):
        return """

ERROR: Backup script failed, details:

%s""" % self.details

class RestoreFailure(Failure):
    """Restore script failed."""
    #: String details of failure
    details = None
    def __init__(self, details):
        self.details = details
    def __str__(self):
        return """

ERROR: Restore script failed, details:

%s""" % self.details