Update MediaWiki config regexes

[wizard.git] / wizard / app / __init__.py

diff --git a/wizard/app/__init__.py b/wizard/app/__init__.py
index 4cfd54daaa85c550989c2d1b10226ab306a7d8de..b653f524074eb5a3038041f6baa8e9f1473be1b0 100644 (file)
--- a/wizard/app/__init__.py
+++ b/wizard/app/__init__.py
@@ -33,6 +33,7 @@ functions and data that may be used by applications in that language.  See:
 """
 
 import os.path
 """
 
 import os.path

+import subprocess

 import re
 import distutils.version
 import decorator
 import re
 import distutils.version
 import decorator


@@ -45,53 +46,44 @@ import string
 import urlparse
 import tempfile
 import pkg_resources
 import urlparse
 import tempfile
 import pkg_resources

+import traceback

 
 import wizard
 
 import wizard

-from wizard import resolve, scripts, shell, sql, util
-
-# SCRIPTS SPECIFIC
-_scripts_application_list = [
-    "mediawiki", "wordpress", "joomla", "e107", "gallery2",
-    "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
-    "rails",
-    # these are technically deprecated
-    "advancedpoll", "gallery",
-]
-def _scripts_make(name):
-    """Makes an application, but uses the correct subtype if available."""
-    try:
-        __import__("wizard.app." + name)
-        return getattr(wizard.app, name).Application(name)
-    except ImportError as error:
-        # XXX ugly hack to check if the import error is from the top level
-        # module we care about or a submodule. should be an archetectural change.
-        if error.args[0].split()[-1]==name:
-            return Application(name)
-        else:
-            raise
+from wizard import plugin, resolve, shell, sql, util

 
 _applications = None
 def applications():
     """Hash table for looking up string application name to instance"""
     global _applications
     if not _applications:
 
 _applications = None
 def applications():
     """Hash table for looking up string application name to instance"""
     global _applications
     if not _applications:

-        # SCRIPTS SPECIFIC
-        _applications = dict([(n,_scripts_make(n)) for n in _scripts_application_list ])
-        # setup plugins
+        _applications = dict()

         for dist in pkg_resources.working_set:
         for dist in pkg_resources.working_set:

-            for appname, appclass in dist.get_entry_map("wizard.app").items():
+            for appname, entry in dist.get_entry_map("wizard.app").items():

                 if appname in _applications:
                     newname = dist.key + ":" + appname
                     if newname in _applications:
                         raise Exception("Unrecoverable application name conflict for %s from %s", appname, dist.key)
                     logging.warning("Could not overwrite %s, used %s instead", appname, newname)
                     appname = newname
                 if appname in _applications:
                     newname = dist.key + ":" + appname
                     if newname in _applications:
                         raise Exception("Unrecoverable application name conflict for %s from %s", appname, dist.key)
                     logging.warning("Could not overwrite %s, used %s instead", appname, newname)
                     appname = newname

+                appclass = entry.load()

                 _applications[appname] = appclass(appname)
                 _applications[appname] = appclass(appname)

+        # setup dummy apps
+        for entry in pkg_resources.iter_entry_points("wizard.dummy_apps"):
+            appfun = entry.load()
+            dummy_apps = appfun()
+            for appname in dummy_apps:
+                # a dummy app that already exists is not a fatal error
+                if appname in _applications:
+                    continue
+                _applications[appname] = Application(appname)

     return _applications
 
 def getApplication(appname):
     """Retrieves application instance given a name"""
     return _applications
 
 def getApplication(appname):
     """Retrieves application instance given a name"""

-    return applications()[appname]
+    try:
+        return applications()[appname]
+    except KeyError:
+        raise NoSuchApplication(appname)

 
 class Application(object):
     """
 
 class Application(object):
     """


@@ -104,6 +96,8 @@ class Application(object):
     """
     #: String name of the application
     name = None
     """
     #: String name of the application
     name = None

+    #: Human-readable name of the application
+    fullname = None

     #: Dictionary of version strings to :class:`ApplicationVersion`.
     #: See also :meth:`makeVersion`.
     versions = None
     #: Dictionary of version strings to :class:`ApplicationVersion`.
     #: See also :meth:`makeVersion`.
     versions = None


@@ -184,7 +178,7 @@ class Application(object):
     def dsn(self, deployment):
         """
         Returns the deployment specific database URL.  Uses the override file
     def dsn(self, deployment):
         """
         Returns the deployment specific database URL.  Uses the override file

-        in :file:`.scripts` if it exists, and otherwise attempt to extract the
+        in :file:`.wizard` if it exists, and otherwise attempt to extract the

         variables from the source files.
 
         Under some cases, the database URL will contain only the database
         variables from the source files.
 
         Under some cases, the database URL will contain only the database


@@ -230,7 +224,7 @@ class Application(object):
     def url(self, deployment):
         """
         Returns the deployment specific web URL.  Uses the override file
     def url(self, deployment):
         """
         Returns the deployment specific web URL.  Uses the override file

-        in :file:`.scripts` if it exists, and otherwise attempt to extract
+        in :file:`.wizard` if it exists, and otherwise attempt to extract

         the variables from the source files.
 
         This function might return ``None``, which indicates we couldn't figure
         the variables from the source files.
 
         This function might return ``None``, which indicates we couldn't figure


@@ -260,18 +254,27 @@ class Application(object):
         Takes a generic source checkout and parametrizes it according to the
         values of ``deployment``.  This function operates on the current
         working directory.  ``deployment`` should **not** be the same as the
         Takes a generic source checkout and parametrizes it according to the
         values of ``deployment``.  This function operates on the current
         working directory.  ``deployment`` should **not** be the same as the

-        current working directory.  Default implementation uses
-        :attr:`parametrized_files` and a simple search and replace on those
-        files.
+        current working directory.  See :meth:`parametrizeWithVars` for details
+        on the parametrization.

         """
         # deployment is not used in this implementation, but note that
         # we do have the invariant the current directory matches
         # deployment's directory
         variables = ref_deployment.extract()
         """
         # deployment is not used in this implementation, but note that
         # we do have the invariant the current directory matches
         # deployment's directory
         variables = ref_deployment.extract()

+        self.parametrizeWithVars(variables)
+    def parametrizeWithVars(self, variables):
+        """
+        Takes a generic source checkout and parametrizes it according to
+        the values of ``variables``.  Default implementation uses
+        :attr:`parametrized_files` and a simple search and replace on
+        those files.
+        """

         for file in self.parametrized_files:
         for file in self.parametrized_files:

+            logging.debug("Parametrizing file '%s'\n" % (file, ))

             try:
                 contents = open(file, "r").read()
             except IOError:
             try:
                 contents = open(file, "r").read()
             except IOError:

+                logging.debug("Failed to open file '%s'\n" % (file, ))

                 continue
             for key, value in variables.items():
                 if value is None: continue
                 continue
             for key, value in variables.items():
                 if value is None: continue


@@ -338,7 +341,7 @@ class Application(object):
         """
         for key, subst in self.substitutions.items():
             subs = subst(deployment)
         """
         for key, subst in self.substitutions.items():
             subs = subst(deployment)

-            if not subs and key not in self.deprecated_keys:
+            if not subs and key not in self.deprecated_keys and key not in self.random_keys:

                 logging.warning("No substitutions for %s" % key)
     def install(self, version, options):
         """
                 logging.warning("No substitutions for %s" % key)
     def install(self, version, options):
         """


@@ -402,6 +405,20 @@ class Application(object):
         match = regex.search(contents)
         if not match: return None
         return distutils.version.LooseVersion(shlex.split(match.group(2))[0])
         match = regex.search(contents)
         if not match: return None
         return distutils.version.LooseVersion(shlex.split(match.group(2))[0])

+    # XXX: This signature doesn't really make too much sense...
+    def detectVersionFromGit(self, tagPattern, preStrip = ''):
+        """
+        Helper method that detects a version by using the most recent tag
+        in git that matches the specified pattern.
+        This assumes that the current working directory is the deployment.
+        """
+        sh = wizard.shell.Shell()
+        cmd = ['git', 'describe', '--tags', '--match', tagPattern, ]
+        tag = sh.call(*cmd, strip=True)
+        if tag and len(tag) > len(preStrip) and tag[:len(preStrip)] == preStrip:
+            tag = tag[len(preStrip):]
+        if not tag: return None
+        return distutils.version.LooseVersion(tag)

     def download(self, version):
         """
         Returns a URL that can be used to download a tarball of ``version`` of
     def download(self, version):
         """
         Returns a URL that can be used to download a tarball of ``version`` of


@@ -455,8 +472,8 @@ class Application(object):
         Subclasses should provide an implementation.
         """
         # XXX: Unfortunately, this doesn't quite work because we package
         Subclasses should provide an implementation.
         """
         # XXX: Unfortunately, this doesn't quite work because we package

-        # bogus config files in the -scripts versions of installs.  Maybe
-        # we should check a hash or something?
+        # bogus config files.  Maybe we should check a hash or
+        # something?

         raise NotImplementedError
     def researchFilter(self, filename, added, deleted):
         """
         raise NotImplementedError
     def researchFilter(self, filename, added, deleted):
         """


@@ -495,10 +512,11 @@ class ApplicationVersion(object):
         """
         return "%s-%s" % (self.application, self.version)
     @property
         """
         return "%s-%s" % (self.application, self.version)
     @property

-    def scripts_tag(self):
+    def wizard_tag(self):

         """
         Returns the name of the Git tag for this version.
         """
         """
         Returns the name of the Git tag for this version.
         """

+        # XXX: Scripts specific

         end = str(self.version).partition('-scripts')[2].partition('-')[0]
         return "%s-scripts%s" % (self.pristine_tag, end)
     @property
         end = str(self.version).partition('-scripts')[2].partition('-')[0]
         return "%s-scripts%s" % (self.pristine_tag, end)
     @property


@@ -548,12 +566,9 @@ class ApplicationVersion(object):
         Makes/retrieves a singleton :class:`ApplicationVersion` from
         a``app`` and ``version`` string.
         """
         Makes/retrieves a singleton :class:`ApplicationVersion` from
         a``app`` and ``version`` string.
         """

-        try:
-            # defer to the application for version creation to enforce
-            # singletons
-            return applications()[app].makeVersion(version)
-        except KeyError:
-            raise NoSuchApplication(app)
+        # defer to the application for version creation to enforce
+        # singletons
+        return getApplication(app).makeVersion(version)

 
 def expand_re(val):
     """
 
 def expand_re(val):
     """


@@ -683,77 +698,24 @@ def filename_regex_substitution(key, files, regex):
         return subs
     return h
 
         return subs
     return h
 

-def backup_database(outdir, deployment):
-    """
-    Generic database backup function for MySQL.
-    """
-    # XXX: Change this once deployments support multiple dbs
-    if deployment.application.database == "mysql":
-        return backup_mysql_database(outdir, deployment)
-    else:
-        raise NotImplementedError
-
-def backup_mysql_database(outdir, deployment):
+@decorator.decorator
+def throws_database_errors(f, self, *args, **kwargs):

     """
     """

-    Database backups for MySQL using the :command:`mysqldump` utility.
+    Decorator that takes database errors from :mod:`wizard.sql` and
+    converts them into application script failures from
+    :mod:`wizard.app`.  We can't throw application errors directly from
+    :mod:`wizard.sql` because that would result in a cyclic import;
+    also, it's cleaner to distinguish between a database error and an
+    application script failure.

     """
     """

-    outfile = os.path.join(outdir, "db.sql")

     try:
     try:

-        shell.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment.dsn))
-        shell.call("gzip", "--best", outfile)
-    except shell.CallError as e:
-        raise BackupFailure(e.stderr)
-
-def restore_database(backup_dir, deployment):
-    """
-    Generic database restoration function for MySQL.
-    """
-    # XXX: see backup_database
-    if deployment.application.database == "mysql":
-        return restore_mysql_database(backup_dir, deployment)
-    else:
-        raise NotImplementedError
-
-def restore_mysql_database(backup_dir, deployment):
-    """
-    Database restoration for MySQL by piping SQL commands into :command:`mysql`.
-    """
-    if not os.path.exists(backup_dir):
-        raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
-    sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
-    shell.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
-    sql.seek(0)
-    shell.call("mysql", *get_mysql_args(deployment.dsn), stdin=sql)
-    sql.close()
-
-def remove_database(deployment):
-    """
-    Generic database removal function.  Actually, not so generic because we
-    go and check if we're on scripts and if we are run a different command.
-    """
-    if deployment.dsn.host == "sql.mit.edu":
-        try:
-            shell.call("/mit/scripts/sql/bin/drop-database", deployment.dsn.database)
-            return
-        except shell.CallError:
-            pass
-    engine = sqlalchemy.create_engine(deployment.dsn)
-    engine.execute("DROP DATABASE `%s`" % deployment.dsn.database)
-
-def get_mysql_args(dsn):
-    """
-    Extracts arguments that would be passed to the command line mysql utility
-    from a deployment.
-    """
-    args = []
-    if dsn.host:
-        args += ["-h", dsn.host]
-    if dsn.username:
-        args += ["-u", dsn.username]
-    if dsn.password:
-        args += ["-p" + dsn.password]
-    args += [dsn.database]
-    return args
+        return f(self, *args, **kwargs)
+    except sql.BackupDatabaseError:
+        raise BackupFailure(traceback.format_exc())
+    except sql.RestoreDatabaseError:
+        raise RestoreFailure(traceback.format_exc())
+    except sql.RemoveDatabaseError:
+        raise RemoveFailure(traceback.format_exc())

 
 class Error(wizard.Error):
     """Generic error class for this module."""
 
 class Error(wizard.Error):
     """Generic error class for this module."""


@@ -782,6 +744,8 @@ class DeploymentParseError(Error):
     location = None
     def __init__(self, value):
         self.value = value
     location = None
     def __init__(self, value):
         self.value = value

+    def __str__(self):
+        return "Could not parse '%s' from versions store in '%s'" % (self.value, self.location)

 
 class NoSuchApplication(Error):
     """
 
 class NoSuchApplication(Error):
     """


@@ -795,6 +759,8 @@ class NoSuchApplication(Error):
     location = None
     def __init__(self, app):
         self.app = app
     location = None
     def __init__(self, app):
         self.app = app

+    def __str__(self):
+        return "Wizard doesn't know about an application named '%s'." % self.app

 
 class Failure(Error):
     """
 
 class Failure(Error):
     """