Handle Wordpress random keys correctly on install and upgrade.

[wizard.git] / wizard / app / __init__.py

diff --git a/wizard/app/__init__.py b/wizard/app/__init__.py
index ab4d0080423538c7acdb71d932586c87bb97d3ba..a3664dba694ec340bd6334185875eb899e178cfa 100644 (file)
--- a/wizard/app/__init__.py
+++ b/wizard/app/__init__.py
@@ -6,6 +6,18 @@ You'll need to know how to overload the :class:`Application` class
 and use some of the functions in this module in order to specify
 new applications.
 
 and use some of the functions in this module in order to specify
 new applications.
 

+To specify custom applications as plugins,  add the following ``entry_points``
+configuration::
+
+    [wizard.app]
+    yourappname = your.module:Application
+    otherappname = your.other.module:Application
+
+.. note::
+
+    Wizard will complain loudly if ``yourappname`` conflicts with an
+    application name defined by someone else.
+

 There are some submodules for programming languages that define common
 functions and data that may be used by applications in that language.  See:
 
 There are some submodules for programming languages that define common
 functions and data that may be used by applications in that language.  See:
 


@@ -27,25 +39,58 @@ import decorator
 import shlex
 import logging
 import shutil
 import shlex
 import logging
 import shutil

+import sqlalchemy
+import sqlalchemy.exc
+import string
+import urlparse
+import tempfile
+import pkg_resources

 
 import wizard
 
 import wizard

-from wizard import resolve, scripts, shell, util
+from wizard import resolve, scripts, shell, sql, util

 
 

-_application_list = [
+# SCRIPTS SPECIFIC
+_scripts_application_list = [

     "mediawiki", "wordpress", "joomla", "e107", "gallery2",
     "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
     # these are technically deprecated
     "advancedpoll", "gallery",
 ]
     "mediawiki", "wordpress", "joomla", "e107", "gallery2",
     "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
     # these are technically deprecated
     "advancedpoll", "gallery",
 ]

-_applications = None
+def _scripts_make(name):
+    """Makes an application, but uses the correct subtype if available."""
+    try:
+        __import__("wizard.app." + name)
+        return getattr(wizard.app, name).Application(name)
+    except ImportError as error:
+        # XXX ugly hack to check if the import error is from the top level
+        # module we care about or a submodule. should be an archetectural change.
+        if error.args[0].split()[-1]==name:
+            return Application(name)
+        else:
+            raise

 
 

+_applications = None

 def applications():
     """Hash table for looking up string application name to instance"""
     global _applications
     if not _applications:
 def applications():
     """Hash table for looking up string application name to instance"""
     global _applications
     if not _applications:

-        _applications = dict([(n,Application.make(n)) for n in _application_list ])
+        # SCRIPTS SPECIFIC
+        _applications = dict([(n,_scripts_make(n)) for n in _scripts_application_list ])
+        # setup plugins
+        for dist in pkg_resources.working_set:
+            for appname, appclass in dist.get_entry_map("wizard.app").items():
+                if appname in _applications:
+                    newname = dist.key + ":" + appname
+                    if newname in _applications:
+                        raise Exception("Unrecoverable application name conflict for %s from %s", appname, dist.key)
+                    logging.warning("Could not overwrite %s, used %s instead", appname, newname)
+                    appname = newname
+                _applications[appname] = appclass(appname)

     return _applications
 
     return _applications
 

+def getApplication(appname):
+    """Retrieves application instance given a name"""
+    return applications()[appname]

 
 class Application(object):
     """
 
 class Application(object):
     """


@@ -66,7 +111,12 @@ class Application(object):
     parametrized_files = []
     #: Keys that are used in older versions of the application, but
     #: not for the most recent version.
     parametrized_files = []
     #: Keys that are used in older versions of the application, but
     #: not for the most recent version.

-    deprecated_keys = []
+    deprecated_keys = set()
+    #: Keys that we can simply generate random strings for if they're missing
+    random_keys = set()
+    #: Values that are not sufficiently random for a random key.  This can
+    #: include default values for a random configuration option,
+    random_blacklist = set()

     #: Dictionary of variable names to extractor functions.  These functions
     #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
     #: the variable, or ``None`` if it could not be found.
     #: Dictionary of variable names to extractor functions.  These functions
     #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
     #: the variable, or ``None`` if it could not be found.


@@ -84,6 +134,12 @@ class Application(object):
     #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
     #: this application requires.
     install_schema = None
     #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
     #: this application requires.
     install_schema = None

+    #: Name of the database that this application uses, i.e. ``mysql`` or
+    #: ``postgres``.  If we end up supporting multiple databases for a single
+    #: application, there should also be a value for this in
+    #: :class:`wizard.deploy.Deployment`; the value here is merely the preferred
+    #: value.
+    database = None

     def __init__(self, name):
         self.name = name
         self.versions = {}
     def __init__(self, name):
         self.name = name
         self.versions = {}


@@ -117,7 +173,85 @@ class Application(object):
         result = {}
         for k,extractor in self.extractors.items():
             result[k] = extractor(deployment)
         result = {}
         for k,extractor in self.extractors.items():
             result[k] = extractor(deployment)

+        # XXX: ugh... we have to do quoting
+        for k in self.random_keys:
+            if result[k] is None or result[k] in self.random_blacklist:
+                result[k] = "'%s'" % util.random_key()

         return result
         return result

+    def dsn(self, deployment):
+        """
+        Returns the deployment specific database URL.  Uses the override file
+        in :file:`.scripts` if it exists, and otherwise attempt to extract the
+        variables from the source files.
+
+        Under some cases, the database URL will contain only the database
+        property, and no other values.  This indicates that the actual DSN
+        should be determined from the environment.
+
+        This function might return ``None``.
+
+        .. note::
+
+            We are allowed to batch these two together, because the full precedence
+            chain for determining the database of an application combines these
+            two together.  If this was not the case, we would have to call
+            :meth:`databaseUrlFromOverride` and :meth:`databaseUrlFromExtract` manually.
+        """
+        url = self.dsnFromOverride(deployment)
+        if url:
+            return url
+        return self.dsnFromExtract(deployment)
+    def dsnFromOverride(self, deployment):
+        """
+        Extracts database URL from an explicit dsn override file.
+        """
+        try:
+            return sqlalchemy.engine.url.make_url(open(deployment.dsn_file).read().strip())
+        except IOError:
+            return None
+    def dsnFromExtract(self, deployment):
+        """
+        Extracts database URL from a deployment, and returns them as
+        a :class:`sqlalchemy.engine.url.URL`.  Returns ``None`` if we
+        can't figure it out: i.e. the conventional variables are not defined
+        for this application.
+        """
+        if not self.database:
+            return None
+        vars = self.extract(deployment)
+        names = ("WIZARD_DBSERVER", "WIZARD_DBUSER", "WIZARD_DBPASSWORD", "WIZARD_DBNAME")
+        host, user, password, database = (shlex.split(vars[x])[0] if vars[x] is not None else None for x in names)
+        # XXX: You'd have to put support for an explicit different database
+        # type here
+        return sqlalchemy.engine.url.URL(self.database, username=user, password=password, host=host, database=database)
+    def url(self, deployment):
+        """
+        Returns the deployment specific web URL.  Uses the override file
+        in :file:`.scripts` if it exists, and otherwise attempt to extract
+        the variables from the source files.
+
+        This function might return ``None``, which indicates we couldn't figure
+        it out.
+        """
+        url = self.urlFromOverride(deployment)
+        if url:
+            return url
+        return self.urlFromExtract(deployment)
+    def urlFromOverride(self, deployment):
+        """
+        Extracts URL from explicit url override file.
+        """
+        try:
+            return urlparse.urlparse(open(deployment.url_file).read().strip())
+        except IOError:
+            return None
+    def urlFromExtract(self, deployment):
+        """
+        Extracts URL from a deployment, and returns ``None`` if we can't
+        figure it out.  Default implementation is to fail; we might
+        do something clever with extractable variables in the future.
+        """
+        return None

     def parametrize(self, deployment, ref_deployment):
         """
         Takes a generic source checkout and parametrizes it according to the
     def parametrize(self, deployment, ref_deployment):
         """
         Takes a generic source checkout and parametrizes it according to the


@@ -127,6 +261,9 @@ class Application(object):
         :attr:`parametrized_files` and a simple search and replace on those
         files.
         """
         :attr:`parametrized_files` and a simple search and replace on those
         files.
         """

+        # deployment is not used in this implementation, but note that
+        # we do have the invariant the current directory matches
+        # deployment's directory

         variables = ref_deployment.extract()
         for file in self.parametrized_files:
             try:
         variables = ref_deployment.extract()
         for file in self.parametrized_files:
             try:


@@ -146,9 +283,24 @@ class Application(object):
         default implementation uses :attr:`resolutions`.
         """
         resolved = True
         default implementation uses :attr:`resolutions`.
         """
         resolved = True

-        sh = shell.Shell()
-        for status in sh.eval("git", "ls-files", "--unmerged").splitlines():
-            file = status.split()[-1]
+        files = set()
+        files = {}
+        for status in shell.eval("git", "ls-files", "--unmerged").splitlines():
+            mode, hash, role, name = status.split()
+            files.setdefault(name, set()).add(int(role))
+        for file, roles in files.items():
+            # some automatic resolutions
+            if 1 not in roles and 2 not in roles and 3 in roles:
+                # upstream added a file, but it conflicted for whatever reason
+                shell.call("git", "add", file)
+                continue
+            elif 1 in roles and 2 not in roles and 3 in roles:
+                # user deleted the file, but upstream changed it
+                shell.call("git", "rm", file)
+                continue
+            # manual resolutions
+            # XXX: this functionality is mostly subsumed by the rerere
+            # tricks we do

             if file in self.resolutions:
                 contents = open(file, "r").read()
                 for spec, result in self.resolutions[file]:
             if file in self.resolutions:
                 contents = open(file, "r").read()
                 for spec, result in self.resolutions[file]:


@@ -158,7 +310,7 @@ class Application(object):
                         logging.info("Did resolution with spec:\n" + spec)
                 open(file, "w").write(contents)
                 if not resolve.is_conflict(contents):
                         logging.info("Did resolution with spec:\n" + spec)
                 open(file, "w").write(contents)
                 if not resolve.is_conflict(contents):

-                    sh.call("git", "add", file)
+                    shell.call("git", "add", file)

                 else:
                     resolved = False
             else:
                 else:
                     resolved = False
             else:


@@ -223,6 +375,13 @@ class Application(object):
         should provide an implementation.
         """
         raise NotImplementedError
         should provide an implementation.
         """
         raise NotImplementedError

+    def remove(self, deployment, options):
+        """
+        Run for 'wizard remove' to delete all database and non-local
+        file data.  This assumes that the current working directory is
+        the deployment.  Subclasses should provide an implementation.
+        """
+        raise NotImplementedError

     def detectVersion(self, deployment):
         """
         Checks source files to determine the version manually.  This assumes
     def detectVersion(self, deployment):
         """
         Checks source files to determine the version manually.  This assumes


@@ -246,13 +405,10 @@ class Application(object):
         this application.
         """
         raise NotImplementedError
         this application.
         """
         raise NotImplementedError

-    def checkWeb(self, deployment, output=None):
+    def checkWeb(self, deployment):

         """
         """

-        Checks if the autoinstall is viewable from the web.  To get
-        the HTML source that was retrieved, pass a variable containing
-        an empty list to ``output``; it will be mutated to have its
-        first element be the output.  Subclasses should provide an
-        implementation.
+        Checks if the autoinstall is viewable from the web.  Subclasses should
+        provide an implementation.

 
         .. note::
             Finding a reasonable heuristic that works across skinning
 
         .. note::
             Finding a reasonable heuristic that works across skinning


@@ -262,6 +418,33 @@ class Application(object):
             not to depend on pages that are not the main page.
         """
         raise NotImplementedError
             not to depend on pages that are not the main page.
         """
         raise NotImplementedError

+    def checkDatabase(self, deployment):
+        """
+        Checks if the database is accessible.
+        """
+        try:
+            sql.connect(deployment.dsn)
+            return True
+        except sqlalchemy.exc.DBAPIError:
+            return False
+    def checkWebPage(self, deployment, page, outputs=[], exclude=[]):
+        """
+        Checks if a given page of an autoinstall contains a particular string.
+        """
+        page = deployment.fetch(page)
+        for x in exclude:
+            if page.find(x) != -1:
+                logging.info("checkWebPage (failed due to %s):\n\n%s", x, page)
+                return False
+        votes = 0
+        for output in outputs:
+            votes += page.find(output) != -1
+        if votes > len(outputs) / 2:
+            logging.debug("checkWebPage (passed):\n\n" + page)
+            return True
+        else:
+            logging.info("checkWebPage (failed):\n\n" + page)
+            return False

     def checkConfig(self, deployment):
         """
         Checks whether or not an autoinstall has been configured/installed
     def checkConfig(self, deployment):
         """
         Checks whether or not an autoinstall has been configured/installed


@@ -272,14 +455,23 @@ class Application(object):
         # bogus config files in the -scripts versions of installs.  Maybe
         # we should check a hash or something?
         raise NotImplementedError
         # bogus config files in the -scripts versions of installs.  Maybe
         # we should check a hash or something?
         raise NotImplementedError

-    @staticmethod
-    def make(name):
-        """Makes an application, but uses the correct subtype if available."""
-        try:
-            __import__("wizard.app." + name)
-            return getattr(wizard.app, name).Application(name)
-        except ImportError:
-            return Application(name)
+    def researchFilter(self, filename, added, deleted):
+        """
+        Allows an application to selectively ignore certain diffstat signatures
+        during research; for example, configuration files will have a very
+        specific set of changes, so ignore them; certain installation files
+        may be removed, etc.  Return ``True`` if a diffstat signature should be
+        ignored,
+        """
+        return False
+    def researchVerbose(self, filename):
+        """
+        Allows an application to exclude certain dirty files from the output
+        report; usually this will just be parametrized files, since those are
+        guaranteed to have changes.  Return ``True`` if a file should only
+        be displayed in verbose mode.
+        """
+        return filename in self.parametrized_files

 
 class ApplicationVersion(object):
     """Represents an abstract notion of a version for an application, where
 
 class ApplicationVersion(object):
     """Represents an abstract notion of a version for an application, where


@@ -488,52 +680,76 @@ def filename_regex_substitution(key, files, regex):
         return subs
     return h
 
         return subs
     return h
 

-# XXX: rename to show that it's mysql specific

 def backup_database(outdir, deployment):
     """
 def backup_database(outdir, deployment):
     """

-    Generic database backup function for MySQL.  Assumes that ``WIZARD_DBNAME``
-    is extractable, and that :func:`wizard.scripts.get_sql_credentials`
-    works.
+    Generic database backup function for MySQL.
+    """
+    # XXX: Change this once deployments support multiple dbs
+    if deployment.application.database == "mysql":
+        return backup_mysql_database(outdir, deployment)
+    else:
+        raise NotImplementedError
+
+def backup_mysql_database(outdir, deployment):
+    """
+    Database backups for MySQL using the :command:`mysqldump` utility.

     """
     """

-    sh = shell.Shell()

     outfile = os.path.join(outdir, "db.sql")
     try:
     outfile = os.path.join(outdir, "db.sql")
     try:

-        sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment))
-        sh.call("gzip", "--best", outfile)
+        shell.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment.dsn))
+        shell.call("gzip", "--best", outfile)

     except shell.CallError as e:
     except shell.CallError as e:

-        shutil.rmtree(outdir)

         raise BackupFailure(e.stderr)
 
 def restore_database(backup_dir, deployment):
     """
         raise BackupFailure(e.stderr)
 
 def restore_database(backup_dir, deployment):
     """

-    Generic database restoration function for MySQL.  See :func:`backup_database`
-    for the assumptions that we make.
+    Generic database restoration function for MySQL.
+    """
+    # XXX: see backup_database
+    if deployment.application.database == "mysql":
+        return restore_mysql_database(backup_dir, deployment)
+    else:
+        raise NotImplementedError
+
+def restore_mysql_database(backup_dir, deployment):
+    """
+    Database restoration for MySQL by piping SQL commands into :command:`mysql`.

     """
     """

-    sh = shell.Shell()

     if not os.path.exists(backup_dir):
         raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
     sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
     if not os.path.exists(backup_dir):
         raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
     sql = open(os.path.join(backup_dir, "db.sql"), 'w+')

-    sh.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
+    shell.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)

     sql.seek(0)
     sql.seek(0)

-    sh.call("mysql", *get_mysql_args(deployment), stdin=sql)
+    shell.call("mysql", *get_mysql_args(deployment.dsn), stdin=sql)

     sql.close()
 
     sql.close()
 

-def get_mysql_args(d):
+def remove_database(deployment):
+    """
+    Generic database removal function.  Actually, not so generic because we
+    go and check if we're on scripts and if we are run a different command.
+    """
+    if deployment.dsn.host == "sql.mit.edu":
+        try:
+            shell.call("/mit/scripts/sql/bin/drop-database", deployment.dsn.database)
+            return
+        except shell.CallError:
+            pass
+    engine = sqlalchemy.create_engine(deployment.dsn)
+    engine.execute("DROP DATABASE `%s`" % deployment.dsn.database)
+
+def get_mysql_args(dsn):

     """
     Extracts arguments that would be passed to the command line mysql utility
     from a deployment.
     """
     """
     Extracts arguments that would be passed to the command line mysql utility
     from a deployment.
     """

-    # XXX: add support for getting these out of options
-    vars = d.extract()
-    if 'WIZARD_DBNAME' not in vars:
-        raise BackupFailure("Could not determine database name")
-    triplet = scripts.get_sql_credentials(vars)

     args = []
     args = []

-    if triplet is not None:
-        server, user, password = triplet
-        args += ["-h", server, "-u", user, "-p" + password]
-    name = shlex.split(vars['WIZARD_DBNAME'])[0]
-    args.append(name)
+    if dsn.host:
+        args += ["-h", dsn.host]
+    if dsn.username:
+        args += ["-u", dsn.username]
+    if dsn.password:
+        args += ["-p" + dsn.password]
+    args += [dsn.database]

     return args
 
 class Error(wizard.Error):
     return args
 
 class Error(wizard.Error):


@@ -587,7 +803,12 @@ class Failure(Error):
 
 class InstallFailure(Error):
     """Installation failed for unknown reason."""
 
 class InstallFailure(Error):
     """Installation failed for unknown reason."""

-    pass
+    def __str__(self):
+        return """
+
+ERROR: Installation failed for unknown reason.  You can
+retry the installation by appending --retry to the installation
+command."""

 
 class RecoverableInstallFailure(InstallFailure):
     """
 
 class RecoverableInstallFailure(InstallFailure):
     """


@@ -600,7 +821,12 @@ class RecoverableInstallFailure(InstallFailure):
     def __init__(self, errors):
         self.errors = errors
     def __str__(self):
     def __init__(self, errors):
         self.errors = errors
     def __str__(self):

-        return """Installation failed due to the following errors: %s""" % ", ".join(self.errors)
+        return """
+
+ERROR: Installation failed due to the following errors:  %s
+
+You can retry the installation by appending --retry to the
+installation command.""" % ", ".join(self.errors)

 
 class UpgradeFailure(Failure):
     """Upgrade script failed."""
 
 class UpgradeFailure(Failure):
     """Upgrade script failed."""


@@ -617,16 +843,11 @@ ERROR: Upgrade script failed, details:
 
 class UpgradeVerificationFailure(Failure):
     """Upgrade script passed, but website wasn't accessible afterwards"""
 
 class UpgradeVerificationFailure(Failure):
     """Upgrade script passed, but website wasn't accessible afterwards"""

-    #: String details of failure (possibly stdout or stderr output)
-    details = None
-    def __init__(self, details):
-        self.details = details

     def __str__(self):
         return """
 
     def __str__(self):
         return """
 

-ERROR: Upgrade script passed, but website wasn't accessible afterwards. Details:
-
-%s""" % self.details
+ERROR: Upgrade script passed, but website wasn't accessible afterwards.  Check
+the debug logs for the contents of the page."""

 
 class BackupFailure(Failure):
     """Backup script failed."""
 
 class BackupFailure(Failure):
     """Backup script failed."""


@@ -653,3 +874,16 @@ class RestoreFailure(Failure):
 ERROR: Restore script failed, details:
 
 %s""" % self.details
 ERROR: Restore script failed, details:
 
 %s""" % self.details

+
+class RemoveFailure(Failure):
+    """Remove script failed."""
+    #: String details of failure
+    details = None
+    def __init__(self, details):
+        self.details = details
+    def __str__(self):
+        return """
+
+ERROR: Remove script failed, details:
+
+%s""" % self.details