and use some of the functions in this module in order to specify
new applications.
+To specify custom applications as plugins, add the following ``entry_points``
+configuration::
+
+ [wizard.app]
+ yourappname = your.module:Application
+ otherappname = your.other.module:Application
+
+.. note::
+
+ Wizard will complain loudly if ``yourappname`` conflicts with an
+ application name defined by someone else.
+
+There are some submodules for programming languages that define common
+functions and data that may be used by applications in that language. See:
+
+* :mod:`wizard.app.php`
+
.. testsetup:: *
import re
"""
import os.path
+import subprocess
import re
import distutils.version
import decorator
+import shlex
+import logging
+import shutil
+import sqlalchemy
+import sqlalchemy.exc
+import string
+import urlparse
+import tempfile
+import pkg_resources
+import traceback
import wizard
-from wizard import util
-
-_application_list = [
- "mediawiki", "wordpress", "joomla", "e107", "gallery2",
- "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
- # these are technically deprecated
- "advancedpoll", "gallery",
-]
-_applications = None
+from wizard import plugin, resolve, shell, sql, util
+_applications = None
def applications():
"""Hash table for looking up string application name to instance"""
global _applications
if not _applications:
- _applications = dict([(n,Application.make(n)) for n in _application_list ])
+ _applications = dict()
+ for dist in pkg_resources.working_set:
+ for appname, entry in dist.get_entry_map("wizard.app").items():
+ if appname in _applications:
+ newname = dist.key + ":" + appname
+ if newname in _applications:
+ raise Exception("Unrecoverable application name conflict for %s from %s", appname, dist.key)
+ logging.warning("Could not overwrite %s, used %s instead", appname, newname)
+ appname = newname
+ appclass = entry.load()
+ _applications[appname] = appclass(appname)
+ # setup dummy apps
+ for entry in pkg_resources.iter_entry_points("wizard.dummy_apps"):
+ appfun = entry.load()
+ dummy_apps = appfun()
+ for appname in dummy_apps:
+ # a dummy app that already exists is not a fatal error
+ if appname in _applications:
+ continue
+ _applications[appname] = Application(appname)
return _applications
+def getApplication(appname):
+ """Retrieves application instance given a name"""
+ try:
+ return applications()[appname]
+ except KeyError:
+ raise NoSuchApplication(appname)
class Application(object):
"""
"""
#: String name of the application
name = None
+ #: Human-readable name of the application
+ fullname = None
#: Dictionary of version strings to :class:`ApplicationVersion`.
#: See also :meth:`makeVersion`.
versions = None
parametrized_files = []
#: Keys that are used in older versions of the application, but
#: not for the most recent version.
- deprecated_keys = []
+ deprecated_keys = set()
+ #: Keys that we can simply generate random strings for if they're missing
+ random_keys = set()
+ #: Values that are not sufficiently random for a random key. This can
+ #: include default values for a random configuration option,
+ random_blacklist = set()
+ #: Dictionary of variable names to extractor functions. These functions
+ #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
+ #: the variable, or ``None`` if it could not be found.
+ #: See also :func:`filename_regex_extractor`.
+ extractors = {}
+ #: Dictionary of variable names to substitution functions. These functions
+ #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
+ #: that an explicit instance of the variable is released with the generic
+ #: ``WIZARD_*`` constant. See also :func:`filename_regex_substitution`.
+ substitutions = {}
+ #: Dictionary of file names to a list of resolutions, which are tuples of
+ #: a conflict marker string and a result list. See :mod:`wizard.resolve`
+ #: for more information.
+ resolutions = {}
+ #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
+ #: this application requires.
+ install_schema = None
+ #: Name of the database that this application uses, i.e. ``mysql`` or
+ #: ``postgres``. If we end up supporting multiple databases for a single
+ #: application, there should also be a value for this in
+ #: :class:`wizard.deploy.Deployment`; the value here is merely the preferred
+ #: value.
+ database = None
+ #: Indicates whether or not a web stub is necessary.
+ needs_web_stub = False
def __init__(self, name):
self.name = name
self.versions = {}
result = {}
for k,extractor in self.extractors.items():
result[k] = extractor(deployment)
+ # XXX: ugh... we have to do quoting
+ for k in self.random_keys:
+ if result[k] is None or result[k] in self.random_blacklist:
+ result[k] = "'%s'" % util.random_key()
return result
- def parametrize(self, deployment):
+ def dsn(self, deployment):
+ """
+ Returns the deployment specific database URL. Uses the override file
+ in :file:`.wizard` if it exists, and otherwise attempt to extract the
+ variables from the source files.
+
+ Under some cases, the database URL will contain only the database
+ property, and no other values. This indicates that the actual DSN
+ should be determined from the environment.
+
+ This function might return ``None``.
+
+ .. note::
+
+ We are allowed to batch these two together, because the full precedence
+ chain for determining the database of an application combines these
+ two together. If this was not the case, we would have to call
+ :meth:`databaseUrlFromOverride` and :meth:`databaseUrlFromExtract` manually.
+ """
+ url = self.dsnFromOverride(deployment)
+ if url:
+ return url
+ return self.dsnFromExtract(deployment)
+ def dsnFromOverride(self, deployment):
+ """
+ Extracts database URL from an explicit dsn override file.
+ """
+ try:
+ return sqlalchemy.engine.url.make_url(open(deployment.dsn_file).read().strip())
+ except IOError:
+ return None
+ def dsnFromExtract(self, deployment):
+ """
+ Extracts database URL from a deployment, and returns them as
+ a :class:`sqlalchemy.engine.url.URL`. Returns ``None`` if we
+ can't figure it out: i.e. the conventional variables are not defined
+ for this application.
+ """
+ if not self.database:
+ return None
+ vars = self.extract(deployment)
+ names = ("WIZARD_DBSERVER", "WIZARD_DBUSER", "WIZARD_DBPASSWORD", "WIZARD_DBNAME")
+ host, user, password, database = (shlex.split(vars[x])[0] if vars[x] is not None else None for x in names)
+ # XXX: You'd have to put support for an explicit different database
+ # type here
+ return sqlalchemy.engine.url.URL(self.database, username=user, password=password, host=host, database=database)
+ def url(self, deployment):
+ """
+ Returns the deployment specific web URL. Uses the override file
+ in :file:`.wizard` if it exists, and otherwise attempt to extract
+ the variables from the source files.
+
+ This function might return ``None``, which indicates we couldn't figure
+ it out.
+ """
+ url = self.urlFromOverride(deployment)
+ if url:
+ return url
+ return self.urlFromExtract(deployment)
+ def urlFromOverride(self, deployment):
+ """
+ Extracts URL from explicit url override file.
+ """
+ try:
+ return urlparse.urlparse(open(deployment.url_file).read().strip())
+ except IOError:
+ return None
+ def urlFromExtract(self, deployment):
+ """
+ Extracts URL from a deployment, and returns ``None`` if we can't
+ figure it out. Default implementation is to fail; we might
+ do something clever with extractable variables in the future.
+ """
+ return None
+ def parametrize(self, deployment, ref_deployment):
"""
Takes a generic source checkout and parametrizes it according to the
values of ``deployment``. This function operates on the current
working directory. ``deployment`` should **not** be the same as the
- current working directory. Default implementation uses
- :attr:`parametrized_files` and a simple search and replace on those
- files.
+ current working directory. See :meth:`parametrizeWithVars` for details
+ on the parametrization.
+ """
+ # deployment is not used in this implementation, but note that
+ # we do have the invariant the current directory matches
+ # deployment's directory
+ variables = ref_deployment.extract()
+ self.parametrizeWithVars(variables)
+ def parametrizeWithVars(self, variables):
+ """
+ Takes a generic source checkout and parametrizes it according to
+ the values of ``variables``. Default implementation uses
+ :attr:`parametrized_files` and a simple search and replace on
+ those files.
"""
- variables = deployment.extract()
for file in self.parametrized_files:
+ logging.debug("Parametrizing file '%s'\n" % (file, ))
try:
contents = open(file, "r").read()
except IOError:
+ logging.debug("Failed to open file '%s'\n" % (file, ))
continue
for key, value in variables.items():
if value is None: continue
"""
Resolves conflicted files in the current working directory. Returns
whether or not all conflicted files were resolved or not. Fully
- resolved files are added to the index, but no commit is made. By
- default this is a no-op and returns ``False``; subclasses should
- replace this with useful behavior.
- """
- return False
+ resolved files are added to the index, but no commit is made. The
+ default implementation uses :attr:`resolutions`.
+ """
+ resolved = True
+ files = set()
+ files = {}
+ for status in shell.eval("git", "ls-files", "--unmerged").splitlines():
+ mode, hash, role, name = status.split()
+ files.setdefault(name, set()).add(int(role))
+ for file, roles in files.items():
+ # some automatic resolutions
+ if 1 not in roles and 2 not in roles and 3 in roles:
+ # upstream added a file, but it conflicted for whatever reason
+ shell.call("git", "add", file)
+ continue
+ elif 1 in roles and 2 not in roles and 3 in roles:
+ # user deleted the file, but upstream changed it
+ shell.call("git", "rm", file)
+ continue
+ # manual resolutions
+ # XXX: this functionality is mostly subsumed by the rerere
+ # tricks we do
+ if file in self.resolutions:
+ contents = open(file, "r").read()
+ for spec, result in self.resolutions[file]:
+ old_contents = contents
+ contents = resolve.resolve(contents, spec, result)
+ if old_contents != contents:
+ logging.info("Did resolution with spec:\n" + spec)
+ open(file, "w").write(contents)
+ if not resolve.is_conflict(contents):
+ shell.call("git", "add", file)
+ else:
+ resolved = False
+ else:
+ resolved = False
+ return resolved
def prepareMerge(self, deployment):
"""
Performs various edits to files in the current working directory in
"""
for key, subst in self.substitutions.items():
subs = subst(deployment)
- if not subs and key not in self.deprecated_keys:
+ if not subs and key not in self.deprecated_keys and key not in self.random_keys:
logging.warning("No substitutions for %s" % key)
def install(self, version, options):
"""
take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should
provide an implementation.
"""
- raise NotImplemented
+ raise NotImplementedError
def upgrade(self, deployment, version, options):
"""
Run for 'wizard upgrade' to upgrade database schemas and other
upgraded. This assumes that the current working directory is the
deployment. Subclasses should provide an implementation.
"""
- raise NotImplemented
- def backup(self, deployment, options):
+ raise NotImplementedError
+ def backup(self, deployment, outdir, options):
"""
Run for 'wizard backup' and upgrades to backup database schemas
- and other non-versioned data in an application. This assumes
+ and other non-versioned data in an application. ``outdir`` is
+ the directory that backup files should be placed. This assumes
that the current working directory is the deployment. Subclasses
- should provide an implementation.
+ should provide an implementation, even if it is a no-op.
.. note::
Static user files may not need to be backed up, since in
many applications upgrades do not modify static files.
"""
- raise NotImplemented
- def restore(self, deployment, backup, options):
+ raise NotImplementedError
+ def restore(self, deployment, backup_dir, options):
"""
Run for 'wizard restore' and failed upgrades to restore database
and other non-versioned data to a backed up version. This assumes
that the current working directory is the deployment. Subclasses
should provide an implementation.
"""
- raise NotImplemented
+ raise NotImplementedError
+ def remove(self, deployment, options):
+ """
+ Run for 'wizard remove' to delete all database and non-local
+ file data. This assumes that the current working directory is
+ the deployment. Subclasses should provide an implementation.
+ """
+ raise NotImplementedError
def detectVersion(self, deployment):
"""
Checks source files to determine the version manually. This assumes
that the current working directory is the deployment. Subclasses
should provide an implementation.
"""
- raise NotImplemented
- def checkWeb(self, deployment, output=None):
+ raise NotImplementedError
+ def detectVersionFromFile(self, filename, regex):
+ """
+ Helper method that detects a version by using a regular expression
+ from a file. The regexed value is passed through :mod:`shlex`.
+ This assumes that the current working directory is the deployment.
"""
- Checks if the autoinstall is viewable from the web. To get
- the HTML source that was retrieved, pass a variable containing
- an empty list to ``output``; it will be mutated to have its
- first element be the output. Subclasses should provide an
- implementation.
+ contents = open(filename).read()
+ match = regex.search(contents)
+ if not match: return None
+ return distutils.version.LooseVersion(shlex.split(match.group(2))[0])
+ # XXX: This signature doesn't really make too much sense...
+ def detectVersionFromGit(self, tagPattern, preStrip = ''):
+ """
+ Helper method that detects a version by using the most recent tag
+ in git that matches the specified pattern.
+ This assumes that the current working directory is the deployment.
+ """
+ sh = wizard.shell.Shell()
+ cmd = ['git', 'describe', '--tags', '--match', tagPattern, ]
+ tag = sh.call(*cmd, strip=True)
+ if tag and len(tag) > len(preStrip) and tag[:len(preStrip)] == preStrip:
+ tag = tag[len(preStrip):]
+ if not tag: return None
+ return distutils.version.LooseVersion(tag)
+ def download(self, version):
+ """
+ Returns a URL that can be used to download a tarball of ``version`` of
+ this application.
+ """
+ raise NotImplementedError
+ def checkWeb(self, deployment):
+ """
+ Checks if the autoinstall is viewable from the web. Subclasses should
+ provide an implementation.
.. note::
Finding a reasonable heuristic that works across skinning
page does not contain the features you search for. Try
not to depend on pages that are not the main page.
"""
- raise NotImplemented
+ raise NotImplementedError
+ def checkDatabase(self, deployment):
+ """
+ Checks if the database is accessible.
+ """
+ try:
+ sql.connect(deployment.dsn)
+ return True
+ except sqlalchemy.exc.DBAPIError:
+ return False
+ def checkWebPage(self, deployment, page, outputs=[], exclude=[]):
+ """
+ Checks if a given page of an autoinstall contains a particular string.
+ """
+ page = deployment.fetch(page)
+ for x in exclude:
+ if page.find(x) != -1:
+ logging.info("checkWebPage (failed due to %s):\n\n%s", x, page)
+ return False
+ votes = 0
+ for output in outputs:
+ votes += page.find(output) != -1
+ if votes > len(outputs) / 2:
+ logging.debug("checkWebPage (passed):\n\n" + page)
+ return True
+ else:
+ logging.info("checkWebPage (failed):\n\n" + page)
+ return False
def checkConfig(self, deployment):
"""
Checks whether or not an autoinstall has been configured/installed
for use. Assumes that the current working directory is the deployment.
Subclasses should provide an implementation.
"""
- raise NotImplemented
- @property
- def extractors(self):
+ # XXX: Unfortunately, this doesn't quite work because we package
+ # bogus config files. Maybe we should check a hash or
+ # something?
+ raise NotImplementedError
+ def researchFilter(self, filename, added, deleted):
"""
- Dictionary of variable names to extractor functions. These functions
- take a :class:`wizard.deploy.Deployment` as an argument and return the value of
- the variable, or ``None`` if it could not be found.
- See also :func:`filename_regex_extractor`.
+ Allows an application to selectively ignore certain diffstat signatures
+ during research; for example, configuration files will have a very
+ specific set of changes, so ignore them; certain installation files
+ may be removed, etc. Return ``True`` if a diffstat signature should be
+ ignored,
"""
- return {}
- @property
- def substitutions(self):
+ return False
+ def researchVerbose(self, filename):
"""
- Dictionary of variable names to substitution functions. These functions
- take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
- that an explicit instance of the variable is released with the generic
- WIZARD_* constant. See also :func:`filename_regex_substitution`.
+ Allows an application to exclude certain dirty files from the output
+ report; usually this will just be parametrized files, since those are
+ guaranteed to have changes. Return ``True`` if a file should only
+ be displayed in verbose mode.
"""
- return {}
- @staticmethod
- def make(name):
- """Makes an application, but uses the correct subtype if available."""
- try:
- __import__("wizard.app." + name)
- return getattr(wizard.app, name).Application(name)
- except ImportError:
- return Application(name)
+ return filename in self.parametrized_files
class ApplicationVersion(object):
"""Represents an abstract notion of a version for an application, where
"""
return "%s-%s" % (self.application, self.version)
@property
- def scripts_tag(self):
+ def wizard_tag(self):
"""
Returns the name of the Git tag for this version.
"""
+ # XXX: Scripts specific
end = str(self.version).partition('-scripts')[2].partition('-')[0]
return "%s-scripts%s" % (self.pristine_tag, end)
@property
Makes/retrieves a singleton :class:`ApplicationVersion` from
a``app`` and ``version`` string.
"""
- try:
- # defer to the application for version creation to enforce
- # singletons
- return applications()[app].makeVersion(version)
- except KeyError:
- raise NoSuchApplication(app)
+ # defer to the application for version creation to enforce
+ # singletons
+ return getApplication(app).makeVersion(version)
def expand_re(val):
"""
def make_extractors(seed):
"""
- Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into
+ Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into
extractor functions (which take a :class:`wizard.deploy.Deployment`
and return the value of the second subpattern of ``regex`` when matched
with the contents of ``file``).
def make_substitutions(seed):
"""
- Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into substitution
+ Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into substitution
functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
"""
return subs
return h
+@decorator.decorator
+def throws_database_errors(f, self, *args, **kwargs):
+ """
+ Decorator that takes database errors from :mod:`wizard.sql` and
+ converts them into application script failures from
+ :mod:`wizard.app`. We can't throw application errors directly from
+ :mod:`wizard.sql` because that would result in a cyclic import;
+ also, it's cleaner to distinguish between a database error and an
+ application script failure.
+ """
+ try:
+ return f(self, *args, **kwargs)
+ except sql.BackupDatabaseError:
+ raise BackupFailure(traceback.format_exc())
+ except sql.RestoreDatabaseError:
+ raise RestoreFailure(traceback.format_exc())
+ except sql.RemoveDatabaseError:
+ raise RemoveFailure(traceback.format_exc())
+
class Error(wizard.Error):
"""Generic error class for this module."""
pass
location = None
def __init__(self, value):
self.value = value
+ def __str__(self):
+ return "Could not parse '%s' from versions store in '%s'" % (self.value, self.location)
class NoSuchApplication(Error):
"""
location = None
def __init__(self, app):
self.app = app
+ def __str__(self):
+ return "Wizard doesn't know about an application named '%s'." % self.app
+
+class Failure(Error):
+ """
+ Represents a failure when performing some double-dispatched operation
+ such as an installation or an upgrade. Failure classes are postfixed
+ with Failure, not Error.
+ """
+ pass
+
+class InstallFailure(Error):
+ """Installation failed for unknown reason."""
+ def __str__(self):
+ return """
+
+ERROR: Installation failed for unknown reason. You can
+retry the installation by appending --retry to the installation
+command."""
+
+class RecoverableInstallFailure(InstallFailure):
+ """
+ Installation failed, but we were able to determine what the
+ error was, and should give the user a second chance if we were
+ running interactively.
+ """
+ #: List of the errors that were found.
+ errors = None
+ def __init__(self, errors):
+ self.errors = errors
+ def __str__(self):
+ return """
+
+ERROR: Installation failed due to the following errors: %s
+
+You can retry the installation by appending --retry to the
+installation command.""" % ", ".join(self.errors)
-class UpgradeFailure(Error):
+class UpgradeFailure(Failure):
"""Upgrade script failed."""
#: String details of failure (possibly stdout or stderr output)
details = None
%s""" % self.details
-class UpgradeVerificationFailure(Error):
+class UpgradeVerificationFailure(Failure):
"""Upgrade script passed, but website wasn't accessible afterwards"""
- #: String details of failure (possibly stdout or stderr output)
+ def __str__(self):
+ return """
+
+ERROR: Upgrade script passed, but website wasn't accessible afterwards. Check
+the debug logs for the contents of the page."""
+
+class BackupFailure(Failure):
+ """Backup script failed."""
+ #: String details of failure
details = None
def __init__(self, details):
self.details = details
def __str__(self):
return """
-ERROR: Upgrade script passed, but website wasn't accessible afterwards. Details:
+ERROR: Backup script failed, details:
%s""" % self.details
-class BackupFailure(Error):
- """Backup script failed."""
+class RestoreFailure(Failure):
+ """Restore script failed."""
#: String details of failure
details = None
def __init__(self, details):
def __str__(self):
return """
-ERROR: Backup script failed, details:
+ERROR: Restore script failed, details:
%s""" % self.details
-class RestoreFailure(Error):
- """Restore script failed."""
+class RemoveFailure(Failure):
+ """Remove script failed."""
#: String details of failure
details = None
def __init__(self, details):
def __str__(self):
return """
-ERROR: Restore script failed, details:
+ERROR: Remove script failed, details:
%s""" % self.details