and use some of the functions in this module in order to specify
new applications.
+To specify custom applications as plugins, add the following ``entry_points``
+configuration::
+
+ [wizard.app]
+ yourappname = your.module:Application
+ otherappname = your.other.module:Application
+
+.. note::
+
+ Wizard will complain loudly if ``yourappname`` conflicts with an
+ application name defined by someone else.
+
There are some submodules for programming languages that define common
functions and data that may be used by applications in that language. See:
"""
import os.path
+import subprocess
import re
import distutils.version
import decorator
import logging
import shutil
import sqlalchemy
-import random
+import sqlalchemy.exc
import string
import urlparse
import tempfile
+import pkg_resources
+import traceback
import wizard
-from wizard import resolve, scripts, shell, util
-
-_application_list = [
- "mediawiki", "wordpress", "joomla", "e107", "gallery2",
- "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
- # these are technically deprecated
- "advancedpoll", "gallery",
-]
-_applications = None
+from wizard import plugin, resolve, shell, sql, util
+_applications = None
def applications():
"""Hash table for looking up string application name to instance"""
global _applications
if not _applications:
- _applications = dict([(n,Application.make(n)) for n in _application_list ])
+ _applications = dict()
+ for dist in pkg_resources.working_set:
+ for appname, entry in dist.get_entry_map("wizard.app").items():
+ if appname in _applications:
+ newname = dist.key + ":" + appname
+ if newname in _applications:
+ raise Exception("Unrecoverable application name conflict for %s from %s", appname, dist.key)
+ logging.warning("Could not overwrite %s, used %s instead", appname, newname)
+ appname = newname
+ appclass = entry.load()
+ _applications[appname] = appclass(appname)
+ # setup dummy apps
+ for entry in pkg_resources.iter_entry_points("wizard.dummy_apps"):
+ appfun = entry.load()
+ dummy_apps = appfun()
+ for appname in dummy_apps:
+ # a dummy app that already exists is not a fatal error
+ if appname in _applications:
+ continue
+ _applications[appname] = Application(appname)
return _applications
def getApplication(appname):
"""Retrieves application instance given a name"""
- return applications()[appname]
+ try:
+ return applications()[appname]
+ except KeyError:
+ raise NoSuchApplication(appname)
class Application(object):
"""
deprecated_keys = set()
#: Keys that we can simply generate random strings for if they're missing
random_keys = set()
+ #: Values that are not sufficiently random for a random key. This can
+ #: include default values for a random configuration option,
+ random_blacklist = set()
#: Dictionary of variable names to extractor functions. These functions
#: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
#: the variable, or ``None`` if it could not be found.
#: :class:`wizard.deploy.Deployment`; the value here is merely the preferred
#: value.
database = None
+ #: Indicates whether or not a web stub is necessary.
+ needs_web_stub = False
def __init__(self, name):
self.name = name
self.versions = {}
result[k] = extractor(deployment)
# XXX: ugh... we have to do quoting
for k in self.random_keys:
- if result[k] is None:
- result[k] = "'%s'" % ''.join(random.choice(string.letters + string.digits) for i in xrange(30))
+ if result[k] is None or result[k] in self.random_blacklist:
+ result[k] = "'%s'" % util.random_key()
return result
def dsn(self, deployment):
"""
Returns the deployment specific database URL. Uses the override file
- in :file:`.scripts` if it exists, and otherwise attempt to extract the
+ in :file:`.wizard` if it exists, and otherwise attempt to extract the
variables from the source files.
Under some cases, the database URL will contain only the database
def url(self, deployment):
"""
Returns the deployment specific web URL. Uses the override file
- in :file:`.scripts` if it exists, and otherwise attempt to extract
+ in :file:`.wizard` if it exists, and otherwise attempt to extract
the variables from the source files.
This function might return ``None``, which indicates we couldn't figure
Takes a generic source checkout and parametrizes it according to the
values of ``deployment``. This function operates on the current
working directory. ``deployment`` should **not** be the same as the
- current working directory. Default implementation uses
- :attr:`parametrized_files` and a simple search and replace on those
- files.
+ current working directory. See :meth:`parametrizeWithVars` for details
+ on the parametrization.
"""
+ # deployment is not used in this implementation, but note that
+ # we do have the invariant the current directory matches
+ # deployment's directory
variables = ref_deployment.extract()
+ self.parametrizeWithVars(variables)
+ def parametrizeWithVars(self, variables):
+ """
+ Takes a generic source checkout and parametrizes it according to
+ the values of ``variables``. Default implementation uses
+ :attr:`parametrized_files` and a simple search and replace on
+ those files.
+ """
for file in self.parametrized_files:
+ logging.debug("Parametrizing file '%s'\n" % (file, ))
try:
contents = open(file, "r").read()
except IOError:
+ logging.debug("Failed to open file '%s'\n" % (file, ))
continue
for key, value in variables.items():
if value is None: continue
shell.call("git", "rm", file)
continue
# manual resolutions
+ # XXX: this functionality is mostly subsumed by the rerere
+ # tricks we do
if file in self.resolutions:
contents = open(file, "r").read()
for spec, result in self.resolutions[file]:
"""
for key, subst in self.substitutions.items():
subs = subst(deployment)
- if not subs and key not in self.deprecated_keys:
+ if not subs and key not in self.deprecated_keys and key not in self.random_keys:
logging.warning("No substitutions for %s" % key)
def install(self, version, options):
"""
match = regex.search(contents)
if not match: return None
return distutils.version.LooseVersion(shlex.split(match.group(2))[0])
+ # XXX: This signature doesn't really make too much sense...
+ def detectVersionFromGit(self, tagPattern, preStrip = ''):
+ """
+ Helper method that detects a version by using the most recent tag
+ in git that matches the specified pattern.
+ This assumes that the current working directory is the deployment.
+ """
+ sh = wizard.shell.Shell()
+ cmd = ['git', 'describe', '--tags', '--match', tagPattern, ]
+ tag = sh.call(*cmd, strip=True)
+ if tag and len(tag) > len(preStrip) and tag[:len(preStrip)] == preStrip:
+ tag = tag[len(preStrip):]
+ if not tag: return None
+ return distutils.version.LooseVersion(tag)
def download(self, version):
"""
Returns a URL that can be used to download a tarball of ``version`` of
not to depend on pages that are not the main page.
"""
raise NotImplementedError
+ def checkDatabase(self, deployment):
+ """
+ Checks if the database is accessible.
+ """
+ try:
+ sql.connect(deployment.dsn)
+ return True
+ except sqlalchemy.exc.DBAPIError:
+ return False
def checkWebPage(self, deployment, page, outputs=[], exclude=[]):
"""
Checks if a given page of an autoinstall contains a particular string.
Subclasses should provide an implementation.
"""
# XXX: Unfortunately, this doesn't quite work because we package
- # bogus config files in the -scripts versions of installs. Maybe
- # we should check a hash or something?
+ # bogus config files. Maybe we should check a hash or
+ # something?
raise NotImplementedError
def researchFilter(self, filename, added, deleted):
"""
be displayed in verbose mode.
"""
return filename in self.parametrized_files
- @staticmethod
- def make(name):
- """Makes an application, but uses the correct subtype if available."""
- try:
- __import__("wizard.app." + name)
- return getattr(wizard.app, name).Application(name)
- except ImportError as error:
- # XXX ugly hack to check if the import error is from the top level
- # module we care about or a submodule. should be an archetectural change.
- if error.args[0].split()[-1]==name:
- return Application(name)
- else:
- raise
class ApplicationVersion(object):
"""Represents an abstract notion of a version for an application, where
"""
return "%s-%s" % (self.application, self.version)
@property
- def scripts_tag(self):
+ def wizard_tag(self):
"""
Returns the name of the Git tag for this version.
"""
+ # XXX: Scripts specific
end = str(self.version).partition('-scripts')[2].partition('-')[0]
return "%s-scripts%s" % (self.pristine_tag, end)
@property
Makes/retrieves a singleton :class:`ApplicationVersion` from
a``app`` and ``version`` string.
"""
- try:
- # defer to the application for version creation to enforce
- # singletons
- return applications()[app].makeVersion(version)
- except KeyError:
- raise NoSuchApplication(app)
+ # defer to the application for version creation to enforce
+ # singletons
+ return getApplication(app).makeVersion(version)
def expand_re(val):
"""
return subs
return h
-def backup_database(outdir, deployment):
+@decorator.decorator
+def throws_database_errors(f, self, *args, **kwargs):
"""
- Generic database backup function for MySQL.
+ Decorator that takes database errors from :mod:`wizard.sql` and
+ converts them into application script failures from
+ :mod:`wizard.app`. We can't throw application errors directly from
+ :mod:`wizard.sql` because that would result in a cyclic import;
+ also, it's cleaner to distinguish between a database error and an
+ application script failure.
"""
- # XXX: Change this once deployments support multiple dbs
- if deployment.application.database == "mysql":
- return backup_mysql_database(outdir, deployment)
- else:
- raise NotImplementedError
-
-def backup_mysql_database(outdir, deployment):
- """
- Database backups for MySQL using the :command:`mysqldump` utility.
- """
- outfile = os.path.join(outdir, "db.sql")
try:
- shell.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment.dsn))
- shell.call("gzip", "--best", outfile)
- except shell.CallError as e:
- raise BackupFailure(e.stderr)
-
-def restore_database(backup_dir, deployment):
- """
- Generic database restoration function for MySQL.
- """
- # XXX: see backup_database
- if deployment.application.database == "mysql":
- return restore_mysql_database(backup_dir, deployment)
- else:
- raise NotImplementedError
-
-def restore_mysql_database(backup_dir, deployment):
- """
- Database restoration for MySQL by piping SQL commands into :command:`mysql`.
- """
- if not os.path.exists(backup_dir):
- raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
- sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
- shell.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
- sql.seek(0)
- shell.call("mysql", *get_mysql_args(deployment.dsn), stdin=sql)
- sql.close()
-
-def remove_database(deployment):
- """
- Generic database removal function. Actually, not so generic because we
- go and check if we're on scripts and if we are run a different command.
- """
- if deployment.dsn.host == "sql.mit.edu":
- try:
- shell.call("/mit/scripts/sql/bin/drop-database", deployment.dsn.database)
- return
- except shell.CallError:
- pass
- engine = sqlalchemy.create_engine(deployment.dsn)
- engine.execute("DROP DATABASE `%s`" % deployment.dsn.database)
-
-def get_mysql_args(dsn):
- """
- Extracts arguments that would be passed to the command line mysql utility
- from a deployment.
- """
- args = []
- if dsn.host:
- args += ["-h", dsn.host]
- if dsn.username:
- args += ["-u", dsn.username]
- if dsn.password:
- args += ["-p" + dsn.password]
- args += [dsn.database]
- return args
+ return f(self, *args, **kwargs)
+ except sql.BackupDatabaseError:
+ raise BackupFailure(traceback.format_exc())
+ except sql.RestoreDatabaseError:
+ raise RestoreFailure(traceback.format_exc())
+ except sql.RemoveDatabaseError:
+ raise RemoveFailure(traceback.format_exc())
class Error(wizard.Error):
"""Generic error class for this module."""
location = None
def __init__(self, value):
self.value = value
+ def __str__(self):
+ return "Could not parse '%s' from versions store in '%s'" % (self.value, self.location)
class NoSuchApplication(Error):
"""
location = None
def __init__(self, app):
self.app = app
+ def __str__(self):
+ return "Wizard doesn't know about an application named '%s'." % self.app
class Failure(Error):
"""
scripts.mit.edu / wizard.git / blobdiff