+"""
+Plumbing object model for representing applications we want to
+install. This module does the heavy lifting, but you probably
+want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
+You'll need to know how to overload the :class:`Application` class
+and use some of the functions in this module in order to specify
+new applications.
+
+.. testsetup:: *
+
+ import re
+ import shutil
+ import os
+ from wizard import deploy, util
+ from wizard.app import *
+"""
+
import os.path
import re
import distutils.version
+import decorator
+import shlex
+import logging
+import shutil
import wizard
+from wizard import resolve, scripts, shell, util
_application_list = [
"mediawiki", "wordpress", "joomla", "e107", "gallery2",
.. note::
Many of these methods assume a specific working
directory; prefer using the corresponding methods
- in :class:`deploy.Deployment` and its subclasses.
+ in :class:`wizard.deploy.Deployment` and its subclasses.
"""
#: String name of the application
name = None
#: Keys that are used in older versions of the application, but
#: not for the most recent version.
deprecated_keys = []
+ #: Dictionary of variable names to extractor functions. These functions
+ #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
+ #: the variable, or ``None`` if it could not be found.
+ #: See also :func:`filename_regex_extractor`.
+ extractors = {}
+ #: Dictionary of variable names to substitution functions. These functions
+ #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
+ #: that an explicit instance of the variable is released with the generic
+ #: ``WIZARD_*`` constant. See also :func:`filename_regex_substitution`.
+ substitutions = {}
+ #: Dictionary of file names to a list of resolutions, which are tuples of
+ #: a conflict marker string and a result list. See :mod:`wizard.resolve`
+ #: for more information.
+ resolutions = {}
def __init__(self, name):
self.name = name
self.versions = {}
self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
return self.versions[version]
def extract(self, deployment):
- """Extracts wizard variables from a deployment."""
+ """
+ Extracts wizard variables from a deployment. Default implementation
+ uses :attr:`extractors`.
+ """
result = {}
for k,extractor in self.extractors.items():
result[k] = extractor(deployment)
return result
- def parametrize(self, deployment):
+ def parametrize(self, deployment, ref_deployment):
"""
- Takes a generic source checkout and parametrizes
- it according to the values of deployment. This function
- operates on the current working directory.
+ Takes a generic source checkout and parametrizes it according to the
+ values of ``deployment``. This function operates on the current
+ working directory. ``deployment`` should **not** be the same as the
+ current working directory. Default implementation uses
+ :attr:`parametrized_files` and a simple search and replace on those
+ files.
"""
- variables = deployment.extract()
+ variables = ref_deployment.extract()
for file in self.parametrized_files:
try:
contents = open(file, "r").read()
f.write(contents)
def resolveConflicts(self, deployment):
"""
- Resolves conflicted files in the current working
- directory. Returns whether or not all conflicted
- files were resolved or not. Fully resolved files are
- added to the index, but no commit is made. By default
- this is a no-op and returns ``False``.
+ Resolves conflicted files in the current working directory. Returns
+ whether or not all conflicted files were resolved or not. Fully
+ resolved files are added to the index, but no commit is made. The
+ default implementation uses :attr:`resolutions`.
"""
- return False
+ resolved = True
+ sh = shell.Shell()
+ for status in sh.eval("git", "ls-files", "--unmerged").splitlines():
+ file = status.split()[-1]
+ if file in self.resolutions:
+ contents = open(file, "r").read()
+ for spec, result in self.resolutions[file]:
+ old_contents = contents
+ contents = resolve.resolve(contents, spec, result)
+ if old_contents != contents:
+ logging.info("Did resolution with spec:\n" + spec)
+ open(file, "w").write(contents)
+ if not resolve.is_conflict(contents):
+ sh.call("git", "add", file)
+ else:
+ resolved = False
+ else:
+ resolved = False
+ return resolved
def prepareMerge(self, deployment):
"""
Performs various edits to files in the current working directory in
order to make a merge go more smoothly. This is usually
used to fix botched line-endings. If you add new files,
you have to 'git add' them; this is not necessary for edits.
- By default this is a no-op.
+ By default this is a no-op; subclasses should replace this
+ with useful behavior.
"""
pass
def prepareConfig(self, deployment):
"""
Takes a deployment and replaces any explicit instances
of a configuration variable with generic ``WIZARD_*`` constants.
- The default implementation uses :attr:`substitutions`;
- you can override this method to provide arbitrary extra
- behavior.
+ The default implementation uses :attr:`substitutions`, and
+ emits warnings when it encounters keys in :attr:`deprecated_keys`.
"""
for key, subst in self.substitutions.items():
subs = subst(deployment)
logging.warning("No substitutions for %s" % key)
def install(self, version, options):
"""
- Run for 'wizard configure' (and, by proxy, 'wizard install')
- to configure an application. This assumes that the current
- working directory is a deployment. (This function does not take
- a :class:`deploy.Deployment` as a parameter, as those operations are
- not meaningful yet.)
+ Run for 'wizard configure' (and, by proxy, 'wizard install') to
+ configure an application. This assumes that the current working
+ directory is a deployment. (Unlike its kin, this function does not
+ take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should
+ provide an implementation.
"""
raise NotImplemented
def upgrade(self, deployment, version, options):
"""
Run for 'wizard upgrade' to upgrade database schemas and other
- non-versioned data in an application. This assumes that
- the current working directory is the deployment.
+ non-versioned data in an application after the filesystem has been
+ upgraded. This assumes that the current working directory is the
+ deployment. Subclasses should provide an implementation.
"""
raise NotImplemented
- def backup(self, deployment, options):
+ def backup(self, deployment, outdir, options):
"""
Run for 'wizard backup' and upgrades to backup database schemas
- and other non-versioned data in an application. This assumes
- that the current working directory is the deployment.
+ and other non-versioned data in an application. ``outdir`` is
+ the directory that backup files should be placed. This assumes
+ that the current working directory is the deployment. Subclasses
+ should provide an implementation, even if it is a no-op.
+
+ .. note::
+ Static user files may not need to be backed up, since in
+ many applications upgrades do not modify static files.
"""
raise NotImplemented
- def restore(self, deployment, backup, options):
+ def restore(self, deployment, backup_dir, options):
"""
Run for 'wizard restore' and failed upgrades to restore database
and other non-versioned data to a backed up version. This assumes
- that the current working directory is the deployment.
+ that the current working directory is the deployment. Subclasses
+ should provide an implementation.
"""
raise NotImplemented
def detectVersion(self, deployment):
"""
Checks source files to determine the version manually. This assumes
- that the current working directory is the deployment.
+ that the current working directory is the deployment. Subclasses
+ should provide an implementation.
"""
- return None
+ raise NotImplemented
def checkWeb(self, deployment, output=None):
"""
Checks if the autoinstall is viewable from the web. To get
the HTML source that was retrieved, pass a variable containing
an empty list to ``output``; it will be mutated to have its
- first element be the output.
+ first element be the output. Subclasses should provide an
+ implementation.
+
+ .. note::
+ Finding a reasonable heuristic that works across skinning
+ choices can be difficult. We've had reasonable success
+ searching for metadata. Be sure that the standard error
+ page does not contain the features you search for. Try
+ not to depend on pages that are not the main page.
"""
raise NotImplemented
def checkConfig(self, deployment):
"""
Checks whether or not an autoinstall has been configured/installed
for use. Assumes that the current working directory is the deployment.
+ Subclasses should provide an implementation.
"""
raise NotImplemented
- @property
- def extractors(self):
- """
- Dictionary of variable names to extractor functions. These functions
- take a :class:`deploy.Deployment` as an argument and return the value of
- the variable, or ``None`` if it could not be found.
- See also :func:`wizard.app.filename_regex_extractor`.
- """
- return {}
- @property
- def substitutions(self):
- """
- Dictionary of variable names to substitution functions. These functions
- take a :class:`deploy.Deployment` as an argument and modify the deployment such
- that an explicit instance of the variable is released with the generic
- WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`.
- """
- return {}
@staticmethod
def make(name):
"""Makes an application, but uses the correct subtype if available."""
Use this only for cases when speed is of primary importance;
the data in version is unreliable and when possible, you should
- prefer directly instantiating a :class:`deploy.Deployment` and having it query
+ prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
the autoinstall itself for information.
The `value` to parse will vary. For old style installs, it
raise NoSuchApplication(app)
def expand_re(val):
+ """
+ Takes a tree of values (implement using nested lists) and
+ transforms them into regular expressions.
+
+ >>> expand_re('*')
+ '\\\\*'
+ >>> expand_re(['a', 'b'])
+ '(?:a|b)'
+ >>> expand_re(['*', ['b', 'c']])
+ '(?:\\\\*|(?:b|c))'
+ """
if isinstance(val, str):
return re.escape(val)
else:
return '(?:' + '|'.join(map(expand_re, val)) + ')'
-def filename_regex_extractor(f):
+def make_extractors(seed):
+ """
+ Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into
+ extractor functions (which take a :class:`wizard.deploy.Deployment`
+ and return the value of the second subpattern of ``regex`` when matched
+ with the contents of ``file``).
"""
- This is a decorator to apply to functions that take a name and return
- (filename, RegexObject) tuples. It converts it into a function
- that takes a name and returns another function (the actual extractor)
- which takes a deployment and returns the value of the extracted variable.
+ return util.dictmap(lambda a: filename_regex_extractor(*a), seed)
- The regular expression requires a very specific form, essentially ()()()
- (with the second subgroup being the value we care about), so that we can
- reuse the regex for other things.
+def make_substitutions(seed):
+ """
+ Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into substitution
+ functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
+ of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
+ """
+ return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)
+
+# The following two functions are *highly* functional, and I recommend
+# not touching them unless you know what you're doing.
+
+def filename_regex_extractor(file, regex):
+ """
+ .. highlight:: haskell
+
+ Given a relative file name ``file``, a regular expression ``regex``, and a
+ :class:`wizard.deploy.Deployment` extracts a value out of the file in that
+ deployment. This function is curried, so you pass just ``file`` and
+ ``regex``, and then pass ``deployment`` to the resulting function.
Its Haskell-style type signature would be::
- (String -> (Filename, Regex)) -> (String -> (Deployment -> String))
+ Filename -> Regex -> (Deployment -> String)
+
+ The regular expression requires a very specific form, essentially ``()()()``
+ (with the second subgroup being the value to extract). These enables
+ the regular expression to be used equivalently with filename
+
+ .. highlight:: python
- For convenience purposes, we also accept [Filename], in which case
+ For convenience purposes, we also accept ``[Filename]``, in which case
we use the first entry (index 0). Passing an empty list is invalid.
+
+ >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
+ >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
+ >>> f(deploy.Deployment("."))
+ '3'
+ >>> os.unlink("test-settings.extractor.ini")
+
+ .. note::
+ The first application of ``regex`` and ``file`` is normally performed
+ at compile-time inside a submodule; the second application is
+ performed at runtime.
"""
- def g(var):
- file, regex = f(var)
- if not isinstance(file, str):
- file = file[0]
- def h(deployment):
- try:
- contents = deployment.read(file) # cached
- except IOError:
- return None
- match = regex.search(contents)
- if not match: return None
- # assumes that the second match is the one we want.
- return match.group(2)
- return h
- return g
-
-def filename_regex_substitution(f):
+ if not isinstance(file, str):
+ file = file[0]
+ def h(deployment):
+ try:
+ contents = deployment.read(file) # cached
+ except IOError:
+ return None
+ match = regex.search(contents)
+ if not match: return None
+ # assumes that the second match is the one we want.
+ return match.group(2)
+ return h
+
+def filename_regex_substitution(key, files, regex):
"""
- This is a decorator to apply to functions that take a name and return
- (filename, RegexObject) tuples. It converts it into a function
- that takes a name and returns another function (that does substitution)
- which takes a deployment and modifies its files to replace explicit
- values with their generic WIZARD_* equivalents. The final function returns
- the number of replacements made.
+ .. highlight:: haskell
- The regular expression requires a very specific form, essentially ()()()
- (with the second subgroup being the value to be replaced).
+ Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
+ regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
+ performs a substitution of the second subpattern of ``regex``
+ with ``key``. Returns the number of replacements made. This function
+ is curried, so you pass just ``key``, ``files`` and ``regex``, and
+ then pass ``deployment`` to the resulting function.
Its Haskell-style type signature would be::
- (String -> ([Filename], Regex)) -> (String -> (Deployment -> IO Int))
+ Key -> ([File], Regex) -> (Deployment -> IO Int)
- For convenience purposes, we also accept Filename, in which case it is treated
+ .. highlight:: python
+
+ For convenience purposes, we also accept ``Filename``, in which case it is treated
as a single item list.
+
+ >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
+ >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
+ >>> f(deploy.Deployment("."))
+ 1
+ >>> print open("test-settings.substitution.ini", "r").read()
+ config_var = WIZARD_KEY
+ >>> os.unlink("test-settings.substitution.ini")
+ """
+ if isinstance(files, str):
+ files = (files,)
+ def h(deployment):
+ base = deployment.location
+ subs = 0
+ for file in files:
+ file = os.path.join(base, file)
+ try:
+ contents = open(file, "r").read()
+ contents, n = regex.subn("\\1" + key + "\\3", contents)
+ subs += n
+ open(file, "w").write(contents)
+ except IOError:
+ pass
+ return subs
+ return h
+
+# XXX: rename to show that it's mysql specific
+def backup_database(outdir, deployment):
+ """
+ Generic database backup function for MySQL. Assumes that ``WIZARD_DBNAME``
+ is extractable, and that :func:`wizard.scripts.get_sql_credentials`
+ works.
+ """
+ sh = shell.Shell()
+ outfile = os.path.join(outdir, "db.sql")
+ try:
+ sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment))
+ sh.call("gzip", "--best", outfile)
+ except shell.CallError as e:
+ shutil.rmtree(outdir)
+ raise BackupFailure(e.stderr)
+
+def restore_database(backup_dir, deployment):
+ """
+ Generic database restoration function for MySQL. See :func:`backup_database`
+ for the assumptions that we make.
+ """
+ sh = shell.Shell()
+ if not os.path.exists(backup_dir):
+ raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
+ sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
+ sh.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
+ sql.seek(0)
+ sh.call("mysql", *get_mysql_args(deployment), stdin=sql)
+ sql.close()
+
+def get_mysql_args(d):
+ """
+ Extracts arguments that would be passed to the command line mysql utility
+ from a deployment.
"""
- def g(key, var):
- files, regex = f(var)
- if isinstance(files, str):
- files = (files,)
- def h(deployment):
- base = deployment.location
- subs = 0
- for file in files:
- file = os.path.join(base, file)
- try:
- contents = open(file, "r").read()
- contents, n = regex.subn("\\1" + key + "\\3", contents)
- subs += n
- open(file, "w").write(contents)
- except IOError:
- pass
- return subs
- return h
- return g
+ # XXX: add support for getting these out of options
+ vars = d.extract()
+ if 'WIZARD_DBNAME' not in vars:
+ raise app.BackupFailure("Could not determine database name")
+ triplet = scripts.get_sql_credentials(vars)
+ args = []
+ if triplet is not None:
+ server, user, password = triplet
+ args += ["-h", server, "-u", user, "-p" + password]
+ name = shlex.split(vars['WIZARD_DBNAME'])[0]
+ args.append(name)
+ return args
class Error(wizard.Error):
"""Generic error class for this module."""
scripts.mit.edu / wizard.git / blobdiff