2 Plumbing object model for representing applications we want to
3 install. This module does the heavy lifting, but you probably
4 want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
5 You'll need to know how to overload the :class:`Application` class
6 and use some of the functions in this module in order to specify
9 There are some submodules for programming languages that define common
10 functions and data that may be used by applications in that language. See:
12 * :mod:`wizard.app.php`
19 from wizard import deploy, util
20 from wizard.app import *
25 import distutils.version
32 from wizard import resolve, scripts, shell, util
35 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
36 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
37 # these are technically deprecated
38 "advancedpoll", "gallery",
43 """Hash table for looking up string application name to instance"""
46 _applications = dict([(n,Application.make(n)) for n in _application_list ])
50 class Application(object):
52 Represents an application, i.e. mediawiki or phpbb.
55 Many of these methods assume a specific working
56 directory; prefer using the corresponding methods
57 in :class:`wizard.deploy.Deployment` and its subclasses.
59 #: String name of the application
61 #: Dictionary of version strings to :class:`ApplicationVersion`.
62 #: See also :meth:`makeVersion`.
64 #: List of files that need to be modified when parametrizing.
65 #: This is a class-wide constant, and should not normally be modified.
66 parametrized_files = []
67 #: Keys that are used in older versions of the application, but
68 #: not for the most recent version.
70 #: Dictionary of variable names to extractor functions. These functions
71 #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
72 #: the variable, or ``None`` if it could not be found.
73 #: See also :func:`filename_regex_extractor`.
75 #: Dictionary of variable names to substitution functions. These functions
76 #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
77 #: that an explicit instance of the variable is released with the generic
78 #: ``WIZARD_*`` constant. See also :func:`filename_regex_substitution`.
80 #: Dictionary of file names to a list of resolutions, which are tuples of
81 #: a conflict marker string and a result list. See :mod:`wizard.resolve`
82 #: for more information.
84 #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
85 #: this application requires.
87 def __init__(self, name):
92 self._substitutions = {}
93 def repository(self, srv_path):
95 Returns the Git repository that would contain this application.
96 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
98 repo = os.path.join(srv_path, self.name + ".git")
99 if not os.path.isdir(repo):
100 repo = os.path.join(srv_path, self.name, ".git")
101 if not os.path.isdir(repo):
102 raise NoRepositoryError(self.name)
104 def makeVersion(self, version):
106 Creates or retrieves the :class:`ApplicationVersion` singleton for the
109 if version not in self.versions:
110 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
111 return self.versions[version]
112 def extract(self, deployment):
114 Extracts wizard variables from a deployment. Default implementation
115 uses :attr:`extractors`.
118 for k,extractor in self.extractors.items():
119 result[k] = extractor(deployment)
121 def parametrize(self, deployment, ref_deployment):
123 Takes a generic source checkout and parametrizes it according to the
124 values of ``deployment``. This function operates on the current
125 working directory. ``deployment`` should **not** be the same as the
126 current working directory. Default implementation uses
127 :attr:`parametrized_files` and a simple search and replace on those
130 variables = ref_deployment.extract()
131 for file in self.parametrized_files:
133 contents = open(file, "r").read()
136 for key, value in variables.items():
137 if value is None: continue
138 contents = contents.replace(key, value)
141 def resolveConflicts(self, deployment):
143 Resolves conflicted files in the current working directory. Returns
144 whether or not all conflicted files were resolved or not. Fully
145 resolved files are added to the index, but no commit is made. The
146 default implementation uses :attr:`resolutions`.
150 for status in sh.eval("git", "ls-files", "--unmerged").splitlines():
151 file = status.split()[-1]
152 if file in self.resolutions:
153 contents = open(file, "r").read()
154 for spec, result in self.resolutions[file]:
155 old_contents = contents
156 contents = resolve.resolve(contents, spec, result)
157 if old_contents != contents:
158 logging.info("Did resolution with spec:\n" + spec)
159 open(file, "w").write(contents)
160 if not resolve.is_conflict(contents):
161 sh.call("git", "add", file)
167 def prepareMerge(self, deployment):
169 Performs various edits to files in the current working directory in
170 order to make a merge go more smoothly. This is usually
171 used to fix botched line-endings. If you add new files,
172 you have to 'git add' them; this is not necessary for edits.
173 By default this is a no-op; subclasses should replace this
174 with useful behavior.
177 def prepareConfig(self, deployment):
179 Takes a deployment and replaces any explicit instances
180 of a configuration variable with generic ``WIZARD_*`` constants.
181 The default implementation uses :attr:`substitutions`, and
182 emits warnings when it encounters keys in :attr:`deprecated_keys`.
184 for key, subst in self.substitutions.items():
185 subs = subst(deployment)
186 if not subs and key not in self.deprecated_keys:
187 logging.warning("No substitutions for %s" % key)
188 def install(self, version, options):
190 Run for 'wizard configure' (and, by proxy, 'wizard install') to
191 configure an application. This assumes that the current working
192 directory is a deployment. (Unlike its kin, this function does not
193 take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should
194 provide an implementation.
196 raise NotImplementedError
197 def upgrade(self, deployment, version, options):
199 Run for 'wizard upgrade' to upgrade database schemas and other
200 non-versioned data in an application after the filesystem has been
201 upgraded. This assumes that the current working directory is the
202 deployment. Subclasses should provide an implementation.
204 raise NotImplementedError
205 def backup(self, deployment, outdir, options):
207 Run for 'wizard backup' and upgrades to backup database schemas
208 and other non-versioned data in an application. ``outdir`` is
209 the directory that backup files should be placed. This assumes
210 that the current working directory is the deployment. Subclasses
211 should provide an implementation, even if it is a no-op.
214 Static user files may not need to be backed up, since in
215 many applications upgrades do not modify static files.
217 raise NotImplementedError
218 def restore(self, deployment, backup_dir, options):
220 Run for 'wizard restore' and failed upgrades to restore database
221 and other non-versioned data to a backed up version. This assumes
222 that the current working directory is the deployment. Subclasses
223 should provide an implementation.
225 raise NotImplementedError
226 def detectVersion(self, deployment):
228 Checks source files to determine the version manually. This assumes
229 that the current working directory is the deployment. Subclasses
230 should provide an implementation.
232 raise NotImplementedError
233 def detectVersionFromFile(self, filename, regex):
235 Helper method that detects a version by using a regular expression
236 from a file. The regexed value is passed through :mod:`shlex`.
237 This assumes that the current working directory is the deployment.
239 contents = open(filename).read()
240 match = regex.search(contents)
241 if not match: return None
242 return distutils.version.LooseVersion(shlex.split(match.group(2))[0])
243 def download(self, version):
245 Returns a URL that can be used to download a tarball of ``version`` of
248 raise NotImplementedError
249 def checkWeb(self, deployment, output=None):
251 Checks if the autoinstall is viewable from the web. To get
252 the HTML source that was retrieved, pass a variable containing
253 an empty list to ``output``; it will be mutated to have its
254 first element be the output. Subclasses should provide an
258 Finding a reasonable heuristic that works across skinning
259 choices can be difficult. We've had reasonable success
260 searching for metadata. Be sure that the standard error
261 page does not contain the features you search for. Try
262 not to depend on pages that are not the main page.
264 raise NotImplementedError
265 def checkConfig(self, deployment):
267 Checks whether or not an autoinstall has been configured/installed
268 for use. Assumes that the current working directory is the deployment.
269 Subclasses should provide an implementation.
271 # XXX: Unfortunately, this doesn't quite work because we package
272 # bogus config files in the -scripts versions of installs. Maybe
273 # we should check a hash or something?
274 raise NotImplementedError
277 """Makes an application, but uses the correct subtype if available."""
279 __import__("wizard.app." + name)
280 return getattr(wizard.app, name).Application(name)
282 return Application(name)
284 class ApplicationVersion(object):
285 """Represents an abstract notion of a version for an application, where
286 ``version`` is a :class:`distutils.version.LooseVersion` and
287 ``application`` is a :class:`Application`."""
288 #: The :class:`distutils.version.LooseVersion` of this instance.
290 #: The :class:`Application` of this instance.
292 def __init__(self, version, application):
293 self.version = version
294 self.application = application
298 Returns the name of the git describe tag for the commit the user is
299 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
301 return "%s-%s" % (self.application, self.version)
303 def scripts_tag(self):
305 Returns the name of the Git tag for this version.
307 end = str(self.version).partition('-scripts')[2].partition('-')[0]
308 return "%s-scripts%s" % (self.pristine_tag, end)
310 def pristine_tag(self):
312 Returns the name of the Git tag for the pristine version corresponding
315 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
316 def __cmp__(self, y):
317 return cmp(self.version, y.version)
321 Parses a line from the :term:`versions store` and return
322 :class:`ApplicationVersion`.
324 Use this only for cases when speed is of primary importance;
325 the data in version is unreliable and when possible, you should
326 prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
327 the autoinstall itself for information.
329 The `value` to parse will vary. For old style installs, it
332 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
334 For new style installs, it will look like::
338 name = value.split("/")[-1]
340 if name.find("-") != -1:
341 app, _, version = name.partition("-")
343 # kind of poor, maybe should error. Generally this
344 # will actually result in a not found error
348 raise DeploymentParseError(value)
349 return ApplicationVersion.make(app, version)
351 def make(app, version):
353 Makes/retrieves a singleton :class:`ApplicationVersion` from
354 a``app`` and ``version`` string.
357 # defer to the application for version creation to enforce
359 return applications()[app].makeVersion(version)
361 raise NoSuchApplication(app)
365 Takes a tree of values (implement using nested lists) and
366 transforms them into regular expressions.
370 >>> expand_re(['a', 'b'])
372 >>> expand_re(['*', ['b', 'c']])
375 if isinstance(val, str):
376 return re.escape(val)
378 return '(?:' + '|'.join(map(expand_re, val)) + ')'
380 def make_extractors(seed):
382 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into
383 extractor functions (which take a :class:`wizard.deploy.Deployment`
384 and return the value of the second subpattern of ``regex`` when matched
385 with the contents of ``file``).
387 return util.dictmap(lambda a: filename_regex_extractor(*a), seed)
389 def make_substitutions(seed):
391 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into substitution
392 functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
393 of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
395 return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)
397 # The following two functions are *highly* functional, and I recommend
398 # not touching them unless you know what you're doing.
400 def filename_regex_extractor(file, regex):
402 .. highlight:: haskell
404 Given a relative file name ``file``, a regular expression ``regex``, and a
405 :class:`wizard.deploy.Deployment` extracts a value out of the file in that
406 deployment. This function is curried, so you pass just ``file`` and
407 ``regex``, and then pass ``deployment`` to the resulting function.
409 Its Haskell-style type signature would be::
411 Filename -> Regex -> (Deployment -> String)
413 The regular expression requires a very specific form, essentially ``()()()``
414 (with the second subgroup being the value to extract). These enables
415 the regular expression to be used equivalently with filename
417 .. highlight:: python
419 For convenience purposes, we also accept ``[Filename]``, in which case
420 we use the first entry (index 0). Passing an empty list is invalid.
422 >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
423 >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
424 >>> f(deploy.Deployment("."))
426 >>> os.unlink("test-settings.extractor.ini")
429 The first application of ``regex`` and ``file`` is normally performed
430 at compile-time inside a submodule; the second application is
431 performed at runtime.
433 if not isinstance(file, str):
437 contents = deployment.read(file) # cached
440 match = regex.search(contents)
441 if not match: return None
442 # assumes that the second match is the one we want.
443 return match.group(2)
446 def filename_regex_substitution(key, files, regex):
448 .. highlight:: haskell
450 Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
451 regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
452 performs a substitution of the second subpattern of ``regex``
453 with ``key``. Returns the number of replacements made. This function
454 is curried, so you pass just ``key``, ``files`` and ``regex``, and
455 then pass ``deployment`` to the resulting function.
457 Its Haskell-style type signature would be::
459 Key -> ([File], Regex) -> (Deployment -> IO Int)
461 .. highlight:: python
463 For convenience purposes, we also accept ``Filename``, in which case it is treated
464 as a single item list.
466 >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
467 >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
468 >>> f(deploy.Deployment("."))
470 >>> print open("test-settings.substitution.ini", "r").read()
471 config_var = WIZARD_KEY
472 >>> os.unlink("test-settings.substitution.ini")
474 if isinstance(files, str):
477 base = deployment.location
480 file = os.path.join(base, file)
482 contents = open(file, "r").read()
483 contents, n = regex.subn("\\1" + key + "\\3", contents)
485 open(file, "w").write(contents)
491 # XXX: rename to show that it's mysql specific
492 def backup_database(outdir, deployment):
494 Generic database backup function for MySQL. Assumes that ``WIZARD_DBNAME``
495 is extractable, and that :func:`wizard.scripts.get_sql_credentials`
499 outfile = os.path.join(outdir, "db.sql")
501 sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment))
502 sh.call("gzip", "--best", outfile)
503 except shell.CallError as e:
504 shutil.rmtree(outdir)
505 raise BackupFailure(e.stderr)
507 def restore_database(backup_dir, deployment):
509 Generic database restoration function for MySQL. See :func:`backup_database`
510 for the assumptions that we make.
513 if not os.path.exists(backup_dir):
514 raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
515 sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
516 sh.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
518 sh.call("mysql", *get_mysql_args(deployment), stdin=sql)
521 def get_mysql_args(d):
523 Extracts arguments that would be passed to the command line mysql utility
526 # XXX: add support for getting these out of options
528 if 'WIZARD_DBNAME' not in vars:
529 raise BackupFailure("Could not determine database name")
530 triplet = scripts.get_sql_credentials(vars)
532 if triplet is not None:
533 server, user, password = triplet
534 args += ["-h", server, "-u", user, "-p" + password]
535 name = shlex.split(vars['WIZARD_DBNAME'])[0]
539 class Error(wizard.Error):
540 """Generic error class for this module."""
543 class NoRepositoryError(Error):
545 :class:`Application` does not appear to have a Git repository
546 in the normal location.
548 #: The name of the application that does not have a Git repository.
550 def __init__(self, app):
553 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
555 class DeploymentParseError(Error):
557 Could not parse ``value`` from :term:`versions store`.
559 #: The value that failed to parse.
561 #: The location of the autoinstall that threw this variable.
562 #: This should be set by error handling code when it is available.
564 def __init__(self, value):
567 class NoSuchApplication(Error):
569 You attempted to reference a :class:`Application` named
570 ``app``, which is not recognized by Wizard.
572 #: The name of the application that does not exist.
574 #: The location of the autoinstall that threw this variable.
575 #: This should be set by error handling code when it is availble.
577 def __init__(self, app):
580 class Failure(Error):
582 Represents a failure when performing some double-dispatched operation
583 such as an installation or an upgrade. Failure classes are postfixed
584 with Failure, not Error.
588 class InstallFailure(Error):
589 """Installation failed for unknown reason."""
592 class RecoverableInstallFailure(InstallFailure):
594 Installation failed, but we were able to determine what the
595 error was, and should give the user a second chance if we were
596 running interactively.
598 #: List of the errors that were found.
600 def __init__(self, errors):
603 return """Installation failed due to the following errors: %s""" % ", ".join(self.errors)
605 class UpgradeFailure(Failure):
606 """Upgrade script failed."""
607 #: String details of failure (possibly stdout or stderr output)
609 def __init__(self, details):
610 self.details = details
614 ERROR: Upgrade script failed, details:
618 class UpgradeVerificationFailure(Failure):
619 """Upgrade script passed, but website wasn't accessible afterwards"""
620 #: String details of failure (possibly stdout or stderr output)
622 def __init__(self, details):
623 self.details = details
627 ERROR: Upgrade script passed, but website wasn't accessible afterwards. Details:
631 class BackupFailure(Failure):
632 """Backup script failed."""
633 #: String details of failure
635 def __init__(self, details):
636 self.details = details
640 ERROR: Backup script failed, details:
644 class RestoreFailure(Failure):
645 """Restore script failed."""
646 #: String details of failure
648 def __init__(self, details):
649 self.details = details
653 ERROR: Restore script failed, details: