2 Plumbing object model for representing applications we want to
3 install. This module does the heavy lifting, but you probably
4 want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
5 You'll need to know how to overload the :class:`Application` class
6 and use some of the functions in this module in order to specify
9 There are some submodules for programming languages that define common
10 functions and data that may be used by applications in that language. See:
12 * :mod:`wizard.app.php`
19 from wizard import deploy, util
20 from wizard.app import *
25 import distutils.version
32 from wizard import resolve, scripts, shell, util
35 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
36 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
37 # these are technically deprecated
38 "advancedpoll", "gallery",
43 """Hash table for looking up string application name to instance"""
46 _applications = dict([(n,Application.make(n)) for n in _application_list ])
50 class Application(object):
52 Represents an application, i.e. mediawiki or phpbb.
55 Many of these methods assume a specific working
56 directory; prefer using the corresponding methods
57 in :class:`wizard.deploy.Deployment` and its subclasses.
59 #: String name of the application
61 #: Dictionary of version strings to :class:`ApplicationVersion`.
62 #: See also :meth:`makeVersion`.
64 #: List of files that need to be modified when parametrizing.
65 #: This is a class-wide constant, and should not normally be modified.
66 parametrized_files = []
67 #: Keys that are used in older versions of the application, but
68 #: not for the most recent version.
70 #: Dictionary of variable names to extractor functions. These functions
71 #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
72 #: the variable, or ``None`` if it could not be found.
73 #: See also :func:`filename_regex_extractor`.
75 #: Dictionary of variable names to substitution functions. These functions
76 #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
77 #: that an explicit instance of the variable is released with the generic
78 #: ``WIZARD_*`` constant. See also :func:`filename_regex_substitution`.
80 #: Dictionary of file names to a list of resolutions, which are tuples of
81 #: a conflict marker string and a result list. See :mod:`wizard.resolve`
82 #: for more information.
84 #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
85 #: this application requires.
87 def __init__(self, name):
92 self._substitutions = {}
93 def repository(self, srv_path):
95 Returns the Git repository that would contain this application.
96 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
98 repo = os.path.join(srv_path, self.name + ".git")
99 if not os.path.isdir(repo):
100 repo = os.path.join(srv_path, self.name, ".git")
101 if not os.path.isdir(repo):
102 raise NoRepositoryError(self.name)
104 def makeVersion(self, version):
106 Creates or retrieves the :class:`ApplicationVersion` singleton for the
109 if version not in self.versions:
110 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
111 return self.versions[version]
112 def extract(self, deployment):
114 Extracts wizard variables from a deployment. Default implementation
115 uses :attr:`extractors`.
118 for k,extractor in self.extractors.items():
119 result[k] = extractor(deployment)
121 def parametrize(self, deployment, ref_deployment):
123 Takes a generic source checkout and parametrizes it according to the
124 values of ``deployment``. This function operates on the current
125 working directory. ``deployment`` should **not** be the same as the
126 current working directory. Default implementation uses
127 :attr:`parametrized_files` and a simple search and replace on those
130 variables = ref_deployment.extract()
131 for file in self.parametrized_files:
133 contents = open(file, "r").read()
136 for key, value in variables.items():
137 if value is None: continue
138 contents = contents.replace(key, value)
141 def resolveConflicts(self, deployment):
143 Resolves conflicted files in the current working directory. Returns
144 whether or not all conflicted files were resolved or not. Fully
145 resolved files are added to the index, but no commit is made. The
146 default implementation uses :attr:`resolutions`.
150 for status in sh.eval("git", "ls-files", "--unmerged").splitlines():
151 file = status.split()[-1]
152 if file in self.resolutions:
153 contents = open(file, "r").read()
154 for spec, result in self.resolutions[file]:
155 old_contents = contents
156 contents = resolve.resolve(contents, spec, result)
157 if old_contents != contents:
158 logging.info("Did resolution with spec:\n" + spec)
159 open(file, "w").write(contents)
160 if not resolve.is_conflict(contents):
161 sh.call("git", "add", file)
167 def prepareMerge(self, deployment):
169 Performs various edits to files in the current working directory in
170 order to make a merge go more smoothly. This is usually
171 used to fix botched line-endings. If you add new files,
172 you have to 'git add' them; this is not necessary for edits.
173 By default this is a no-op; subclasses should replace this
174 with useful behavior.
177 def prepareConfig(self, deployment):
179 Takes a deployment and replaces any explicit instances
180 of a configuration variable with generic ``WIZARD_*`` constants.
181 The default implementation uses :attr:`substitutions`, and
182 emits warnings when it encounters keys in :attr:`deprecated_keys`.
184 for key, subst in self.substitutions.items():
185 subs = subst(deployment)
186 if not subs and key not in self.deprecated_keys:
187 logging.warning("No substitutions for %s" % key)
188 def install(self, version, options):
190 Run for 'wizard configure' (and, by proxy, 'wizard install') to
191 configure an application. This assumes that the current working
192 directory is a deployment. (Unlike its kin, this function does not
193 take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should
194 provide an implementation.
196 raise NotImplementedError
197 def upgrade(self, deployment, version, options):
199 Run for 'wizard upgrade' to upgrade database schemas and other
200 non-versioned data in an application after the filesystem has been
201 upgraded. This assumes that the current working directory is the
202 deployment. Subclasses should provide an implementation.
204 raise NotImplementedError
205 def backup(self, deployment, outdir, options):
207 Run for 'wizard backup' and upgrades to backup database schemas
208 and other non-versioned data in an application. ``outdir`` is
209 the directory that backup files should be placed. This assumes
210 that the current working directory is the deployment. Subclasses
211 should provide an implementation, even if it is a no-op.
214 Static user files may not need to be backed up, since in
215 many applications upgrades do not modify static files.
217 raise NotImplementedError
218 def restore(self, deployment, backup_dir, options):
220 Run for 'wizard restore' and failed upgrades to restore database
221 and other non-versioned data to a backed up version. This assumes
222 that the current working directory is the deployment. Subclasses
223 should provide an implementation.
225 raise NotImplementedError
226 def detectVersion(self, deployment):
228 Checks source files to determine the version manually. This assumes
229 that the current working directory is the deployment. Subclasses
230 should provide an implementation.
232 raise NotImplementedError
233 def detectVersionFromFile(self, filename, regex):
235 Helper method that detects a version by using a regular expression
236 from a file. The regexed value is passed through :mod:`shlex`.
237 This assumes that the current working directory is the deployment.
239 contents = open(filename).read()
240 match = regex.search(contents)
241 if not match: return None
242 return distutils.version.LooseVersion(shlex.split(match.group(2))[0])
243 def download(self, version):
245 Returns a URL that can be used to download a tarball of ``version`` of
248 raise NotImplementedError
249 def checkWeb(self, deployment):
251 Checks if the autoinstall is viewable from the web. Subclasses should
252 provide an implementation.
255 Finding a reasonable heuristic that works across skinning
256 choices can be difficult. We've had reasonable success
257 searching for metadata. Be sure that the standard error
258 page does not contain the features you search for. Try
259 not to depend on pages that are not the main page.
261 raise NotImplementedError
262 def checkWebPage(self, deployment, page, output):
264 Checks if a given page of an autoinstall contains a particular string.
266 page = deployment.fetch(page)
267 logging.info("checkWebPage:\n\n" + page)
268 return page.find(output) != -1
269 def checkConfig(self, deployment):
271 Checks whether or not an autoinstall has been configured/installed
272 for use. Assumes that the current working directory is the deployment.
273 Subclasses should provide an implementation.
275 # XXX: Unfortunately, this doesn't quite work because we package
276 # bogus config files in the -scripts versions of installs. Maybe
277 # we should check a hash or something?
278 raise NotImplementedError
281 """Makes an application, but uses the correct subtype if available."""
283 __import__("wizard.app." + name)
284 return getattr(wizard.app, name).Application(name)
286 return Application(name)
288 class ApplicationVersion(object):
289 """Represents an abstract notion of a version for an application, where
290 ``version`` is a :class:`distutils.version.LooseVersion` and
291 ``application`` is a :class:`Application`."""
292 #: The :class:`distutils.version.LooseVersion` of this instance.
294 #: The :class:`Application` of this instance.
296 def __init__(self, version, application):
297 self.version = version
298 self.application = application
302 Returns the name of the git describe tag for the commit the user is
303 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
305 return "%s-%s" % (self.application, self.version)
307 def scripts_tag(self):
309 Returns the name of the Git tag for this version.
311 end = str(self.version).partition('-scripts')[2].partition('-')[0]
312 return "%s-scripts%s" % (self.pristine_tag, end)
314 def pristine_tag(self):
316 Returns the name of the Git tag for the pristine version corresponding
319 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
320 def __cmp__(self, y):
321 return cmp(self.version, y.version)
325 Parses a line from the :term:`versions store` and return
326 :class:`ApplicationVersion`.
328 Use this only for cases when speed is of primary importance;
329 the data in version is unreliable and when possible, you should
330 prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
331 the autoinstall itself for information.
333 The `value` to parse will vary. For old style installs, it
336 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
338 For new style installs, it will look like::
342 name = value.split("/")[-1]
344 if name.find("-") != -1:
345 app, _, version = name.partition("-")
347 # kind of poor, maybe should error. Generally this
348 # will actually result in a not found error
352 raise DeploymentParseError(value)
353 return ApplicationVersion.make(app, version)
355 def make(app, version):
357 Makes/retrieves a singleton :class:`ApplicationVersion` from
358 a``app`` and ``version`` string.
361 # defer to the application for version creation to enforce
363 return applications()[app].makeVersion(version)
365 raise NoSuchApplication(app)
369 Takes a tree of values (implement using nested lists) and
370 transforms them into regular expressions.
374 >>> expand_re(['a', 'b'])
376 >>> expand_re(['*', ['b', 'c']])
379 if isinstance(val, str):
380 return re.escape(val)
382 return '(?:' + '|'.join(map(expand_re, val)) + ')'
384 def make_extractors(seed):
386 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into
387 extractor functions (which take a :class:`wizard.deploy.Deployment`
388 and return the value of the second subpattern of ``regex`` when matched
389 with the contents of ``file``).
391 return util.dictmap(lambda a: filename_regex_extractor(*a), seed)
393 def make_substitutions(seed):
395 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into substitution
396 functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
397 of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
399 return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)
401 # The following two functions are *highly* functional, and I recommend
402 # not touching them unless you know what you're doing.
404 def filename_regex_extractor(file, regex):
406 .. highlight:: haskell
408 Given a relative file name ``file``, a regular expression ``regex``, and a
409 :class:`wizard.deploy.Deployment` extracts a value out of the file in that
410 deployment. This function is curried, so you pass just ``file`` and
411 ``regex``, and then pass ``deployment`` to the resulting function.
413 Its Haskell-style type signature would be::
415 Filename -> Regex -> (Deployment -> String)
417 The regular expression requires a very specific form, essentially ``()()()``
418 (with the second subgroup being the value to extract). These enables
419 the regular expression to be used equivalently with filename
421 .. highlight:: python
423 For convenience purposes, we also accept ``[Filename]``, in which case
424 we use the first entry (index 0). Passing an empty list is invalid.
426 >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
427 >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
428 >>> f(deploy.Deployment("."))
430 >>> os.unlink("test-settings.extractor.ini")
433 The first application of ``regex`` and ``file`` is normally performed
434 at compile-time inside a submodule; the second application is
435 performed at runtime.
437 if not isinstance(file, str):
441 contents = deployment.read(file) # cached
444 match = regex.search(contents)
445 if not match: return None
446 # assumes that the second match is the one we want.
447 return match.group(2)
450 def filename_regex_substitution(key, files, regex):
452 .. highlight:: haskell
454 Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
455 regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
456 performs a substitution of the second subpattern of ``regex``
457 with ``key``. Returns the number of replacements made. This function
458 is curried, so you pass just ``key``, ``files`` and ``regex``, and
459 then pass ``deployment`` to the resulting function.
461 Its Haskell-style type signature would be::
463 Key -> ([File], Regex) -> (Deployment -> IO Int)
465 .. highlight:: python
467 For convenience purposes, we also accept ``Filename``, in which case it is treated
468 as a single item list.
470 >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
471 >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
472 >>> f(deploy.Deployment("."))
474 >>> print open("test-settings.substitution.ini", "r").read()
475 config_var = WIZARD_KEY
476 >>> os.unlink("test-settings.substitution.ini")
478 if isinstance(files, str):
481 base = deployment.location
484 file = os.path.join(base, file)
486 contents = open(file, "r").read()
487 contents, n = regex.subn("\\1" + key + "\\3", contents)
489 open(file, "w").write(contents)
495 # XXX: rename to show that it's mysql specific
496 def backup_database(outdir, deployment):
498 Generic database backup function for MySQL. Assumes that ``WIZARD_DBNAME``
499 is extractable, and that :func:`wizard.scripts.get_sql_credentials`
503 outfile = os.path.join(outdir, "db.sql")
505 sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment))
506 sh.call("gzip", "--best", outfile)
507 except shell.CallError as e:
508 shutil.rmtree(outdir)
509 raise BackupFailure(e.stderr)
511 def restore_database(backup_dir, deployment):
513 Generic database restoration function for MySQL. See :func:`backup_database`
514 for the assumptions that we make.
517 if not os.path.exists(backup_dir):
518 raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
519 sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
520 sh.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
522 sh.call("mysql", *get_mysql_args(deployment), stdin=sql)
525 def get_mysql_args(d):
527 Extracts arguments that would be passed to the command line mysql utility
530 # XXX: add support for getting these out of options
532 if 'WIZARD_DBNAME' not in vars:
533 raise BackupFailure("Could not determine database name")
534 triplet = scripts.get_sql_credentials(vars)
536 if triplet is not None:
537 server, user, password = triplet
538 args += ["-h", server, "-u", user, "-p" + password]
539 name = shlex.split(vars['WIZARD_DBNAME'])[0]
543 class Error(wizard.Error):
544 """Generic error class for this module."""
547 class NoRepositoryError(Error):
549 :class:`Application` does not appear to have a Git repository
550 in the normal location.
552 #: The name of the application that does not have a Git repository.
554 def __init__(self, app):
557 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
559 class DeploymentParseError(Error):
561 Could not parse ``value`` from :term:`versions store`.
563 #: The value that failed to parse.
565 #: The location of the autoinstall that threw this variable.
566 #: This should be set by error handling code when it is available.
568 def __init__(self, value):
571 class NoSuchApplication(Error):
573 You attempted to reference a :class:`Application` named
574 ``app``, which is not recognized by Wizard.
576 #: The name of the application that does not exist.
578 #: The location of the autoinstall that threw this variable.
579 #: This should be set by error handling code when it is availble.
581 def __init__(self, app):
584 class Failure(Error):
586 Represents a failure when performing some double-dispatched operation
587 such as an installation or an upgrade. Failure classes are postfixed
588 with Failure, not Error.
592 class InstallFailure(Error):
593 """Installation failed for unknown reason."""
596 class RecoverableInstallFailure(InstallFailure):
598 Installation failed, but we were able to determine what the
599 error was, and should give the user a second chance if we were
600 running interactively.
602 #: List of the errors that were found.
604 def __init__(self, errors):
607 return """Installation failed due to the following errors: %s""" % ", ".join(self.errors)
609 class UpgradeFailure(Failure):
610 """Upgrade script failed."""
611 #: String details of failure (possibly stdout or stderr output)
613 def __init__(self, details):
614 self.details = details
618 ERROR: Upgrade script failed, details:
622 class UpgradeVerificationFailure(Failure):
623 """Upgrade script passed, but website wasn't accessible afterwards"""
624 #: String details of failure (possibly stdout or stderr output)
626 def __init__(self, details):
627 self.details = details
631 ERROR: Upgrade script passed, but website wasn't accessible afterwards. Details:
635 class BackupFailure(Failure):
636 """Backup script failed."""
637 #: String details of failure
639 def __init__(self, details):
640 self.details = details
644 ERROR: Backup script failed, details:
648 class RestoreFailure(Failure):
649 """Restore script failed."""
650 #: String details of failure
652 def __init__(self, details):
653 self.details = details
657 ERROR: Restore script failed, details: