2 Plumbing object model for representing applications we want to
3 install. This module does the heavy lifting, but you probably
4 want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
5 You'll need to know how to overload the :class:`Application` class
6 and use some of the functions in this module in order to specify
14 from wizard import deploy, util
15 from wizard.app import *
20 import distutils.version
27 from wizard import resolve, scripts, shell, util
30 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
31 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
32 # these are technically deprecated
33 "advancedpoll", "gallery",
38 """Hash table for looking up string application name to instance"""
41 _applications = dict([(n,Application.make(n)) for n in _application_list ])
45 class Application(object):
47 Represents an application, i.e. mediawiki or phpbb.
50 Many of these methods assume a specific working
51 directory; prefer using the corresponding methods
52 in :class:`wizard.deploy.Deployment` and its subclasses.
54 #: String name of the application
56 #: Dictionary of version strings to :class:`ApplicationVersion`.
57 #: See also :meth:`makeVersion`.
59 #: List of files that need to be modified when parametrizing.
60 #: This is a class-wide constant, and should not normally be modified.
61 parametrized_files = []
62 #: Keys that are used in older versions of the application, but
63 #: not for the most recent version.
65 #: Dictionary of variable names to extractor functions. These functions
66 #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
67 #: the variable, or ``None`` if it could not be found.
68 #: See also :func:`filename_regex_extractor`.
70 #: Dictionary of variable names to substitution functions. These functions
71 #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
72 #: that an explicit instance of the variable is released with the generic
73 #: ``WIZARD_*`` constant. See also :func:`filename_regex_substitution`.
75 #: Dictionary of file names to a list of resolutions, which are tuples of
76 #: a conflict marker string and a result list. See :mod:`wizard.resolve`
77 #: for more information.
79 #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
80 #: this application requires.
82 def __init__(self, name):
87 self._substitutions = {}
88 def repository(self, srv_path):
90 Returns the Git repository that would contain this application.
91 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
93 repo = os.path.join(srv_path, self.name + ".git")
94 if not os.path.isdir(repo):
95 repo = os.path.join(srv_path, self.name, ".git")
96 if not os.path.isdir(repo):
97 raise NoRepositoryError(self.name)
99 def makeVersion(self, version):
101 Creates or retrieves the :class:`ApplicationVersion` singleton for the
104 if version not in self.versions:
105 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
106 return self.versions[version]
107 def extract(self, deployment):
109 Extracts wizard variables from a deployment. Default implementation
110 uses :attr:`extractors`.
113 for k,extractor in self.extractors.items():
114 result[k] = extractor(deployment)
116 def parametrize(self, deployment, ref_deployment):
118 Takes a generic source checkout and parametrizes it according to the
119 values of ``deployment``. This function operates on the current
120 working directory. ``deployment`` should **not** be the same as the
121 current working directory. Default implementation uses
122 :attr:`parametrized_files` and a simple search and replace on those
125 variables = ref_deployment.extract()
126 for file in self.parametrized_files:
128 contents = open(file, "r").read()
131 for key, value in variables.items():
132 if value is None: continue
133 contents = contents.replace(key, value)
136 def resolveConflicts(self, deployment):
138 Resolves conflicted files in the current working directory. Returns
139 whether or not all conflicted files were resolved or not. Fully
140 resolved files are added to the index, but no commit is made. The
141 default implementation uses :attr:`resolutions`.
145 for status in sh.eval("git", "ls-files", "--unmerged").splitlines():
146 file = status.split()[-1]
147 if file in self.resolutions:
148 contents = open(file, "r").read()
149 for spec, result in self.resolutions[file]:
150 old_contents = contents
151 contents = resolve.resolve(contents, spec, result)
152 if old_contents != contents:
153 logging.info("Did resolution with spec:\n" + spec)
154 open(file, "w").write(contents)
155 if not resolve.is_conflict(contents):
156 sh.call("git", "add", file)
162 def prepareMerge(self, deployment):
164 Performs various edits to files in the current working directory in
165 order to make a merge go more smoothly. This is usually
166 used to fix botched line-endings. If you add new files,
167 you have to 'git add' them; this is not necessary for edits.
168 By default this is a no-op; subclasses should replace this
169 with useful behavior.
172 def prepareConfig(self, deployment):
174 Takes a deployment and replaces any explicit instances
175 of a configuration variable with generic ``WIZARD_*`` constants.
176 The default implementation uses :attr:`substitutions`, and
177 emits warnings when it encounters keys in :attr:`deprecated_keys`.
179 for key, subst in self.substitutions.items():
180 subs = subst(deployment)
181 if not subs and key not in self.deprecated_keys:
182 logging.warning("No substitutions for %s" % key)
183 def install(self, version, options):
185 Run for 'wizard configure' (and, by proxy, 'wizard install') to
186 configure an application. This assumes that the current working
187 directory is a deployment. (Unlike its kin, this function does not
188 take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should
189 provide an implementation.
191 raise NotImplementedError
192 def upgrade(self, deployment, version, options):
194 Run for 'wizard upgrade' to upgrade database schemas and other
195 non-versioned data in an application after the filesystem has been
196 upgraded. This assumes that the current working directory is the
197 deployment. Subclasses should provide an implementation.
199 raise NotImplementedError
200 def backup(self, deployment, outdir, options):
202 Run for 'wizard backup' and upgrades to backup database schemas
203 and other non-versioned data in an application. ``outdir`` is
204 the directory that backup files should be placed. This assumes
205 that the current working directory is the deployment. Subclasses
206 should provide an implementation, even if it is a no-op.
209 Static user files may not need to be backed up, since in
210 many applications upgrades do not modify static files.
212 raise NotImplementedError
213 def restore(self, deployment, backup_dir, options):
215 Run for 'wizard restore' and failed upgrades to restore database
216 and other non-versioned data to a backed up version. This assumes
217 that the current working directory is the deployment. Subclasses
218 should provide an implementation.
220 raise NotImplementedError
221 def detectVersion(self, deployment):
223 Checks source files to determine the version manually. This assumes
224 that the current working directory is the deployment. Subclasses
225 should provide an implementation.
227 raise NotImplementedError
228 def download(self, version):
230 Returns a URL that can be used to download a tarball of ``version`` of
233 raise NotImplementedError
234 def checkWeb(self, deployment, output=None):
236 Checks if the autoinstall is viewable from the web. To get
237 the HTML source that was retrieved, pass a variable containing
238 an empty list to ``output``; it will be mutated to have its
239 first element be the output. Subclasses should provide an
243 Finding a reasonable heuristic that works across skinning
244 choices can be difficult. We've had reasonable success
245 searching for metadata. Be sure that the standard error
246 page does not contain the features you search for. Try
247 not to depend on pages that are not the main page.
249 raise NotImplementedError
250 def checkConfig(self, deployment):
252 Checks whether or not an autoinstall has been configured/installed
253 for use. Assumes that the current working directory is the deployment.
254 Subclasses should provide an implementation.
256 # XXX: Unfortunately, this doesn't quite work because we package
257 # bogus config files in the -scripts versions of installs. Maybe
258 # we should check a hash or something?
259 raise NotImplementedError
262 """Makes an application, but uses the correct subtype if available."""
264 __import__("wizard.app." + name)
265 return getattr(wizard.app, name).Application(name)
267 return Application(name)
269 class ApplicationVersion(object):
270 """Represents an abstract notion of a version for an application, where
271 ``version`` is a :class:`distutils.version.LooseVersion` and
272 ``application`` is a :class:`Application`."""
273 #: The :class:`distutils.version.LooseVersion` of this instance.
275 #: The :class:`Application` of this instance.
277 def __init__(self, version, application):
278 self.version = version
279 self.application = application
283 Returns the name of the git describe tag for the commit the user is
284 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
286 return "%s-%s" % (self.application, self.version)
288 def scripts_tag(self):
290 Returns the name of the Git tag for this version.
292 end = str(self.version).partition('-scripts')[2].partition('-')[0]
293 return "%s-scripts%s" % (self.pristine_tag, end)
295 def pristine_tag(self):
297 Returns the name of the Git tag for the pristine version corresponding
300 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
301 def __cmp__(self, y):
302 return cmp(self.version, y.version)
306 Parses a line from the :term:`versions store` and return
307 :class:`ApplicationVersion`.
309 Use this only for cases when speed is of primary importance;
310 the data in version is unreliable and when possible, you should
311 prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
312 the autoinstall itself for information.
314 The `value` to parse will vary. For old style installs, it
317 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
319 For new style installs, it will look like::
323 name = value.split("/")[-1]
325 if name.find("-") != -1:
326 app, _, version = name.partition("-")
328 # kind of poor, maybe should error. Generally this
329 # will actually result in a not found error
333 raise DeploymentParseError(value)
334 return ApplicationVersion.make(app, version)
336 def make(app, version):
338 Makes/retrieves a singleton :class:`ApplicationVersion` from
339 a``app`` and ``version`` string.
342 # defer to the application for version creation to enforce
344 return applications()[app].makeVersion(version)
346 raise NoSuchApplication(app)
350 Takes a tree of values (implement using nested lists) and
351 transforms them into regular expressions.
355 >>> expand_re(['a', 'b'])
357 >>> expand_re(['*', ['b', 'c']])
360 if isinstance(val, str):
361 return re.escape(val)
363 return '(?:' + '|'.join(map(expand_re, val)) + ')'
365 def make_extractors(seed):
367 Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into
368 extractor functions (which take a :class:`wizard.deploy.Deployment`
369 and return the value of the second subpattern of ``regex`` when matched
370 with the contents of ``file``).
372 return util.dictmap(lambda a: filename_regex_extractor(*a), seed)
374 def make_substitutions(seed):
376 Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into substitution
377 functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
378 of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
380 return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)
382 # The following two functions are *highly* functional, and I recommend
383 # not touching them unless you know what you're doing.
385 def filename_regex_extractor(file, regex):
387 .. highlight:: haskell
389 Given a relative file name ``file``, a regular expression ``regex``, and a
390 :class:`wizard.deploy.Deployment` extracts a value out of the file in that
391 deployment. This function is curried, so you pass just ``file`` and
392 ``regex``, and then pass ``deployment`` to the resulting function.
394 Its Haskell-style type signature would be::
396 Filename -> Regex -> (Deployment -> String)
398 The regular expression requires a very specific form, essentially ``()()()``
399 (with the second subgroup being the value to extract). These enables
400 the regular expression to be used equivalently with filename
402 .. highlight:: python
404 For convenience purposes, we also accept ``[Filename]``, in which case
405 we use the first entry (index 0). Passing an empty list is invalid.
407 >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
408 >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
409 >>> f(deploy.Deployment("."))
411 >>> os.unlink("test-settings.extractor.ini")
414 The first application of ``regex`` and ``file`` is normally performed
415 at compile-time inside a submodule; the second application is
416 performed at runtime.
418 if not isinstance(file, str):
422 contents = deployment.read(file) # cached
425 match = regex.search(contents)
426 if not match: return None
427 # assumes that the second match is the one we want.
428 return match.group(2)
431 def filename_regex_substitution(key, files, regex):
433 .. highlight:: haskell
435 Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
436 regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
437 performs a substitution of the second subpattern of ``regex``
438 with ``key``. Returns the number of replacements made. This function
439 is curried, so you pass just ``key``, ``files`` and ``regex``, and
440 then pass ``deployment`` to the resulting function.
442 Its Haskell-style type signature would be::
444 Key -> ([File], Regex) -> (Deployment -> IO Int)
446 .. highlight:: python
448 For convenience purposes, we also accept ``Filename``, in which case it is treated
449 as a single item list.
451 >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
452 >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
453 >>> f(deploy.Deployment("."))
455 >>> print open("test-settings.substitution.ini", "r").read()
456 config_var = WIZARD_KEY
457 >>> os.unlink("test-settings.substitution.ini")
459 if isinstance(files, str):
462 base = deployment.location
465 file = os.path.join(base, file)
467 contents = open(file, "r").read()
468 contents, n = regex.subn("\\1" + key + "\\3", contents)
470 open(file, "w").write(contents)
476 # XXX: rename to show that it's mysql specific
477 def backup_database(outdir, deployment):
479 Generic database backup function for MySQL. Assumes that ``WIZARD_DBNAME``
480 is extractable, and that :func:`wizard.scripts.get_sql_credentials`
484 outfile = os.path.join(outdir, "db.sql")
486 sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment))
487 sh.call("gzip", "--best", outfile)
488 except shell.CallError as e:
489 shutil.rmtree(outdir)
490 raise BackupFailure(e.stderr)
492 def restore_database(backup_dir, deployment):
494 Generic database restoration function for MySQL. See :func:`backup_database`
495 for the assumptions that we make.
498 if not os.path.exists(backup_dir):
499 raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
500 sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
501 sh.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
503 sh.call("mysql", *get_mysql_args(deployment), stdin=sql)
506 def get_mysql_args(d):
508 Extracts arguments that would be passed to the command line mysql utility
511 # XXX: add support for getting these out of options
513 if 'WIZARD_DBNAME' not in vars:
514 raise app.BackupFailure("Could not determine database name")
515 triplet = scripts.get_sql_credentials(vars)
517 if triplet is not None:
518 server, user, password = triplet
519 args += ["-h", server, "-u", user, "-p" + password]
520 name = shlex.split(vars['WIZARD_DBNAME'])[0]
524 class Error(wizard.Error):
525 """Generic error class for this module."""
528 class RecoverableFailure(Error):
530 The installer failed, but we were able to determine what the
531 error was, and should give the user a second chance if we were
532 running interactively.
534 #: List of the errors that were found.
536 def __init__(self, errors):
539 return """Installation failed due to the following errors: %s""" % ", ".join(self.errors)
541 class NoRepositoryError(Error):
543 :class:`Application` does not appear to have a Git repository
544 in the normal location.
546 #: The name of the application that does not have a Git repository.
548 def __init__(self, app):
551 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
553 class DeploymentParseError(Error):
555 Could not parse ``value`` from :term:`versions store`.
557 #: The value that failed to parse.
559 #: The location of the autoinstall that threw this variable.
560 #: This should be set by error handling code when it is available.
562 def __init__(self, value):
565 class NoSuchApplication(Error):
567 You attempted to reference a :class:`Application` named
568 ``app``, which is not recognized by Wizard.
570 #: The name of the application that does not exist.
572 #: The location of the autoinstall that threw this variable.
573 #: This should be set by error handling code when it is availble.
575 def __init__(self, app):
578 class UpgradeFailure(Error):
579 """Upgrade script failed."""
580 #: String details of failure (possibly stdout or stderr output)
582 def __init__(self, details):
583 self.details = details
587 ERROR: Upgrade script failed, details:
591 class UpgradeVerificationFailure(Error):
592 """Upgrade script passed, but website wasn't accessible afterwards"""
593 #: String details of failure (possibly stdout or stderr output)
595 def __init__(self, details):
596 self.details = details
600 ERROR: Upgrade script passed, but website wasn't accessible afterwards. Details:
604 class BackupFailure(Error):
605 """Backup script failed."""
606 #: String details of failure
608 def __init__(self, details):
609 self.details = details
613 ERROR: Backup script failed, details:
617 class RestoreFailure(Error):
618 """Restore script failed."""
619 #: String details of failure
621 def __init__(self, details):
622 self.details = details
626 ERROR: Restore script failed, details: