2 Plumbing object model for representing applications we want to
3 install. This module does the heavy lifting, but you probably
4 want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
5 You'll need to know how to overload the :class:`Application` class
6 and use some of the functions in this module in order to specify
14 from wizard import deploy, util
15 from wizard.app import *
20 import distutils.version
27 from wizard import resolve, scripts, shell, util
30 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
31 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
32 # these are technically deprecated
33 "advancedpoll", "gallery",
38 """Hash table for looking up string application name to instance"""
41 _applications = dict([(n,Application.make(n)) for n in _application_list ])
45 class Application(object):
47 Represents an application, i.e. mediawiki or phpbb.
50 Many of these methods assume a specific working
51 directory; prefer using the corresponding methods
52 in :class:`wizard.deploy.Deployment` and its subclasses.
54 #: String name of the application
56 #: Dictionary of version strings to :class:`ApplicationVersion`.
57 #: See also :meth:`makeVersion`.
59 #: List of files that need to be modified when parametrizing.
60 #: This is a class-wide constant, and should not normally be modified.
61 parametrized_files = []
62 #: Keys that are used in older versions of the application, but
63 #: not for the most recent version.
65 #: Dictionary of variable names to extractor functions. These functions
66 #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
67 #: the variable, or ``None`` if it could not be found.
68 #: See also :func:`filename_regex_extractor`.
70 #: Dictionary of variable names to substitution functions. These functions
71 #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
72 #: that an explicit instance of the variable is released with the generic
73 #: ``WIZARD_*`` constant. See also :func:`filename_regex_substitution`.
75 #: Dictionary of file names to a list of resolutions, which are tuples of
76 #: a conflict marker string and a result list. See :mod:`wizard.resolve`
77 #: for more information.
79 def __init__(self, name):
84 self._substitutions = {}
85 def repository(self, srv_path):
87 Returns the Git repository that would contain this application.
88 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
90 repo = os.path.join(srv_path, self.name + ".git")
91 if not os.path.isdir(repo):
92 repo = os.path.join(srv_path, self.name, ".git")
93 if not os.path.isdir(repo):
94 raise NoRepositoryError(self.name)
96 def makeVersion(self, version):
98 Creates or retrieves the :class:`ApplicationVersion` singleton for the
101 if version not in self.versions:
102 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
103 return self.versions[version]
104 def extract(self, deployment):
106 Extracts wizard variables from a deployment. Default implementation
107 uses :attr:`extractors`.
110 for k,extractor in self.extractors.items():
111 result[k] = extractor(deployment)
113 def parametrize(self, deployment, ref_deployment):
115 Takes a generic source checkout and parametrizes it according to the
116 values of ``deployment``. This function operates on the current
117 working directory. ``deployment`` should **not** be the same as the
118 current working directory. Default implementation uses
119 :attr:`parametrized_files` and a simple search and replace on those
122 variables = ref_deployment.extract()
123 for file in self.parametrized_files:
125 contents = open(file, "r").read()
128 for key, value in variables.items():
129 if value is None: continue
130 contents = contents.replace(key, value)
133 def resolveConflicts(self, deployment):
135 Resolves conflicted files in the current working directory. Returns
136 whether or not all conflicted files were resolved or not. Fully
137 resolved files are added to the index, but no commit is made. The
138 default implementation uses :attr:`resolutions`.
142 for status in sh.eval("git", "ls-files", "--unmerged").splitlines():
143 file = status.split()[-1]
144 if file in self.resolutions:
145 contents = open(file, "r").read()
146 for spec, result in self.resolutions[file]:
147 old_contents = contents
148 contents = resolve.resolve(contents, spec, result)
149 if old_contents != contents:
150 logging.info("Did resolution with spec:\n" + spec)
151 open(file, "w").write(contents)
152 if not resolve.is_conflict(contents):
153 sh.call("git", "add", file)
159 def prepareMerge(self, deployment):
161 Performs various edits to files in the current working directory in
162 order to make a merge go more smoothly. This is usually
163 used to fix botched line-endings. If you add new files,
164 you have to 'git add' them; this is not necessary for edits.
165 By default this is a no-op; subclasses should replace this
166 with useful behavior.
169 def prepareConfig(self, deployment):
171 Takes a deployment and replaces any explicit instances
172 of a configuration variable with generic ``WIZARD_*`` constants.
173 The default implementation uses :attr:`substitutions`, and
174 emits warnings when it encounters keys in :attr:`deprecated_keys`.
176 for key, subst in self.substitutions.items():
177 subs = subst(deployment)
178 if not subs and key not in self.deprecated_keys:
179 logging.warning("No substitutions for %s" % key)
180 def install(self, version, options):
182 Run for 'wizard configure' (and, by proxy, 'wizard install') to
183 configure an application. This assumes that the current working
184 directory is a deployment. (Unlike its kin, this function does not
185 take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should
186 provide an implementation.
188 raise NotImplementedError
189 def upgrade(self, deployment, version, options):
191 Run for 'wizard upgrade' to upgrade database schemas and other
192 non-versioned data in an application after the filesystem has been
193 upgraded. This assumes that the current working directory is the
194 deployment. Subclasses should provide an implementation.
196 raise NotImplementedError
197 def backup(self, deployment, outdir, options):
199 Run for 'wizard backup' and upgrades to backup database schemas
200 and other non-versioned data in an application. ``outdir`` is
201 the directory that backup files should be placed. This assumes
202 that the current working directory is the deployment. Subclasses
203 should provide an implementation, even if it is a no-op.
206 Static user files may not need to be backed up, since in
207 many applications upgrades do not modify static files.
209 raise NotImplementedError
210 def restore(self, deployment, backup_dir, options):
212 Run for 'wizard restore' and failed upgrades to restore database
213 and other non-versioned data to a backed up version. This assumes
214 that the current working directory is the deployment. Subclasses
215 should provide an implementation.
217 raise NotImplementedError
218 def detectVersion(self, deployment):
220 Checks source files to determine the version manually. This assumes
221 that the current working directory is the deployment. Subclasses
222 should provide an implementation.
224 raise NotImplementedError
225 def download(self, version):
227 Returns a URL that can be used to download a tarball of ``version`` of
230 raise NotImplementedError
231 def checkWeb(self, deployment, output=None):
233 Checks if the autoinstall is viewable from the web. To get
234 the HTML source that was retrieved, pass a variable containing
235 an empty list to ``output``; it will be mutated to have its
236 first element be the output. Subclasses should provide an
240 Finding a reasonable heuristic that works across skinning
241 choices can be difficult. We've had reasonable success
242 searching for metadata. Be sure that the standard error
243 page does not contain the features you search for. Try
244 not to depend on pages that are not the main page.
246 raise NotImplementedError
247 def checkConfig(self, deployment):
249 Checks whether or not an autoinstall has been configured/installed
250 for use. Assumes that the current working directory is the deployment.
251 Subclasses should provide an implementation.
253 # XXX: Unfortunately, this doesn't quite work because we package
254 # bogus config files in the -scripts versions of installs. Maybe
255 # we should check a hash or something?
256 raise NotImplementedError
259 """Makes an application, but uses the correct subtype if available."""
261 __import__("wizard.app." + name)
262 return getattr(wizard.app, name).Application(name)
264 return Application(name)
266 class ApplicationVersion(object):
267 """Represents an abstract notion of a version for an application, where
268 ``version`` is a :class:`distutils.version.LooseVersion` and
269 ``application`` is a :class:`Application`."""
270 #: The :class:`distutils.version.LooseVersion` of this instance.
272 #: The :class:`Application` of this instance.
274 def __init__(self, version, application):
275 self.version = version
276 self.application = application
280 Returns the name of the git describe tag for the commit the user is
281 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
283 return "%s-%s" % (self.application, self.version)
285 def scripts_tag(self):
287 Returns the name of the Git tag for this version.
289 end = str(self.version).partition('-scripts')[2].partition('-')[0]
290 return "%s-scripts%s" % (self.pristine_tag, end)
292 def pristine_tag(self):
294 Returns the name of the Git tag for the pristine version corresponding
297 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
298 def __cmp__(self, y):
299 return cmp(self.version, y.version)
303 Parses a line from the :term:`versions store` and return
304 :class:`ApplicationVersion`.
306 Use this only for cases when speed is of primary importance;
307 the data in version is unreliable and when possible, you should
308 prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
309 the autoinstall itself for information.
311 The `value` to parse will vary. For old style installs, it
314 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
316 For new style installs, it will look like::
320 name = value.split("/")[-1]
322 if name.find("-") != -1:
323 app, _, version = name.partition("-")
325 # kind of poor, maybe should error. Generally this
326 # will actually result in a not found error
330 raise DeploymentParseError(value)
331 return ApplicationVersion.make(app, version)
333 def make(app, version):
335 Makes/retrieves a singleton :class:`ApplicationVersion` from
336 a``app`` and ``version`` string.
339 # defer to the application for version creation to enforce
341 return applications()[app].makeVersion(version)
343 raise NoSuchApplication(app)
347 Takes a tree of values (implement using nested lists) and
348 transforms them into regular expressions.
352 >>> expand_re(['a', 'b'])
354 >>> expand_re(['*', ['b', 'c']])
357 if isinstance(val, str):
358 return re.escape(val)
360 return '(?:' + '|'.join(map(expand_re, val)) + ')'
362 def make_extractors(seed):
364 Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into
365 extractor functions (which take a :class:`wizard.deploy.Deployment`
366 and return the value of the second subpattern of ``regex`` when matched
367 with the contents of ``file``).
369 return util.dictmap(lambda a: filename_regex_extractor(*a), seed)
371 def make_substitutions(seed):
373 Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into substitution
374 functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
375 of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
377 return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)
379 # The following two functions are *highly* functional, and I recommend
380 # not touching them unless you know what you're doing.
382 def filename_regex_extractor(file, regex):
384 .. highlight:: haskell
386 Given a relative file name ``file``, a regular expression ``regex``, and a
387 :class:`wizard.deploy.Deployment` extracts a value out of the file in that
388 deployment. This function is curried, so you pass just ``file`` and
389 ``regex``, and then pass ``deployment`` to the resulting function.
391 Its Haskell-style type signature would be::
393 Filename -> Regex -> (Deployment -> String)
395 The regular expression requires a very specific form, essentially ``()()()``
396 (with the second subgroup being the value to extract). These enables
397 the regular expression to be used equivalently with filename
399 .. highlight:: python
401 For convenience purposes, we also accept ``[Filename]``, in which case
402 we use the first entry (index 0). Passing an empty list is invalid.
404 >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
405 >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
406 >>> f(deploy.Deployment("."))
408 >>> os.unlink("test-settings.extractor.ini")
411 The first application of ``regex`` and ``file`` is normally performed
412 at compile-time inside a submodule; the second application is
413 performed at runtime.
415 if not isinstance(file, str):
419 contents = deployment.read(file) # cached
422 match = regex.search(contents)
423 if not match: return None
424 # assumes that the second match is the one we want.
425 return match.group(2)
428 def filename_regex_substitution(key, files, regex):
430 .. highlight:: haskell
432 Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
433 regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
434 performs a substitution of the second subpattern of ``regex``
435 with ``key``. Returns the number of replacements made. This function
436 is curried, so you pass just ``key``, ``files`` and ``regex``, and
437 then pass ``deployment`` to the resulting function.
439 Its Haskell-style type signature would be::
441 Key -> ([File], Regex) -> (Deployment -> IO Int)
443 .. highlight:: python
445 For convenience purposes, we also accept ``Filename``, in which case it is treated
446 as a single item list.
448 >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
449 >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
450 >>> f(deploy.Deployment("."))
452 >>> print open("test-settings.substitution.ini", "r").read()
453 config_var = WIZARD_KEY
454 >>> os.unlink("test-settings.substitution.ini")
456 if isinstance(files, str):
459 base = deployment.location
462 file = os.path.join(base, file)
464 contents = open(file, "r").read()
465 contents, n = regex.subn("\\1" + key + "\\3", contents)
467 open(file, "w").write(contents)
473 # XXX: rename to show that it's mysql specific
474 def backup_database(outdir, deployment):
476 Generic database backup function for MySQL. Assumes that ``WIZARD_DBNAME``
477 is extractable, and that :func:`wizard.scripts.get_sql_credentials`
481 outfile = os.path.join(outdir, "db.sql")
483 sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment))
484 sh.call("gzip", "--best", outfile)
485 except shell.CallError as e:
486 shutil.rmtree(outdir)
487 raise BackupFailure(e.stderr)
489 def restore_database(backup_dir, deployment):
491 Generic database restoration function for MySQL. See :func:`backup_database`
492 for the assumptions that we make.
495 if not os.path.exists(backup_dir):
496 raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
497 sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
498 sh.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
500 sh.call("mysql", *get_mysql_args(deployment), stdin=sql)
503 def get_mysql_args(d):
505 Extracts arguments that would be passed to the command line mysql utility
508 # XXX: add support for getting these out of options
510 if 'WIZARD_DBNAME' not in vars:
511 raise app.BackupFailure("Could not determine database name")
512 triplet = scripts.get_sql_credentials(vars)
514 if triplet is not None:
515 server, user, password = triplet
516 args += ["-h", server, "-u", user, "-p" + password]
517 name = shlex.split(vars['WIZARD_DBNAME'])[0]
521 class Error(wizard.Error):
522 """Generic error class for this module."""
525 class RecoverableFailure(Error):
527 The installer failed, but we were able to determine what the
528 error was, and should give the user a second chance if we were
529 running interactively.
531 #: List of the errors that were found.
533 def __init__(self, errors):
536 return """Installation failed due to the following errors: %s""" % ", ".join(self.errors)
538 class NoRepositoryError(Error):
540 :class:`Application` does not appear to have a Git repository
541 in the normal location.
543 #: The name of the application that does not have a Git repository.
545 def __init__(self, app):
548 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
550 class DeploymentParseError(Error):
552 Could not parse ``value`` from :term:`versions store`.
554 #: The value that failed to parse.
556 #: The location of the autoinstall that threw this variable.
557 #: This should be set by error handling code when it is available.
559 def __init__(self, value):
562 class NoSuchApplication(Error):
564 You attempted to reference a :class:`Application` named
565 ``app``, which is not recognized by Wizard.
567 #: The name of the application that does not exist.
569 #: The location of the autoinstall that threw this variable.
570 #: This should be set by error handling code when it is availble.
572 def __init__(self, app):
575 class UpgradeFailure(Error):
576 """Upgrade script failed."""
577 #: String details of failure (possibly stdout or stderr output)
579 def __init__(self, details):
580 self.details = details
584 ERROR: Upgrade script failed, details:
588 class UpgradeVerificationFailure(Error):
589 """Upgrade script passed, but website wasn't accessible afterwards"""
590 #: String details of failure (possibly stdout or stderr output)
592 def __init__(self, details):
593 self.details = details
597 ERROR: Upgrade script passed, but website wasn't accessible afterwards. Details:
601 class BackupFailure(Error):
602 """Backup script failed."""
603 #: String details of failure
605 def __init__(self, details):
606 self.details = details
610 ERROR: Backup script failed, details:
614 class RestoreFailure(Error):
615 """Restore script failed."""
616 #: String details of failure
618 def __init__(self, details):
619 self.details = details
623 ERROR: Restore script failed, details: