2 Plumbing object model for representing applications we want to
3 install. This module does the heavy lifting, but you probably
4 want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
5 You'll need to know how to overload the :class:`Application` class
6 and use some of the functions in this module in order to specify
14 from wizard import deploy, util
15 from wizard.app import *
20 import distutils.version
25 from wizard import scripts, shell, util
28 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
29 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
30 # these are technically deprecated
31 "advancedpoll", "gallery",
36 """Hash table for looking up string application name to instance"""
39 _applications = dict([(n,Application.make(n)) for n in _application_list ])
43 class Application(object):
45 Represents an application, i.e. mediawiki or phpbb.
48 Many of these methods assume a specific working
49 directory; prefer using the corresponding methods
50 in :class:`wizard.deploy.Deployment` and its subclasses.
52 #: String name of the application
54 #: Dictionary of version strings to :class:`ApplicationVersion`.
55 #: See also :meth:`makeVersion`.
57 #: List of files that need to be modified when parametrizing.
58 #: This is a class-wide constant, and should not normally be modified.
59 parametrized_files = []
60 #: Keys that are used in older versions of the application, but
61 #: not for the most recent version.
63 #: Dictionary of variable names to extractor functions. These functions
64 #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
65 #: the variable, or ``None`` if it could not be found.
66 #: See also :func:`filename_regex_extractor`.
68 #: Dictionary of variable names to substitution functions. These functions
69 #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
70 #: that an explicit instance of the variable is released with the generic
71 #: ``WIZARD_*`` constant. See also :func:`filename_regex_substitution`.
73 #: Dictionary of file names to a list of resolutions, which are tuples of
74 #: a conflict marker string and a result list. See :mod:`wizard.resolve`
75 #: for more information.
77 def __init__(self, name):
82 self._substitutions = {}
83 def repository(self, srv_path):
85 Returns the Git repository that would contain this application.
86 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
88 repo = os.path.join(srv_path, self.name + ".git")
89 if not os.path.isdir(repo):
90 repo = os.path.join(srv_path, self.name, ".git")
91 if not os.path.isdir(repo):
92 raise NoRepositoryError(self.name)
94 def makeVersion(self, version):
96 Creates or retrieves the :class:`ApplicationVersion` singleton for the
99 if version not in self.versions:
100 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
101 return self.versions[version]
102 def extract(self, deployment):
104 Extracts wizard variables from a deployment. Default implementation
105 uses :attr:`extractors`.
108 for k,extractor in self.extractors.items():
109 result[k] = extractor(deployment)
111 def parametrize(self, deployment, ref_deployment):
113 Takes a generic source checkout and parametrizes it according to the
114 values of ``deployment``. This function operates on the current
115 working directory. ``deployment`` should **not** be the same as the
116 current working directory. Default implementation uses
117 :attr:`parametrized_files` and a simple search and replace on those
120 variables = ref_deployment.extract()
121 for file in self.parametrized_files:
123 contents = open(file, "r").read()
126 for key, value in variables.items():
127 if value is None: continue
128 contents = contents.replace(key, value)
131 def resolveConflicts(self, deployment):
133 Resolves conflicted files in the current working directory. Returns
134 whether or not all conflicted files were resolved or not. Fully
135 resolved files are added to the index, but no commit is made. The
136 default implementation uses :attr:`resolutions`.
140 for status in sh.eval("git", "ls-files", "--unmerged").splitlines():
141 file = status.split()[-1]
142 if file in self.resolutions:
143 contents = open(file, "r").read()
144 for spec, result in self.resolutions[file]:
145 old_contents = contents
146 contents = resolve.resolve(contents, spec, result)
147 if old_contents != contents:
148 logging.info("Did resolution with spec:\n" + spec)
149 open(file, "w").write(contents)
150 if not resolve.is_conflict(contents):
151 sh.call("git", "add", file)
157 def prepareMerge(self, deployment):
159 Performs various edits to files in the current working directory in
160 order to make a merge go more smoothly. This is usually
161 used to fix botched line-endings. If you add new files,
162 you have to 'git add' them; this is not necessary for edits.
163 By default this is a no-op; subclasses should replace this
164 with useful behavior.
167 def prepareConfig(self, deployment):
169 Takes a deployment and replaces any explicit instances
170 of a configuration variable with generic ``WIZARD_*`` constants.
171 The default implementation uses :attr:`substitutions`, and
172 emits warnings when it encounters keys in :attr:`deprecated_keys`.
174 for key, subst in self.substitutions.items():
175 subs = subst(deployment)
176 if not subs and key not in self.deprecated_keys:
177 logging.warning("No substitutions for %s" % key)
178 def install(self, version, options):
180 Run for 'wizard configure' (and, by proxy, 'wizard install') to
181 configure an application. This assumes that the current working
182 directory is a deployment. (Unlike its kin, this function does not
183 take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should
184 provide an implementation.
187 def upgrade(self, deployment, version, options):
189 Run for 'wizard upgrade' to upgrade database schemas and other
190 non-versioned data in an application after the filesystem has been
191 upgraded. This assumes that the current working directory is the
192 deployment. Subclasses should provide an implementation.
195 def backup(self, deployment, outdir, options):
197 Run for 'wizard backup' and upgrades to backup database schemas
198 and other non-versioned data in an application. ``outdir`` is
199 the directory that backup files should be placed. This assumes
200 that the current working directory is the deployment. Subclasses
201 should provide an implementation, even if it is a no-op.
204 Static user files may not need to be backed up, since in
205 many applications upgrades do not modify static files.
208 def restore(self, deployment, backup_dir, options):
210 Run for 'wizard restore' and failed upgrades to restore database
211 and other non-versioned data to a backed up version. This assumes
212 that the current working directory is the deployment. Subclasses
213 should provide an implementation.
216 def detectVersion(self, deployment):
218 Checks source files to determine the version manually. This assumes
219 that the current working directory is the deployment. Subclasses
220 should provide an implementation.
223 def checkWeb(self, deployment, output=None):
225 Checks if the autoinstall is viewable from the web. To get
226 the HTML source that was retrieved, pass a variable containing
227 an empty list to ``output``; it will be mutated to have its
228 first element be the output. Subclasses should provide an
232 Finding a reasonable heuristic that works across skinning
233 choices can be difficult. We've had reasonable success
234 searching for metadata. Be sure that the standard error
235 page does not contain the features you search for. Try
236 not to depend on pages that are not the main page.
239 def checkConfig(self, deployment):
241 Checks whether or not an autoinstall has been configured/installed
242 for use. Assumes that the current working directory is the deployment.
243 Subclasses should provide an implementation.
248 """Makes an application, but uses the correct subtype if available."""
250 __import__("wizard.app." + name)
251 return getattr(wizard.app, name).Application(name)
253 return Application(name)
255 class ApplicationVersion(object):
256 """Represents an abstract notion of a version for an application, where
257 ``version`` is a :class:`distutils.version.LooseVersion` and
258 ``application`` is a :class:`Application`."""
259 #: The :class:`distutils.version.LooseVersion` of this instance.
261 #: The :class:`Application` of this instance.
263 def __init__(self, version, application):
264 self.version = version
265 self.application = application
269 Returns the name of the git describe tag for the commit the user is
270 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
272 return "%s-%s" % (self.application, self.version)
274 def scripts_tag(self):
276 Returns the name of the Git tag for this version.
278 end = str(self.version).partition('-scripts')[2].partition('-')[0]
279 return "%s-scripts%s" % (self.pristine_tag, end)
281 def pristine_tag(self):
283 Returns the name of the Git tag for the pristine version corresponding
286 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
287 def __cmp__(self, y):
288 return cmp(self.version, y.version)
292 Parses a line from the :term:`versions store` and return
293 :class:`ApplicationVersion`.
295 Use this only for cases when speed is of primary importance;
296 the data in version is unreliable and when possible, you should
297 prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
298 the autoinstall itself for information.
300 The `value` to parse will vary. For old style installs, it
303 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
305 For new style installs, it will look like::
309 name = value.split("/")[-1]
311 if name.find("-") != -1:
312 app, _, version = name.partition("-")
314 # kind of poor, maybe should error. Generally this
315 # will actually result in a not found error
319 raise DeploymentParseError(value)
320 return ApplicationVersion.make(app, version)
322 def make(app, version):
324 Makes/retrieves a singleton :class:`ApplicationVersion` from
325 a``app`` and ``version`` string.
328 # defer to the application for version creation to enforce
330 return applications()[app].makeVersion(version)
332 raise NoSuchApplication(app)
336 Takes a tree of values (implement using nested lists) and
337 transforms them into regular expressions.
341 >>> expand_re(['a', 'b'])
343 >>> expand_re(['*', ['b', 'c']])
346 if isinstance(val, str):
347 return re.escape(val)
349 return '(?:' + '|'.join(map(expand_re, val)) + ')'
351 def make_extractors(seed):
353 Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into
354 extractor functions (which take a :class:`wizard.deploy.Deployment`
355 and return the value of the second subpattern of ``regex`` when matched
356 with the contents of ``file``).
358 return util.dictmap(lambda a: filename_regex_extractor(*a), seed)
360 def make_substitutions(seed):
362 Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into substitution
363 functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
364 of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
366 return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)
368 # The following two functions are *highly* functional, and I recommend
369 # not touching them unless you know what you're doing.
371 def filename_regex_extractor(file, regex):
373 .. highlight:: haskell
375 Given a relative file name ``file``, a regular expression ``regex``, and a
376 :class:`wizard.deploy.Deployment` extracts a value out of the file in that
377 deployment. This function is curried, so you pass just ``file`` and
378 ``regex``, and then pass ``deployment`` to the resulting function.
380 Its Haskell-style type signature would be::
382 Filename -> Regex -> (Deployment -> String)
384 The regular expression requires a very specific form, essentially ``()()()``
385 (with the second subgroup being the value to extract). These enables
386 the regular expression to be used equivalently with filename
388 .. highlight:: python
390 For convenience purposes, we also accept ``[Filename]``, in which case
391 we use the first entry (index 0). Passing an empty list is invalid.
393 >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
394 >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
395 >>> f(deploy.Deployment("."))
397 >>> os.unlink("test-settings.extractor.ini")
400 The first application of ``regex`` and ``file`` is normally performed
401 at compile-time inside a submodule; the second application is
402 performed at runtime.
404 if not isinstance(file, str):
408 contents = deployment.read(file) # cached
411 match = regex.search(contents)
412 if not match: return None
413 # assumes that the second match is the one we want.
414 return match.group(2)
417 def filename_regex_substitution(key, files, regex):
419 .. highlight:: haskell
421 Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
422 regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
423 performs a substitution of the second subpattern of ``regex``
424 with ``key``. Returns the number of replacements made. This function
425 is curried, so you pass just ``key``, ``files`` and ``regex``, and
426 then pass ``deployment`` to the resulting function.
428 Its Haskell-style type signature would be::
430 Key -> ([File], Regex) -> (Deployment -> IO Int)
432 .. highlight:: python
434 For convenience purposes, we also accept ``Filename``, in which case it is treated
435 as a single item list.
437 >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
438 >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
439 >>> f(deploy.Deployment("."))
441 >>> print open("test-settings.substitution.ini", "r").read()
442 config_var = WIZARD_KEY
443 >>> os.unlink("test-settings.substitution.ini")
445 if isinstance(files, str):
448 base = deployment.location
451 file = os.path.join(base, file)
453 contents = open(file, "r").read()
454 contents, n = regex.subn("\\1" + key + "\\3", contents)
456 open(file, "w").write(contents)
462 def backup_database(outdir, deployment):
464 Generic database backup function. Assumes that ``WIZARD_DBNAME``
465 is extractable, and that :func:`wizard.scripts.get_sql_credentials`
469 outfile = os.path.join(outdir, "db.sql")
471 sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment))
472 sh.call("gzip", "--best", outfile)
473 except shell.CallError as e:
474 shutil.rmtree(outdir)
475 raise BackupFailure(e.stderr)
477 def restore_database(backup_dir, deployment):
479 if not os.path.exists(backup_dir):
480 raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
481 sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
482 sh.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
484 sh.call("mysql", *get_mysql_args(deployment), stdin=sql)
487 def get_mysql_args(d):
488 # XXX: add support for getting these out of options
490 if 'WIZARD_DBNAME' not in vars:
491 raise app.BackupFailure("Could not determine database name")
492 triplet = scripts.get_sql_credentials(vars)
494 if triplet is not None:
495 server, user, password = triplet
496 args += ["-h", server, "-u", user, "-p" + password]
497 name = shlex.split(vars['WIZARD_DBNAME'])[0]
501 class Error(wizard.Error):
502 """Generic error class for this module."""
505 class NoRepositoryError(Error):
507 :class:`Application` does not appear to have a Git repository
508 in the normal location.
510 #: The name of the application that does not have a Git repository.
512 def __init__(self, app):
515 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
517 class DeploymentParseError(Error):
519 Could not parse ``value`` from :term:`versions store`.
521 #: The value that failed to parse.
523 #: The location of the autoinstall that threw this variable.
524 #: This should be set by error handling code when it is available.
526 def __init__(self, value):
529 class NoSuchApplication(Error):
531 You attempted to reference a :class:`Application` named
532 ``app``, which is not recognized by Wizard.
534 #: The name of the application that does not exist.
536 #: The location of the autoinstall that threw this variable.
537 #: This should be set by error handling code when it is availble.
539 def __init__(self, app):
542 class UpgradeFailure(Error):
543 """Upgrade script failed."""
544 #: String details of failure (possibly stdout or stderr output)
546 def __init__(self, details):
547 self.details = details
551 ERROR: Upgrade script failed, details:
555 class UpgradeVerificationFailure(Error):
556 """Upgrade script passed, but website wasn't accessible afterwards"""
557 #: String details of failure (possibly stdout or stderr output)
559 def __init__(self, details):
560 self.details = details
564 ERROR: Upgrade script passed, but website wasn't accessible afterwards. Details:
568 class BackupFailure(Error):
569 """Backup script failed."""
570 #: String details of failure
572 def __init__(self, details):
573 self.details = details
577 ERROR: Backup script failed, details:
581 class RestoreFailure(Error):
582 """Restore script failed."""
583 #: String details of failure
585 def __init__(self, details):
586 self.details = details
590 ERROR: Restore script failed, details: