2 Plumbing object model for representing applications we want to
3 install. This module does the heavy lifting, but you probably
4 want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
5 You'll need to know how to overload the :class:`Application` class
6 and use some of the functions in this module in order to specify
9 To specify custom applications as plugins, add the following ``entry_points``
13 yourappname = your.module:Application
14 otherappname = your.other.module:Application
18 Wizard will complain loudly if ``yourappname`` conflicts with an
19 application name defined by someone else.
21 There are some submodules for programming languages that define common
22 functions and data that may be used by applications in that language. See:
24 * :mod:`wizard.app.php`
31 from wizard import deploy, util
32 from wizard.app import *
38 import distutils.version
52 from wizard import plugin, resolve, shell, sql, util
56 """Hash table for looking up string application name to instance"""
59 _applications = dict()
60 for dist in pkg_resources.working_set:
61 for appname, entry in dist.get_entry_map("wizard.app").items():
62 if appname in _applications:
63 newname = dist.key + ":" + appname
64 if newname in _applications:
65 raise Exception("Unrecoverable application name conflict for %s from %s", appname, dist.key)
66 logging.warning("Could not overwrite %s, used %s instead", appname, newname)
68 appclass = entry.load()
69 _applications[appname] = appclass(appname)
71 for entry in pkg_resources.iter_entry_points("wizard.dummy_apps"):
74 for appname in dummy_apps:
75 # a dummy app that already exists is not a fatal error
76 if appname in _applications:
78 _applications[appname] = Application(appname)
81 def getApplication(appname):
82 """Retrieves application instance given a name"""
84 return applications()[appname]
86 raise NoSuchApplication(appname)
88 class Application(object):
90 Represents an application, i.e. mediawiki or phpbb.
93 Many of these methods assume a specific working
94 directory; prefer using the corresponding methods
95 in :class:`wizard.deploy.Deployment` and its subclasses.
97 #: String name of the application
99 #: Dictionary of version strings to :class:`ApplicationVersion`.
100 #: See also :meth:`makeVersion`.
102 #: List of files that need to be modified when parametrizing.
103 #: This is a class-wide constant, and should not normally be modified.
104 parametrized_files = []
105 #: Keys that are used in older versions of the application, but
106 #: not for the most recent version.
107 deprecated_keys = set()
108 #: Keys that we can simply generate random strings for if they're missing
110 #: Values that are not sufficiently random for a random key. This can
111 #: include default values for a random configuration option,
112 random_blacklist = set()
113 #: Dictionary of variable names to extractor functions. These functions
114 #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
115 #: the variable, or ``None`` if it could not be found.
116 #: See also :func:`filename_regex_extractor`.
118 #: Dictionary of variable names to substitution functions. These functions
119 #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
120 #: that an explicit instance of the variable is released with the generic
121 #: ``WIZARD_*`` constant. See also :func:`filename_regex_substitution`.
123 #: Dictionary of file names to a list of resolutions, which are tuples of
124 #: a conflict marker string and a result list. See :mod:`wizard.resolve`
125 #: for more information.
127 #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
128 #: this application requires.
129 install_schema = None
130 #: Name of the database that this application uses, i.e. ``mysql`` or
131 #: ``postgres``. If we end up supporting multiple databases for a single
132 #: application, there should also be a value for this in
133 #: :class:`wizard.deploy.Deployment`; the value here is merely the preferred
136 #: Indicates whether or not a web stub is necessary.
137 needs_web_stub = False
138 def __init__(self, name):
142 self._extractors = {}
143 self._substitutions = {}
144 def repository(self, srv_path):
146 Returns the Git repository that would contain this application.
147 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
149 repo = os.path.join(srv_path, self.name + ".git")
150 if not os.path.isdir(repo):
151 repo = os.path.join(srv_path, self.name, ".git")
152 if not os.path.isdir(repo):
153 raise NoRepositoryError(self.name)
155 def makeVersion(self, version):
157 Creates or retrieves the :class:`ApplicationVersion` singleton for the
160 if version not in self.versions:
161 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
162 return self.versions[version]
163 def extract(self, deployment):
165 Extracts wizard variables from a deployment. Default implementation
166 uses :attr:`extractors`.
169 for k,extractor in self.extractors.items():
170 result[k] = extractor(deployment)
171 # XXX: ugh... we have to do quoting
172 for k in self.random_keys:
173 if result[k] is None or result[k] in self.random_blacklist:
174 result[k] = "'%s'" % util.random_key()
176 def dsn(self, deployment):
178 Returns the deployment specific database URL. Uses the override file
179 in :file:`.wizard` if it exists, and otherwise attempt to extract the
180 variables from the source files.
182 Under some cases, the database URL will contain only the database
183 property, and no other values. This indicates that the actual DSN
184 should be determined from the environment.
186 This function might return ``None``.
190 We are allowed to batch these two together, because the full precedence
191 chain for determining the database of an application combines these
192 two together. If this was not the case, we would have to call
193 :meth:`databaseUrlFromOverride` and :meth:`databaseUrlFromExtract` manually.
195 url = self.dsnFromOverride(deployment)
198 return self.dsnFromExtract(deployment)
199 def dsnFromOverride(self, deployment):
201 Extracts database URL from an explicit dsn override file.
204 return sqlalchemy.engine.url.make_url(open(deployment.dsn_file).read().strip())
207 def dsnFromExtract(self, deployment):
209 Extracts database URL from a deployment, and returns them as
210 a :class:`sqlalchemy.engine.url.URL`. Returns ``None`` if we
211 can't figure it out: i.e. the conventional variables are not defined
212 for this application.
214 if not self.database:
216 vars = self.extract(deployment)
217 names = ("WIZARD_DBSERVER", "WIZARD_DBUSER", "WIZARD_DBPASSWORD", "WIZARD_DBNAME")
218 host, user, password, database = (shlex.split(vars[x])[0] if vars[x] is not None else None for x in names)
219 # XXX: You'd have to put support for an explicit different database
221 return sqlalchemy.engine.url.URL(self.database, username=user, password=password, host=host, database=database)
222 def url(self, deployment):
224 Returns the deployment specific web URL. Uses the override file
225 in :file:`.wizard` if it exists, and otherwise attempt to extract
226 the variables from the source files.
228 This function might return ``None``, which indicates we couldn't figure
231 url = self.urlFromOverride(deployment)
234 return self.urlFromExtract(deployment)
235 def urlFromOverride(self, deployment):
237 Extracts URL from explicit url override file.
240 return urlparse.urlparse(open(deployment.url_file).read().strip())
243 def urlFromExtract(self, deployment):
245 Extracts URL from a deployment, and returns ``None`` if we can't
246 figure it out. Default implementation is to fail; we might
247 do something clever with extractable variables in the future.
250 def parametrize(self, deployment, ref_deployment):
252 Takes a generic source checkout and parametrizes it according to the
253 values of ``deployment``. This function operates on the current
254 working directory. ``deployment`` should **not** be the same as the
255 current working directory. See :meth:`parametrizeWithVars` for details
256 on the parametrization.
258 # deployment is not used in this implementation, but note that
259 # we do have the invariant the current directory matches
260 # deployment's directory
261 variables = ref_deployment.extract()
262 self.parametrizeWithVars(variables)
263 def parametrizeWithVars(self, variables):
265 Takes a generic source checkout and parametrizes it according to
266 the values of ``variables``. Default implementation uses
267 :attr:`parametrized_files` and a simple search and replace on
270 for file in self.parametrized_files:
271 logging.debug("Parametrizing file '%s'\n" % (file, ))
273 contents = open(file, "r").read()
275 logging.debug("Failed to open file '%s'\n" % (file, ))
277 for key, value in variables.items():
278 if value is None: continue
279 contents = contents.replace(key, value)
282 def resolveConflicts(self, deployment):
284 Resolves conflicted files in the current working directory. Returns
285 whether or not all conflicted files were resolved or not. Fully
286 resolved files are added to the index, but no commit is made. The
287 default implementation uses :attr:`resolutions`.
292 for status in shell.eval("git", "ls-files", "--unmerged").splitlines():
293 mode, hash, role, name = status.split()
294 files.setdefault(name, set()).add(int(role))
295 for file, roles in files.items():
296 # some automatic resolutions
297 if 1 not in roles and 2 not in roles and 3 in roles:
298 # upstream added a file, but it conflicted for whatever reason
299 shell.call("git", "add", file)
301 elif 1 in roles and 2 not in roles and 3 in roles:
302 # user deleted the file, but upstream changed it
303 shell.call("git", "rm", file)
306 # XXX: this functionality is mostly subsumed by the rerere
308 if file in self.resolutions:
309 contents = open(file, "r").read()
310 for spec, result in self.resolutions[file]:
311 old_contents = contents
312 contents = resolve.resolve(contents, spec, result)
313 if old_contents != contents:
314 logging.info("Did resolution with spec:\n" + spec)
315 open(file, "w").write(contents)
316 if not resolve.is_conflict(contents):
317 shell.call("git", "add", file)
323 def prepareMerge(self, deployment):
325 Performs various edits to files in the current working directory in
326 order to make a merge go more smoothly. This is usually
327 used to fix botched line-endings. If you add new files,
328 you have to 'git add' them; this is not necessary for edits.
329 By default this is a no-op; subclasses should replace this
330 with useful behavior.
333 def prepareConfig(self, deployment):
335 Takes a deployment and replaces any explicit instances
336 of a configuration variable with generic ``WIZARD_*`` constants.
337 The default implementation uses :attr:`substitutions`, and
338 emits warnings when it encounters keys in :attr:`deprecated_keys`.
340 for key, subst in self.substitutions.items():
341 subs = subst(deployment)
342 if not subs and key not in self.deprecated_keys and key not in self.random_keys:
343 logging.warning("No substitutions for %s" % key)
344 def install(self, version, options):
346 Run for 'wizard configure' (and, by proxy, 'wizard install') to
347 configure an application. This assumes that the current working
348 directory is a deployment. (Unlike its kin, this function does not
349 take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should
350 provide an implementation.
352 raise NotImplementedError
353 def upgrade(self, deployment, version, options):
355 Run for 'wizard upgrade' to upgrade database schemas and other
356 non-versioned data in an application after the filesystem has been
357 upgraded. This assumes that the current working directory is the
358 deployment. Subclasses should provide an implementation.
360 raise NotImplementedError
361 def backup(self, deployment, outdir, options):
363 Run for 'wizard backup' and upgrades to backup database schemas
364 and other non-versioned data in an application. ``outdir`` is
365 the directory that backup files should be placed. This assumes
366 that the current working directory is the deployment. Subclasses
367 should provide an implementation, even if it is a no-op.
370 Static user files may not need to be backed up, since in
371 many applications upgrades do not modify static files.
373 raise NotImplementedError
374 def restore(self, deployment, backup_dir, options):
376 Run for 'wizard restore' and failed upgrades to restore database
377 and other non-versioned data to a backed up version. This assumes
378 that the current working directory is the deployment. Subclasses
379 should provide an implementation.
381 raise NotImplementedError
382 def remove(self, deployment, options):
384 Run for 'wizard remove' to delete all database and non-local
385 file data. This assumes that the current working directory is
386 the deployment. Subclasses should provide an implementation.
388 raise NotImplementedError
389 def detectVersion(self, deployment):
391 Checks source files to determine the version manually. This assumes
392 that the current working directory is the deployment. Subclasses
393 should provide an implementation.
395 raise NotImplementedError
396 def detectVersionFromFile(self, filename, regex):
398 Helper method that detects a version by using a regular expression
399 from a file. The regexed value is passed through :mod:`shlex`.
400 This assumes that the current working directory is the deployment.
402 contents = open(filename).read()
403 match = regex.search(contents)
404 if not match: return None
405 return distutils.version.LooseVersion(shlex.split(match.group(2))[0])
406 # XXX: This signature doesn't really make too much sense...
407 def detectVersionFromGit(self, tagPattern, preStrip = ''):
409 Helper method that detects a version by using the most recent tag
410 in git that matches the specified pattern.
411 This assumes that the current working directory is the deployment.
413 sh = wizard.shell.Shell()
414 cmd = ['git', 'describe', '--tags', '--match', tagPattern, ]
415 tag = sh.call(*cmd, strip=True)
416 if tag and len(tag) > len(preStrip) and tag[:len(preStrip)] == preStrip:
417 tag = tag[len(preStrip):]
418 if not tag: return None
419 return distutils.version.LooseVersion(tag)
420 def download(self, version):
422 Returns a URL that can be used to download a tarball of ``version`` of
425 raise NotImplementedError
426 def checkWeb(self, deployment):
428 Checks if the autoinstall is viewable from the web. Subclasses should
429 provide an implementation.
432 Finding a reasonable heuristic that works across skinning
433 choices can be difficult. We've had reasonable success
434 searching for metadata. Be sure that the standard error
435 page does not contain the features you search for. Try
436 not to depend on pages that are not the main page.
438 raise NotImplementedError
439 def checkDatabase(self, deployment):
441 Checks if the database is accessible.
444 sql.connect(deployment.dsn)
446 except sqlalchemy.exc.DBAPIError:
448 def checkWebPage(self, deployment, page, outputs=[], exclude=[]):
450 Checks if a given page of an autoinstall contains a particular string.
452 page = deployment.fetch(page)
454 if page.find(x) != -1:
455 logging.info("checkWebPage (failed due to %s):\n\n%s", x, page)
458 for output in outputs:
459 votes += page.find(output) != -1
460 if votes > len(outputs) / 2:
461 logging.debug("checkWebPage (passed):\n\n" + page)
464 logging.info("checkWebPage (failed):\n\n" + page)
466 def checkConfig(self, deployment):
468 Checks whether or not an autoinstall has been configured/installed
469 for use. Assumes that the current working directory is the deployment.
470 Subclasses should provide an implementation.
472 # XXX: Unfortunately, this doesn't quite work because we package
473 # bogus config files. Maybe we should check a hash or
475 raise NotImplementedError
476 def researchFilter(self, filename, added, deleted):
478 Allows an application to selectively ignore certain diffstat signatures
479 during research; for example, configuration files will have a very
480 specific set of changes, so ignore them; certain installation files
481 may be removed, etc. Return ``True`` if a diffstat signature should be
485 def researchVerbose(self, filename):
487 Allows an application to exclude certain dirty files from the output
488 report; usually this will just be parametrized files, since those are
489 guaranteed to have changes. Return ``True`` if a file should only
490 be displayed in verbose mode.
492 return filename in self.parametrized_files
494 class ApplicationVersion(object):
495 """Represents an abstract notion of a version for an application, where
496 ``version`` is a :class:`distutils.version.LooseVersion` and
497 ``application`` is a :class:`Application`."""
498 #: The :class:`distutils.version.LooseVersion` of this instance.
500 #: The :class:`Application` of this instance.
502 def __init__(self, version, application):
503 self.version = version
504 self.application = application
508 Returns the name of the git describe tag for the commit the user is
509 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
511 return "%s-%s" % (self.application, self.version)
513 def wizard_tag(self):
515 Returns the name of the Git tag for this version.
517 # XXX: Scripts specific
518 end = str(self.version).partition('-scripts')[2].partition('-')[0]
519 return "%s-scripts%s" % (self.pristine_tag, end)
521 def pristine_tag(self):
523 Returns the name of the Git tag for the pristine version corresponding
526 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
527 def __cmp__(self, y):
528 return cmp(self.version, y.version)
532 Parses a line from the :term:`versions store` and return
533 :class:`ApplicationVersion`.
535 Use this only for cases when speed is of primary importance;
536 the data in version is unreliable and when possible, you should
537 prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
538 the autoinstall itself for information.
540 The `value` to parse will vary. For old style installs, it
543 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
545 For new style installs, it will look like::
549 name = value.split("/")[-1]
551 if name.find("-") != -1:
552 app, _, version = name.partition("-")
554 # kind of poor, maybe should error. Generally this
555 # will actually result in a not found error
559 raise DeploymentParseError(value)
560 return ApplicationVersion.make(app, version)
562 def make(app, version):
564 Makes/retrieves a singleton :class:`ApplicationVersion` from
565 a``app`` and ``version`` string.
567 # defer to the application for version creation to enforce
569 return getApplication(app).makeVersion(version)
573 Takes a tree of values (implement using nested lists) and
574 transforms them into regular expressions.
578 >>> expand_re(['a', 'b'])
580 >>> expand_re(['*', ['b', 'c']])
583 if isinstance(val, str):
584 return re.escape(val)
586 return '(?:' + '|'.join(map(expand_re, val)) + ')'
588 def make_extractors(seed):
590 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into
591 extractor functions (which take a :class:`wizard.deploy.Deployment`
592 and return the value of the second subpattern of ``regex`` when matched
593 with the contents of ``file``).
595 return util.dictmap(lambda a: filename_regex_extractor(*a), seed)
597 def make_substitutions(seed):
599 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into substitution
600 functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
601 of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
603 return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)
605 # The following two functions are *highly* functional, and I recommend
606 # not touching them unless you know what you're doing.
608 def filename_regex_extractor(file, regex):
610 .. highlight:: haskell
612 Given a relative file name ``file``, a regular expression ``regex``, and a
613 :class:`wizard.deploy.Deployment` extracts a value out of the file in that
614 deployment. This function is curried, so you pass just ``file`` and
615 ``regex``, and then pass ``deployment`` to the resulting function.
617 Its Haskell-style type signature would be::
619 Filename -> Regex -> (Deployment -> String)
621 The regular expression requires a very specific form, essentially ``()()()``
622 (with the second subgroup being the value to extract). These enables
623 the regular expression to be used equivalently with filename
625 .. highlight:: python
627 For convenience purposes, we also accept ``[Filename]``, in which case
628 we use the first entry (index 0). Passing an empty list is invalid.
630 >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
631 >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
632 >>> f(deploy.Deployment("."))
634 >>> os.unlink("test-settings.extractor.ini")
637 The first application of ``regex`` and ``file`` is normally performed
638 at compile-time inside a submodule; the second application is
639 performed at runtime.
641 if not isinstance(file, str):
645 contents = deployment.read(file) # cached
648 match = regex.search(contents)
649 if not match: return None
650 # assumes that the second match is the one we want.
651 return match.group(2)
654 def filename_regex_substitution(key, files, regex):
656 .. highlight:: haskell
658 Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
659 regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
660 performs a substitution of the second subpattern of ``regex``
661 with ``key``. Returns the number of replacements made. This function
662 is curried, so you pass just ``key``, ``files`` and ``regex``, and
663 then pass ``deployment`` to the resulting function.
665 Its Haskell-style type signature would be::
667 Key -> ([File], Regex) -> (Deployment -> IO Int)
669 .. highlight:: python
671 For convenience purposes, we also accept ``Filename``, in which case it is treated
672 as a single item list.
674 >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
675 >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
676 >>> f(deploy.Deployment("."))
678 >>> print open("test-settings.substitution.ini", "r").read()
679 config_var = WIZARD_KEY
680 >>> os.unlink("test-settings.substitution.ini")
682 if isinstance(files, str):
685 base = deployment.location
688 file = os.path.join(base, file)
690 contents = open(file, "r").read()
691 contents, n = regex.subn("\\1" + key + "\\3", contents)
693 open(file, "w").write(contents)
700 def throws_database_errors(f, self, *args, **kwargs):
702 Decorator that takes database errors from :mod:`wizard.sql` and
703 converts them into application script failures from
704 :mod:`wizard.app`. We can't throw application errors directly from
705 :mod:`wizard.sql` because that would result in a cyclic import;
706 also, it's cleaner to distinguish between a database error and an
707 application script failure.
710 return f(self, *args, **kwargs)
711 except sql.BackupDatabaseError:
712 raise BackupFailure(traceback.format_exc())
713 except sql.RestoreDatabaseError:
714 raise RestoreFailure(traceback.format_exc())
715 except sql.RemoveDatabaseError:
716 raise RemoveFailure(traceback.format_exc())
718 class Error(wizard.Error):
719 """Generic error class for this module."""
722 class NoRepositoryError(Error):
724 :class:`Application` does not appear to have a Git repository
725 in the normal location.
727 #: The name of the application that does not have a Git repository.
729 def __init__(self, app):
732 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
734 class DeploymentParseError(Error):
736 Could not parse ``value`` from :term:`versions store`.
738 #: The value that failed to parse.
740 #: The location of the autoinstall that threw this variable.
741 #: This should be set by error handling code when it is available.
743 def __init__(self, value):
746 class NoSuchApplication(Error):
748 You attempted to reference a :class:`Application` named
749 ``app``, which is not recognized by Wizard.
751 #: The name of the application that does not exist.
753 #: The location of the autoinstall that threw this variable.
754 #: This should be set by error handling code when it is availble.
756 def __init__(self, app):
759 class Failure(Error):
761 Represents a failure when performing some double-dispatched operation
762 such as an installation or an upgrade. Failure classes are postfixed
763 with Failure, not Error.
767 class InstallFailure(Error):
768 """Installation failed for unknown reason."""
772 ERROR: Installation failed for unknown reason. You can
773 retry the installation by appending --retry to the installation
776 class RecoverableInstallFailure(InstallFailure):
778 Installation failed, but we were able to determine what the
779 error was, and should give the user a second chance if we were
780 running interactively.
782 #: List of the errors that were found.
784 def __init__(self, errors):
789 ERROR: Installation failed due to the following errors: %s
791 You can retry the installation by appending --retry to the
792 installation command.""" % ", ".join(self.errors)
794 class UpgradeFailure(Failure):
795 """Upgrade script failed."""
796 #: String details of failure (possibly stdout or stderr output)
798 def __init__(self, details):
799 self.details = details
803 ERROR: Upgrade script failed, details:
807 class UpgradeVerificationFailure(Failure):
808 """Upgrade script passed, but website wasn't accessible afterwards"""
812 ERROR: Upgrade script passed, but website wasn't accessible afterwards. Check
813 the debug logs for the contents of the page."""
815 class BackupFailure(Failure):
816 """Backup script failed."""
817 #: String details of failure
819 def __init__(self, details):
820 self.details = details
824 ERROR: Backup script failed, details:
828 class RestoreFailure(Failure):
829 """Restore script failed."""
830 #: String details of failure
832 def __init__(self, details):
833 self.details = details
837 ERROR: Restore script failed, details:
841 class RemoveFailure(Failure):
842 """Remove script failed."""
843 #: String details of failure
845 def __init__(self, details):
846 self.details = details
850 ERROR: Remove script failed, details: