2 Plumbing object model for representing applications we want to
3 install. This module does the heavy lifting, but you probably
4 want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
5 You'll need to know how to overload the :class:`Application` class
6 and use some of the functions in this module in order to specify
9 To specify custom applications as plugins, add the following ``entry_points``
13 yourappname = your.module:Application
14 otherappname = your.other.module:Application
18 Wizard will complain loudly if ``yourappname`` conflicts with an
19 application name defined by someone else.
21 There are some submodules for programming languages that define common
22 functions and data that may be used by applications in that language. See:
24 * :mod:`wizard.app.php`
31 from wizard import deploy, util
32 from wizard.app import *
38 import distutils.version
52 from wizard import plugin, resolve, shell, sql, util
56 """Hash table for looking up string application name to instance"""
59 _applications = dict()
60 for dist in pkg_resources.working_set:
61 for appname, entry in dist.get_entry_map("wizard.app").items():
62 if appname in _applications:
63 newname = dist.key + ":" + appname
64 if newname in _applications:
65 raise Exception("Unrecoverable application name conflict for %s from %s", appname, dist.key)
66 logging.warning("Could not overwrite %s, used %s instead", appname, newname)
68 appclass = entry.load()
69 _applications[appname] = appclass(appname)
71 for entry in pkg_resources.iter_entry_points("wizard.dummy_apps"):
74 for appname in dummy_apps:
75 # a dummy app that already exists is not a fatal error
76 if appname in _applications:
78 _applications[appname] = Application(appname)
81 def getApplication(appname):
82 """Retrieves application instance given a name"""
84 return applications()[appname]
86 raise NoSuchApplication(appname)
88 class Application(object):
90 Represents an application, i.e. mediawiki or phpbb.
93 Many of these methods assume a specific working
94 directory; prefer using the corresponding methods
95 in :class:`wizard.deploy.Deployment` and its subclasses.
97 #: String name of the application
99 #: Human-readable name of the application
101 #: Dictionary of version strings to :class:`ApplicationVersion`.
102 #: See also :meth:`makeVersion`.
104 #: List of files that need to be modified when parametrizing.
105 #: This is a class-wide constant, and should not normally be modified.
106 parametrized_files = []
107 #: Keys that are used in older versions of the application, but
108 #: not for the most recent version.
109 deprecated_keys = set()
110 #: Keys that we can simply generate random strings for if they're missing
112 #: Values that are not sufficiently random for a random key. This can
113 #: include default values for a random configuration option,
114 random_blacklist = set()
115 #: Dictionary of variable names to extractor functions. These functions
116 #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
117 #: the variable, or ``None`` if it could not be found.
118 #: See also :func:`filename_regex_extractor`.
120 #: Dictionary of variable names to substitution functions. These functions
121 #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
122 #: that an explicit instance of the variable is released with the generic
123 #: ``WIZARD_*`` constant. See also :func:`filename_regex_substitution`.
125 #: Dictionary of file names to a list of resolutions, which are tuples of
126 #: a conflict marker string and a result list. See :mod:`wizard.resolve`
127 #: for more information.
129 #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
130 #: this application requires.
131 install_schema = None
132 #: Name of the database that this application uses, i.e. ``mysql`` or
133 #: ``postgres``. If we end up supporting multiple databases for a single
134 #: application, there should also be a value for this in
135 #: :class:`wizard.deploy.Deployment`; the value here is merely the preferred
138 #: Indicates whether or not a web stub is necessary.
139 needs_web_stub = False
140 def __init__(self, name):
144 self._extractors = {}
145 self._substitutions = {}
146 def repository(self, srv_path):
148 Returns the Git repository that would contain this application.
149 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
151 repo = os.path.join(srv_path, self.name + ".git")
152 if not os.path.isdir(repo):
153 repo = os.path.join(srv_path, self.name, ".git")
154 if not os.path.isdir(repo):
155 raise NoRepositoryError(self.name)
157 def makeVersion(self, version):
159 Creates or retrieves the :class:`ApplicationVersion` singleton for the
162 if version not in self.versions:
163 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
164 return self.versions[version]
165 def extract(self, deployment):
167 Extracts wizard variables from a deployment. Default implementation
168 uses :attr:`extractors`.
171 for k,extractor in self.extractors.items():
172 result[k] = extractor(deployment)
173 # XXX: ugh... we have to do quoting
174 for k in self.random_keys:
175 if result[k] is None or result[k] in self.random_blacklist:
176 result[k] = "'%s'" % util.random_key()
178 def dsn(self, deployment):
180 Returns the deployment specific database URL. Uses the override file
181 in :file:`.wizard` if it exists, and otherwise attempt to extract the
182 variables from the source files.
184 Under some cases, the database URL will contain only the database
185 property, and no other values. This indicates that the actual DSN
186 should be determined from the environment.
188 This function might return ``None``.
192 We are allowed to batch these two together, because the full precedence
193 chain for determining the database of an application combines these
194 two together. If this was not the case, we would have to call
195 :meth:`databaseUrlFromOverride` and :meth:`databaseUrlFromExtract` manually.
197 url = self.dsnFromOverride(deployment)
200 return self.dsnFromExtract(deployment)
201 def dsnFromOverride(self, deployment):
203 Extracts database URL from an explicit dsn override file.
206 return sqlalchemy.engine.url.make_url(open(deployment.dsn_file).read().strip())
209 def dsnFromExtract(self, deployment):
211 Extracts database URL from a deployment, and returns them as
212 a :class:`sqlalchemy.engine.url.URL`. Returns ``None`` if we
213 can't figure it out: i.e. the conventional variables are not defined
214 for this application.
216 if not self.database:
218 vars = self.extract(deployment)
219 names = ("WIZARD_DBSERVER", "WIZARD_DBUSER", "WIZARD_DBPASSWORD", "WIZARD_DBNAME")
220 host, user, password, database = (shlex.split(vars[x])[0] if vars[x] is not None else None for x in names)
221 # XXX: You'd have to put support for an explicit different database
223 return sqlalchemy.engine.url.URL(self.database, username=user, password=password, host=host, database=database)
224 def url(self, deployment):
226 Returns the deployment specific web URL. Uses the override file
227 in :file:`.wizard` if it exists, and otherwise attempt to extract
228 the variables from the source files.
230 This function might return ``None``, which indicates we couldn't figure
233 url = self.urlFromOverride(deployment)
236 return self.urlFromExtract(deployment)
237 def urlFromOverride(self, deployment):
239 Extracts URL from explicit url override file.
242 return urlparse.urlparse(open(deployment.url_file).read().strip())
245 def urlFromExtract(self, deployment):
247 Extracts URL from a deployment, and returns ``None`` if we can't
248 figure it out. Default implementation is to fail; we might
249 do something clever with extractable variables in the future.
252 def parametrize(self, deployment, ref_deployment):
254 Takes a generic source checkout and parametrizes it according to the
255 values of ``deployment``. This function operates on the current
256 working directory. ``deployment`` should **not** be the same as the
257 current working directory. See :meth:`parametrizeWithVars` for details
258 on the parametrization.
260 # deployment is not used in this implementation, but note that
261 # we do have the invariant the current directory matches
262 # deployment's directory
263 variables = ref_deployment.extract()
264 self.parametrizeWithVars(variables)
265 def parametrizeWithVars(self, variables):
267 Takes a generic source checkout and parametrizes it according to
268 the values of ``variables``. Default implementation uses
269 :attr:`parametrized_files` and a simple search and replace on
272 for file in self.parametrized_files:
273 logging.debug("Parametrizing file '%s'\n" % (file, ))
275 contents = open(file, "r").read()
277 logging.debug("Failed to open file '%s'\n" % (file, ))
279 for key, value in variables.items():
280 if value is None: continue
281 contents = contents.replace(key, value)
284 def resolveConflicts(self, deployment):
286 Resolves conflicted files in the current working directory. Returns
287 whether or not all conflicted files were resolved or not. Fully
288 resolved files are added to the index, but no commit is made. The
289 default implementation uses :attr:`resolutions`.
294 for status in shell.eval("git", "ls-files", "--unmerged").splitlines():
295 mode, hash, role, name = status.split()
296 files.setdefault(name, set()).add(int(role))
297 for file, roles in files.items():
298 # some automatic resolutions
299 if 1 not in roles and 2 not in roles and 3 in roles:
300 # upstream added a file, but it conflicted for whatever reason
301 shell.call("git", "add", file)
303 elif 1 in roles and 2 not in roles and 3 in roles:
304 # user deleted the file, but upstream changed it
305 shell.call("git", "rm", file)
308 # XXX: this functionality is mostly subsumed by the rerere
310 if file in self.resolutions:
311 contents = open(file, "r").read()
312 for spec, result in self.resolutions[file]:
313 old_contents = contents
314 contents = resolve.resolve(contents, spec, result)
315 if old_contents != contents:
316 logging.info("Did resolution with spec:\n" + spec)
317 open(file, "w").write(contents)
318 if not resolve.is_conflict(contents):
319 shell.call("git", "add", file)
325 def prepareMerge(self, deployment):
327 Performs various edits to files in the current working directory in
328 order to make a merge go more smoothly. This is usually
329 used to fix botched line-endings. If you add new files,
330 you have to 'git add' them; this is not necessary for edits.
331 By default this is a no-op; subclasses should replace this
332 with useful behavior.
335 def prepareConfig(self, deployment):
337 Takes a deployment and replaces any explicit instances
338 of a configuration variable with generic ``WIZARD_*`` constants.
339 The default implementation uses :attr:`substitutions`, and
340 emits warnings when it encounters keys in :attr:`deprecated_keys`.
342 for key, subst in self.substitutions.items():
343 subs = subst(deployment)
344 if not subs and key not in self.deprecated_keys and key not in self.random_keys:
345 logging.warning("No substitutions for %s" % key)
346 def install(self, version, options):
348 Run for 'wizard configure' (and, by proxy, 'wizard install') to
349 configure an application. This assumes that the current working
350 directory is a deployment. (Unlike its kin, this function does not
351 take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should
352 provide an implementation.
354 raise NotImplementedError
355 def upgrade(self, deployment, version, options):
357 Run for 'wizard upgrade' to upgrade database schemas and other
358 non-versioned data in an application after the filesystem has been
359 upgraded. This assumes that the current working directory is the
360 deployment. Subclasses should provide an implementation.
362 raise NotImplementedError
363 def backup(self, deployment, outdir, options):
365 Run for 'wizard backup' and upgrades to backup database schemas
366 and other non-versioned data in an application. ``outdir`` is
367 the directory that backup files should be placed. This assumes
368 that the current working directory is the deployment. Subclasses
369 should provide an implementation, even if it is a no-op.
372 Static user files may not need to be backed up, since in
373 many applications upgrades do not modify static files.
375 raise NotImplementedError
376 def restore(self, deployment, backup_dir, options):
378 Run for 'wizard restore' and failed upgrades to restore database
379 and other non-versioned data to a backed up version. This assumes
380 that the current working directory is the deployment. Subclasses
381 should provide an implementation.
383 raise NotImplementedError
384 def remove(self, deployment, options):
386 Run for 'wizard remove' to delete all database and non-local
387 file data. This assumes that the current working directory is
388 the deployment. Subclasses should provide an implementation.
390 raise NotImplementedError
391 def detectVersion(self, deployment):
393 Checks source files to determine the version manually. This assumes
394 that the current working directory is the deployment. Subclasses
395 should provide an implementation.
397 raise NotImplementedError
398 def detectVersionFromFile(self, filename, regex):
400 Helper method that detects a version by using a regular expression
401 from a file. The regexed value is passed through :mod:`shlex`.
402 This assumes that the current working directory is the deployment.
404 contents = open(filename).read()
405 match = regex.search(contents)
406 if not match: return None
407 return distutils.version.LooseVersion(shlex.split(match.group(2))[0])
408 # XXX: This signature doesn't really make too much sense...
409 def detectVersionFromGit(self, tagPattern, preStrip = ''):
411 Helper method that detects a version by using the most recent tag
412 in git that matches the specified pattern.
413 This assumes that the current working directory is the deployment.
415 sh = wizard.shell.Shell()
416 cmd = ['git', 'describe', '--tags', '--match', tagPattern, ]
417 tag = sh.call(*cmd, strip=True)
418 if tag and len(tag) > len(preStrip) and tag[:len(preStrip)] == preStrip:
419 tag = tag[len(preStrip):]
420 if not tag: return None
421 return distutils.version.LooseVersion(tag)
422 def download(self, version):
424 Returns a URL that can be used to download a tarball of ``version`` of
427 raise NotImplementedError
428 def checkWeb(self, deployment):
430 Checks if the autoinstall is viewable from the web. Subclasses should
431 provide an implementation.
434 Finding a reasonable heuristic that works across skinning
435 choices can be difficult. We've had reasonable success
436 searching for metadata. Be sure that the standard error
437 page does not contain the features you search for. Try
438 not to depend on pages that are not the main page.
440 raise NotImplementedError
441 def checkDatabase(self, deployment):
443 Checks if the database is accessible.
446 sql.connect(deployment.dsn)
448 except sqlalchemy.exc.DBAPIError:
450 def checkWebPage(self, deployment, page, outputs=[], exclude=[]):
452 Checks if a given page of an autoinstall contains a particular string.
454 page = deployment.fetch(page)
456 if page.find(x) != -1:
457 logging.info("checkWebPage (failed due to %s):\n\n%s", x, page)
460 for output in outputs:
461 votes += page.find(output) != -1
462 if votes > len(outputs) / 2:
463 logging.debug("checkWebPage (passed):\n\n" + page)
466 logging.info("checkWebPage (failed):\n\n" + page)
468 def checkConfig(self, deployment):
470 Checks whether or not an autoinstall has been configured/installed
471 for use. Assumes that the current working directory is the deployment.
472 Subclasses should provide an implementation.
474 # XXX: Unfortunately, this doesn't quite work because we package
475 # bogus config files. Maybe we should check a hash or
477 raise NotImplementedError
478 def researchFilter(self, filename, added, deleted):
480 Allows an application to selectively ignore certain diffstat signatures
481 during research; for example, configuration files will have a very
482 specific set of changes, so ignore them; certain installation files
483 may be removed, etc. Return ``True`` if a diffstat signature should be
487 def researchVerbose(self, filename):
489 Allows an application to exclude certain dirty files from the output
490 report; usually this will just be parametrized files, since those are
491 guaranteed to have changes. Return ``True`` if a file should only
492 be displayed in verbose mode.
494 return filename in self.parametrized_files
496 class ApplicationVersion(object):
497 """Represents an abstract notion of a version for an application, where
498 ``version`` is a :class:`distutils.version.LooseVersion` and
499 ``application`` is a :class:`Application`."""
500 #: The :class:`distutils.version.LooseVersion` of this instance.
502 #: The :class:`Application` of this instance.
504 def __init__(self, version, application):
505 self.version = version
506 self.application = application
510 Returns the name of the git describe tag for the commit the user is
511 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
513 return "%s-%s" % (self.application, self.version)
515 def wizard_tag(self):
517 Returns the name of the Git tag for this version.
519 # XXX: Scripts specific
520 end = str(self.version).partition('-scripts')[2].partition('-')[0]
521 return "%s-scripts%s" % (self.pristine_tag, end)
523 def pristine_tag(self):
525 Returns the name of the Git tag for the pristine version corresponding
528 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
529 def __cmp__(self, y):
530 return cmp(self.version, y.version)
534 Parses a line from the :term:`versions store` and return
535 :class:`ApplicationVersion`.
537 Use this only for cases when speed is of primary importance;
538 the data in version is unreliable and when possible, you should
539 prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
540 the autoinstall itself for information.
542 The `value` to parse will vary. For old style installs, it
545 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
547 For new style installs, it will look like::
551 name = value.split("/")[-1]
553 if name.find("-") != -1:
554 app, _, version = name.partition("-")
556 # kind of poor, maybe should error. Generally this
557 # will actually result in a not found error
561 raise DeploymentParseError(value)
562 return ApplicationVersion.make(app, version)
564 def make(app, version):
566 Makes/retrieves a singleton :class:`ApplicationVersion` from
567 a``app`` and ``version`` string.
569 # defer to the application for version creation to enforce
571 return getApplication(app).makeVersion(version)
575 Takes a tree of values (implement using nested lists) and
576 transforms them into regular expressions.
580 >>> expand_re(['a', 'b'])
582 >>> expand_re(['*', ['b', 'c']])
585 if isinstance(val, str):
586 return re.escape(val)
588 return '(?:' + '|'.join(map(expand_re, val)) + ')'
590 def make_extractors(seed):
592 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into
593 extractor functions (which take a :class:`wizard.deploy.Deployment`
594 and return the value of the second subpattern of ``regex`` when matched
595 with the contents of ``file``).
597 return util.dictmap(lambda a: filename_regex_extractor(*a), seed)
599 def make_substitutions(seed):
601 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into substitution
602 functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
603 of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
605 return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)
607 # The following two functions are *highly* functional, and I recommend
608 # not touching them unless you know what you're doing.
610 def filename_regex_extractor(file, regex):
612 .. highlight:: haskell
614 Given a relative file name ``file``, a regular expression ``regex``, and a
615 :class:`wizard.deploy.Deployment` extracts a value out of the file in that
616 deployment. This function is curried, so you pass just ``file`` and
617 ``regex``, and then pass ``deployment`` to the resulting function.
619 Its Haskell-style type signature would be::
621 Filename -> Regex -> (Deployment -> String)
623 The regular expression requires a very specific form, essentially ``()()()``
624 (with the second subgroup being the value to extract). These enables
625 the regular expression to be used equivalently with filename
627 .. highlight:: python
629 For convenience purposes, we also accept ``[Filename]``, in which case
630 we use the first entry (index 0). Passing an empty list is invalid.
632 >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
633 >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
634 >>> f(deploy.Deployment("."))
636 >>> os.unlink("test-settings.extractor.ini")
639 The first application of ``regex`` and ``file`` is normally performed
640 at compile-time inside a submodule; the second application is
641 performed at runtime.
643 if not isinstance(file, str):
647 contents = deployment.read(file) # cached
650 match = regex.search(contents)
651 if not match: return None
652 # assumes that the second match is the one we want.
653 return match.group(2)
656 def filename_regex_substitution(key, files, regex):
658 .. highlight:: haskell
660 Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
661 regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
662 performs a substitution of the second subpattern of ``regex``
663 with ``key``. Returns the number of replacements made. This function
664 is curried, so you pass just ``key``, ``files`` and ``regex``, and
665 then pass ``deployment`` to the resulting function.
667 Its Haskell-style type signature would be::
669 Key -> ([File], Regex) -> (Deployment -> IO Int)
671 .. highlight:: python
673 For convenience purposes, we also accept ``Filename``, in which case it is treated
674 as a single item list.
676 >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
677 >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
678 >>> f(deploy.Deployment("."))
680 >>> print open("test-settings.substitution.ini", "r").read()
681 config_var = WIZARD_KEY
682 >>> os.unlink("test-settings.substitution.ini")
684 if isinstance(files, str):
687 base = deployment.location
690 file = os.path.join(base, file)
692 contents = open(file, "r").read()
693 contents, n = regex.subn("\\1" + key + "\\3", contents)
695 open(file, "w").write(contents)
702 def throws_database_errors(f, self, *args, **kwargs):
704 Decorator that takes database errors from :mod:`wizard.sql` and
705 converts them into application script failures from
706 :mod:`wizard.app`. We can't throw application errors directly from
707 :mod:`wizard.sql` because that would result in a cyclic import;
708 also, it's cleaner to distinguish between a database error and an
709 application script failure.
712 return f(self, *args, **kwargs)
713 except sql.BackupDatabaseError:
714 raise BackupFailure(traceback.format_exc())
715 except sql.RestoreDatabaseError:
716 raise RestoreFailure(traceback.format_exc())
717 except sql.RemoveDatabaseError:
718 raise RemoveFailure(traceback.format_exc())
720 class Error(wizard.Error):
721 """Generic error class for this module."""
724 class NoRepositoryError(Error):
726 :class:`Application` does not appear to have a Git repository
727 in the normal location.
729 #: The name of the application that does not have a Git repository.
731 def __init__(self, app):
734 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
736 class DeploymentParseError(Error):
738 Could not parse ``value`` from :term:`versions store`.
740 #: The value that failed to parse.
742 #: The location of the autoinstall that threw this variable.
743 #: This should be set by error handling code when it is available.
745 def __init__(self, value):
748 return "Could not parse '%s' from versions store in '%s'" % (self.value, self.location)
750 class NoSuchApplication(Error):
752 You attempted to reference a :class:`Application` named
753 ``app``, which is not recognized by Wizard.
755 #: The name of the application that does not exist.
757 #: The location of the autoinstall that threw this variable.
758 #: This should be set by error handling code when it is availble.
760 def __init__(self, app):
763 return "Wizard doesn't know about an application named '%s'." % self.app
765 class Failure(Error):
767 Represents a failure when performing some double-dispatched operation
768 such as an installation or an upgrade. Failure classes are postfixed
769 with Failure, not Error.
773 class InstallFailure(Error):
774 """Installation failed for unknown reason."""
778 ERROR: Installation failed for unknown reason. You can
779 retry the installation by appending --retry to the installation
782 class RecoverableInstallFailure(InstallFailure):
784 Installation failed, but we were able to determine what the
785 error was, and should give the user a second chance if we were
786 running interactively.
788 #: List of the errors that were found.
790 def __init__(self, errors):
795 ERROR: Installation failed due to the following errors: %s
797 You can retry the installation by appending --retry to the
798 installation command.""" % ", ".join(self.errors)
800 class UpgradeFailure(Failure):
801 """Upgrade script failed."""
802 #: String details of failure (possibly stdout or stderr output)
804 def __init__(self, details):
805 self.details = details
809 ERROR: Upgrade script failed, details:
813 class UpgradeVerificationFailure(Failure):
814 """Upgrade script passed, but website wasn't accessible afterwards"""
818 ERROR: Upgrade script passed, but website wasn't accessible afterwards. Check
819 the debug logs for the contents of the page."""
821 class BackupFailure(Failure):
822 """Backup script failed."""
823 #: String details of failure
825 def __init__(self, details):
826 self.details = details
830 ERROR: Backup script failed, details:
834 class RestoreFailure(Failure):
835 """Restore script failed."""
836 #: String details of failure
838 def __init__(self, details):
839 self.details = details
843 ERROR: Restore script failed, details:
847 class RemoveFailure(Failure):
848 """Remove script failed."""
849 #: String details of failure
851 def __init__(self, details):
852 self.details = details
856 ERROR: Remove script failed, details: