2 Plumbing object model for representing applications we want to
3 install. This module does the heavy lifting, but you probably
4 want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
5 You'll need to know how to overload the :class:`Application` class
6 and use some of the functions in this module in order to specify
9 To specify custom applications as plugins, add the following ``entry_points``
13 yourappname = your.module:Application
14 otherappname = your.other.module:Application
18 Wizard will complain loudly if ``yourappname`` conflicts with an
19 application name defined by someone else.
21 There are some submodules for programming languages that define common
22 functions and data that may be used by applications in that language. See:
24 * :mod:`wizard.app.php`
31 from wizard import deploy, util
32 from wizard.app import *
37 import distutils.version
50 from wizard import resolve, scripts, shell, util
53 _scripts_application_list = [
54 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
55 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
56 # these are technically deprecated
57 "advancedpoll", "gallery",
59 def _scripts_make(name):
60 """Makes an application, but uses the correct subtype if available."""
62 __import__("wizard.app." + name)
63 return getattr(wizard.app, name).Application(name)
64 except ImportError as error:
65 # XXX ugly hack to check if the import error is from the top level
66 # module we care about or a submodule. should be an archetectural change.
67 if error.args[0].split()[-1]==name:
68 return Application(name)
74 """Hash table for looking up string application name to instance"""
78 _applications = dict([(n,_scripts_make(n)) for n in _scripts_application_list ])
80 for dist in pkg_resources.working_set:
81 for appname, appclass in dist.get_entry_map("wizard.app").items():
82 if appname in _applications:
83 newname = dist.key + ":" + appname
84 if newname in _applications:
85 raise Exception("Unrecoverable application name conflict for %s from %s", appname, dist.key)
86 logging.warning("Could not overwrite %s, used %s instead", appname, newname)
88 _applications[appname] = appclass(appname)
91 def getApplication(appname):
92 """Retrieves application instance given a name"""
93 return applications()[appname]
95 class Application(object):
97 Represents an application, i.e. mediawiki or phpbb.
100 Many of these methods assume a specific working
101 directory; prefer using the corresponding methods
102 in :class:`wizard.deploy.Deployment` and its subclasses.
104 #: String name of the application
106 #: Dictionary of version strings to :class:`ApplicationVersion`.
107 #: See also :meth:`makeVersion`.
109 #: List of files that need to be modified when parametrizing.
110 #: This is a class-wide constant, and should not normally be modified.
111 parametrized_files = []
112 #: Keys that are used in older versions of the application, but
113 #: not for the most recent version.
114 deprecated_keys = set()
115 #: Keys that we can simply generate random strings for if they're missing
117 #: Dictionary of variable names to extractor functions. These functions
118 #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
119 #: the variable, or ``None`` if it could not be found.
120 #: See also :func:`filename_regex_extractor`.
122 #: Dictionary of variable names to substitution functions. These functions
123 #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
124 #: that an explicit instance of the variable is released with the generic
125 #: ``WIZARD_*`` constant. See also :func:`filename_regex_substitution`.
127 #: Dictionary of file names to a list of resolutions, which are tuples of
128 #: a conflict marker string and a result list. See :mod:`wizard.resolve`
129 #: for more information.
131 #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
132 #: this application requires.
133 install_schema = None
134 #: Name of the database that this application uses, i.e. ``mysql`` or
135 #: ``postgres``. If we end up supporting multiple databases for a single
136 #: application, there should also be a value for this in
137 #: :class:`wizard.deploy.Deployment`; the value here is merely the preferred
140 def __init__(self, name):
144 self._extractors = {}
145 self._substitutions = {}
146 def repository(self, srv_path):
148 Returns the Git repository that would contain this application.
149 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
151 repo = os.path.join(srv_path, self.name + ".git")
152 if not os.path.isdir(repo):
153 repo = os.path.join(srv_path, self.name, ".git")
154 if not os.path.isdir(repo):
155 raise NoRepositoryError(self.name)
157 def makeVersion(self, version):
159 Creates or retrieves the :class:`ApplicationVersion` singleton for the
162 if version not in self.versions:
163 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
164 return self.versions[version]
165 def extract(self, deployment):
167 Extracts wizard variables from a deployment. Default implementation
168 uses :attr:`extractors`.
171 for k,extractor in self.extractors.items():
172 result[k] = extractor(deployment)
173 # XXX: ugh... we have to do quoting
174 for k in self.random_keys:
175 if result[k] is None:
176 result[k] = "'%s'" % ''.join(random.choice(string.letters + string.digits) for i in xrange(30))
178 def dsn(self, deployment):
180 Returns the deployment specific database URL. Uses the override file
181 in :file:`.scripts` if it exists, and otherwise attempt to extract the
182 variables from the source files.
184 Under some cases, the database URL will contain only the database
185 property, and no other values. This indicates that the actual DSN
186 should be determined from the environment.
188 This function might return ``None``.
192 We are allowed to batch these two together, because the full precedence
193 chain for determining the database of an application combines these
194 two together. If this was not the case, we would have to call
195 :meth:`databaseUrlFromOverride` and :meth:`databaseUrlFromExtract` manually.
197 url = self.dsnFromOverride(deployment)
200 return self.dsnFromExtract(deployment)
201 def dsnFromOverride(self, deployment):
203 Extracts database URL from an explicit dsn override file.
206 return sqlalchemy.engine.url.make_url(open(deployment.dsn_file).read().strip())
209 def dsnFromExtract(self, deployment):
211 Extracts database URL from a deployment, and returns them as
212 a :class:`sqlalchemy.engine.url.URL`. Returns ``None`` if we
213 can't figure it out: i.e. the conventional variables are not defined
214 for this application.
216 if not self.database:
218 vars = self.extract(deployment)
219 names = ("WIZARD_DBSERVER", "WIZARD_DBUSER", "WIZARD_DBPASSWORD", "WIZARD_DBNAME")
220 host, user, password, database = (shlex.split(vars[x])[0] if vars[x] is not None else None for x in names)
221 # XXX: You'd have to put support for an explicit different database
223 return sqlalchemy.engine.url.URL(self.database, username=user, password=password, host=host, database=database)
224 def url(self, deployment):
226 Returns the deployment specific web URL. Uses the override file
227 in :file:`.scripts` if it exists, and otherwise attempt to extract
228 the variables from the source files.
230 This function might return ``None``, which indicates we couldn't figure
233 url = self.urlFromOverride(deployment)
236 return self.urlFromExtract(deployment)
237 def urlFromOverride(self, deployment):
239 Extracts URL from explicit url override file.
242 return urlparse.urlparse(open(deployment.url_file).read().strip())
245 def urlFromExtract(self, deployment):
247 Extracts URL from a deployment, and returns ``None`` if we can't
248 figure it out. Default implementation is to fail; we might
249 do something clever with extractable variables in the future.
252 def parametrize(self, deployment, ref_deployment):
254 Takes a generic source checkout and parametrizes it according to the
255 values of ``deployment``. This function operates on the current
256 working directory. ``deployment`` should **not** be the same as the
257 current working directory. Default implementation uses
258 :attr:`parametrized_files` and a simple search and replace on those
261 variables = ref_deployment.extract()
262 for file in self.parametrized_files:
264 contents = open(file, "r").read()
267 for key, value in variables.items():
268 if value is None: continue
269 contents = contents.replace(key, value)
272 def resolveConflicts(self, deployment):
274 Resolves conflicted files in the current working directory. Returns
275 whether or not all conflicted files were resolved or not. Fully
276 resolved files are added to the index, but no commit is made. The
277 default implementation uses :attr:`resolutions`.
282 for status in shell.eval("git", "ls-files", "--unmerged").splitlines():
283 mode, hash, role, name = status.split()
284 files.setdefault(name, set()).add(int(role))
285 for file, roles in files.items():
286 # some automatic resolutions
287 if 1 not in roles and 2 not in roles and 3 in roles:
288 # upstream added a file, but it conflicted for whatever reason
289 shell.call("git", "add", file)
291 elif 1 in roles and 2 not in roles and 3 in roles:
292 # user deleted the file, but upstream changed it
293 shell.call("git", "rm", file)
296 # XXX: this functionality is mostly subsumed by the rerere
298 if file in self.resolutions:
299 contents = open(file, "r").read()
300 for spec, result in self.resolutions[file]:
301 old_contents = contents
302 contents = resolve.resolve(contents, spec, result)
303 if old_contents != contents:
304 logging.info("Did resolution with spec:\n" + spec)
305 open(file, "w").write(contents)
306 if not resolve.is_conflict(contents):
307 shell.call("git", "add", file)
313 def prepareMerge(self, deployment):
315 Performs various edits to files in the current working directory in
316 order to make a merge go more smoothly. This is usually
317 used to fix botched line-endings. If you add new files,
318 you have to 'git add' them; this is not necessary for edits.
319 By default this is a no-op; subclasses should replace this
320 with useful behavior.
323 def prepareConfig(self, deployment):
325 Takes a deployment and replaces any explicit instances
326 of a configuration variable with generic ``WIZARD_*`` constants.
327 The default implementation uses :attr:`substitutions`, and
328 emits warnings when it encounters keys in :attr:`deprecated_keys`.
330 for key, subst in self.substitutions.items():
331 subs = subst(deployment)
332 if not subs and key not in self.deprecated_keys:
333 logging.warning("No substitutions for %s" % key)
334 def install(self, version, options):
336 Run for 'wizard configure' (and, by proxy, 'wizard install') to
337 configure an application. This assumes that the current working
338 directory is a deployment. (Unlike its kin, this function does not
339 take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should
340 provide an implementation.
342 raise NotImplementedError
343 def upgrade(self, deployment, version, options):
345 Run for 'wizard upgrade' to upgrade database schemas and other
346 non-versioned data in an application after the filesystem has been
347 upgraded. This assumes that the current working directory is the
348 deployment. Subclasses should provide an implementation.
350 raise NotImplementedError
351 def backup(self, deployment, outdir, options):
353 Run for 'wizard backup' and upgrades to backup database schemas
354 and other non-versioned data in an application. ``outdir`` is
355 the directory that backup files should be placed. This assumes
356 that the current working directory is the deployment. Subclasses
357 should provide an implementation, even if it is a no-op.
360 Static user files may not need to be backed up, since in
361 many applications upgrades do not modify static files.
363 raise NotImplementedError
364 def restore(self, deployment, backup_dir, options):
366 Run for 'wizard restore' and failed upgrades to restore database
367 and other non-versioned data to a backed up version. This assumes
368 that the current working directory is the deployment. Subclasses
369 should provide an implementation.
371 raise NotImplementedError
372 def remove(self, deployment, options):
374 Run for 'wizard remove' to delete all database and non-local
375 file data. This assumes that the current working directory is
376 the deployment. Subclasses should provide an implementation.
378 raise NotImplementedError
379 def detectVersion(self, deployment):
381 Checks source files to determine the version manually. This assumes
382 that the current working directory is the deployment. Subclasses
383 should provide an implementation.
385 raise NotImplementedError
386 def detectVersionFromFile(self, filename, regex):
388 Helper method that detects a version by using a regular expression
389 from a file. The regexed value is passed through :mod:`shlex`.
390 This assumes that the current working directory is the deployment.
392 contents = open(filename).read()
393 match = regex.search(contents)
394 if not match: return None
395 return distutils.version.LooseVersion(shlex.split(match.group(2))[0])
396 def download(self, version):
398 Returns a URL that can be used to download a tarball of ``version`` of
401 raise NotImplementedError
402 def checkWeb(self, deployment):
404 Checks if the autoinstall is viewable from the web. Subclasses should
405 provide an implementation.
408 Finding a reasonable heuristic that works across skinning
409 choices can be difficult. We've had reasonable success
410 searching for metadata. Be sure that the standard error
411 page does not contain the features you search for. Try
412 not to depend on pages that are not the main page.
414 raise NotImplementedError
415 def checkWebPage(self, deployment, page, outputs=[], exclude=[]):
417 Checks if a given page of an autoinstall contains a particular string.
419 page = deployment.fetch(page)
421 if page.find(x) != -1:
422 logging.info("checkWebPage (failed due to %s):\n\n%s", x, page)
425 for output in outputs:
426 votes += page.find(output) != -1
427 if votes > len(outputs) / 2:
428 logging.debug("checkWebPage (passed):\n\n" + page)
431 logging.info("checkWebPage (failed):\n\n" + page)
433 def checkConfig(self, deployment):
435 Checks whether or not an autoinstall has been configured/installed
436 for use. Assumes that the current working directory is the deployment.
437 Subclasses should provide an implementation.
439 # XXX: Unfortunately, this doesn't quite work because we package
440 # bogus config files in the -scripts versions of installs. Maybe
441 # we should check a hash or something?
442 raise NotImplementedError
443 def researchFilter(self, filename, added, deleted):
445 Allows an application to selectively ignore certain diffstat signatures
446 during research; for example, configuration files will have a very
447 specific set of changes, so ignore them; certain installation files
448 may be removed, etc. Return ``True`` if a diffstat signature should be
452 def researchVerbose(self, filename):
454 Allows an application to exclude certain dirty files from the output
455 report; usually this will just be parametrized files, since those are
456 guaranteed to have changes. Return ``True`` if a file should only
457 be displayed in verbose mode.
459 return filename in self.parametrized_files
461 class ApplicationVersion(object):
462 """Represents an abstract notion of a version for an application, where
463 ``version`` is a :class:`distutils.version.LooseVersion` and
464 ``application`` is a :class:`Application`."""
465 #: The :class:`distutils.version.LooseVersion` of this instance.
467 #: The :class:`Application` of this instance.
469 def __init__(self, version, application):
470 self.version = version
471 self.application = application
475 Returns the name of the git describe tag for the commit the user is
476 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
478 return "%s-%s" % (self.application, self.version)
480 def scripts_tag(self):
482 Returns the name of the Git tag for this version.
484 end = str(self.version).partition('-scripts')[2].partition('-')[0]
485 return "%s-scripts%s" % (self.pristine_tag, end)
487 def pristine_tag(self):
489 Returns the name of the Git tag for the pristine version corresponding
492 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
493 def __cmp__(self, y):
494 return cmp(self.version, y.version)
498 Parses a line from the :term:`versions store` and return
499 :class:`ApplicationVersion`.
501 Use this only for cases when speed is of primary importance;
502 the data in version is unreliable and when possible, you should
503 prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
504 the autoinstall itself for information.
506 The `value` to parse will vary. For old style installs, it
509 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
511 For new style installs, it will look like::
515 name = value.split("/")[-1]
517 if name.find("-") != -1:
518 app, _, version = name.partition("-")
520 # kind of poor, maybe should error. Generally this
521 # will actually result in a not found error
525 raise DeploymentParseError(value)
526 return ApplicationVersion.make(app, version)
528 def make(app, version):
530 Makes/retrieves a singleton :class:`ApplicationVersion` from
531 a``app`` and ``version`` string.
534 # defer to the application for version creation to enforce
536 return applications()[app].makeVersion(version)
538 raise NoSuchApplication(app)
542 Takes a tree of values (implement using nested lists) and
543 transforms them into regular expressions.
547 >>> expand_re(['a', 'b'])
549 >>> expand_re(['*', ['b', 'c']])
552 if isinstance(val, str):
553 return re.escape(val)
555 return '(?:' + '|'.join(map(expand_re, val)) + ')'
557 def make_extractors(seed):
559 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into
560 extractor functions (which take a :class:`wizard.deploy.Deployment`
561 and return the value of the second subpattern of ``regex`` when matched
562 with the contents of ``file``).
564 return util.dictmap(lambda a: filename_regex_extractor(*a), seed)
566 def make_substitutions(seed):
568 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into substitution
569 functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
570 of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
572 return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)
574 # The following two functions are *highly* functional, and I recommend
575 # not touching them unless you know what you're doing.
577 def filename_regex_extractor(file, regex):
579 .. highlight:: haskell
581 Given a relative file name ``file``, a regular expression ``regex``, and a
582 :class:`wizard.deploy.Deployment` extracts a value out of the file in that
583 deployment. This function is curried, so you pass just ``file`` and
584 ``regex``, and then pass ``deployment`` to the resulting function.
586 Its Haskell-style type signature would be::
588 Filename -> Regex -> (Deployment -> String)
590 The regular expression requires a very specific form, essentially ``()()()``
591 (with the second subgroup being the value to extract). These enables
592 the regular expression to be used equivalently with filename
594 .. highlight:: python
596 For convenience purposes, we also accept ``[Filename]``, in which case
597 we use the first entry (index 0). Passing an empty list is invalid.
599 >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
600 >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
601 >>> f(deploy.Deployment("."))
603 >>> os.unlink("test-settings.extractor.ini")
606 The first application of ``regex`` and ``file`` is normally performed
607 at compile-time inside a submodule; the second application is
608 performed at runtime.
610 if not isinstance(file, str):
614 contents = deployment.read(file) # cached
617 match = regex.search(contents)
618 if not match: return None
619 # assumes that the second match is the one we want.
620 return match.group(2)
623 def filename_regex_substitution(key, files, regex):
625 .. highlight:: haskell
627 Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
628 regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
629 performs a substitution of the second subpattern of ``regex``
630 with ``key``. Returns the number of replacements made. This function
631 is curried, so you pass just ``key``, ``files`` and ``regex``, and
632 then pass ``deployment`` to the resulting function.
634 Its Haskell-style type signature would be::
636 Key -> ([File], Regex) -> (Deployment -> IO Int)
638 .. highlight:: python
640 For convenience purposes, we also accept ``Filename``, in which case it is treated
641 as a single item list.
643 >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
644 >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
645 >>> f(deploy.Deployment("."))
647 >>> print open("test-settings.substitution.ini", "r").read()
648 config_var = WIZARD_KEY
649 >>> os.unlink("test-settings.substitution.ini")
651 if isinstance(files, str):
654 base = deployment.location
657 file = os.path.join(base, file)
659 contents = open(file, "r").read()
660 contents, n = regex.subn("\\1" + key + "\\3", contents)
662 open(file, "w").write(contents)
668 def backup_database(outdir, deployment):
670 Generic database backup function for MySQL.
672 # XXX: Change this once deployments support multiple dbs
673 if deployment.application.database == "mysql":
674 return backup_mysql_database(outdir, deployment)
676 raise NotImplementedError
678 def backup_mysql_database(outdir, deployment):
680 Database backups for MySQL using the :command:`mysqldump` utility.
682 outfile = os.path.join(outdir, "db.sql")
684 shell.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment.dsn))
685 shell.call("gzip", "--best", outfile)
686 except shell.CallError as e:
687 raise BackupFailure(e.stderr)
689 def restore_database(backup_dir, deployment):
691 Generic database restoration function for MySQL.
693 # XXX: see backup_database
694 if deployment.application.database == "mysql":
695 return restore_mysql_database(backup_dir, deployment)
697 raise NotImplementedError
699 def restore_mysql_database(backup_dir, deployment):
701 Database restoration for MySQL by piping SQL commands into :command:`mysql`.
703 if not os.path.exists(backup_dir):
704 raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
705 sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
706 shell.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
708 shell.call("mysql", *get_mysql_args(deployment.dsn), stdin=sql)
711 def remove_database(deployment):
713 Generic database removal function. Actually, not so generic because we
714 go and check if we're on scripts and if we are run a different command.
716 if deployment.dsn.host == "sql.mit.edu":
718 shell.call("/mit/scripts/sql/bin/drop-database", deployment.dsn.database)
720 except shell.CallError:
722 engine = sqlalchemy.create_engine(deployment.dsn)
723 engine.execute("DROP DATABASE `%s`" % deployment.dsn.database)
725 def get_mysql_args(dsn):
727 Extracts arguments that would be passed to the command line mysql utility
732 args += ["-h", dsn.host]
734 args += ["-u", dsn.username]
736 args += ["-p" + dsn.password]
737 args += [dsn.database]
740 class Error(wizard.Error):
741 """Generic error class for this module."""
744 class NoRepositoryError(Error):
746 :class:`Application` does not appear to have a Git repository
747 in the normal location.
749 #: The name of the application that does not have a Git repository.
751 def __init__(self, app):
754 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
756 class DeploymentParseError(Error):
758 Could not parse ``value`` from :term:`versions store`.
760 #: The value that failed to parse.
762 #: The location of the autoinstall that threw this variable.
763 #: This should be set by error handling code when it is available.
765 def __init__(self, value):
768 class NoSuchApplication(Error):
770 You attempted to reference a :class:`Application` named
771 ``app``, which is not recognized by Wizard.
773 #: The name of the application that does not exist.
775 #: The location of the autoinstall that threw this variable.
776 #: This should be set by error handling code when it is availble.
778 def __init__(self, app):
781 class Failure(Error):
783 Represents a failure when performing some double-dispatched operation
784 such as an installation or an upgrade. Failure classes are postfixed
785 with Failure, not Error.
789 class InstallFailure(Error):
790 """Installation failed for unknown reason."""
794 ERROR: Installation failed for unknown reason. You can
795 retry the installation by appending --retry to the installation
798 class RecoverableInstallFailure(InstallFailure):
800 Installation failed, but we were able to determine what the
801 error was, and should give the user a second chance if we were
802 running interactively.
804 #: List of the errors that were found.
806 def __init__(self, errors):
811 ERROR: Installation failed due to the following errors: %s
813 You can retry the installation by appending --retry to the
814 installation command.""" % ", ".join(self.errors)
816 class UpgradeFailure(Failure):
817 """Upgrade script failed."""
818 #: String details of failure (possibly stdout or stderr output)
820 def __init__(self, details):
821 self.details = details
825 ERROR: Upgrade script failed, details:
829 class UpgradeVerificationFailure(Failure):
830 """Upgrade script passed, but website wasn't accessible afterwards"""
834 ERROR: Upgrade script passed, but website wasn't accessible afterwards. Check
835 the debug logs for the contents of the page."""
837 class BackupFailure(Failure):
838 """Backup script failed."""
839 #: String details of failure
841 def __init__(self, details):
842 self.details = details
846 ERROR: Backup script failed, details:
850 class RestoreFailure(Failure):
851 """Restore script failed."""
852 #: String details of failure
854 def __init__(self, details):
855 self.details = details
859 ERROR: Restore script failed, details:
863 class RemoveFailure(Failure):
864 """Remove script failed."""
865 #: String details of failure
867 def __init__(self, details):
868 self.details = details
872 ERROR: Remove script failed, details: