2 Plumbing object model for representing applications we want to
3 install. This module does the heavy lifting, but you probably
4 want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
5 You'll need to know how to overload the :class:`Application` class
6 and use some of the functions in this module in order to specify
9 There are some submodules for programming languages that define common
10 functions and data that may be used by applications in that language. See:
12 * :mod:`wizard.app.php`
19 from wizard import deploy, util
20 from wizard.app import *
25 import distutils.version
36 from wizard import resolve, scripts, shell, util
39 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
40 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
41 # these are technically deprecated
42 "advancedpoll", "gallery",
47 """Hash table for looking up string application name to instance"""
50 _applications = dict([(n,Application.make(n)) for n in _application_list ])
54 class Application(object):
56 Represents an application, i.e. mediawiki or phpbb.
59 Many of these methods assume a specific working
60 directory; prefer using the corresponding methods
61 in :class:`wizard.deploy.Deployment` and its subclasses.
63 #: String name of the application
65 #: Dictionary of version strings to :class:`ApplicationVersion`.
66 #: See also :meth:`makeVersion`.
68 #: List of files that need to be modified when parametrizing.
69 #: This is a class-wide constant, and should not normally be modified.
70 parametrized_files = []
71 #: Keys that are used in older versions of the application, but
72 #: not for the most recent version.
73 deprecated_keys = set()
74 #: Keys that we can simply generate random strings for if they're missing
76 #: Dictionary of variable names to extractor functions. These functions
77 #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
78 #: the variable, or ``None`` if it could not be found.
79 #: See also :func:`filename_regex_extractor`.
81 #: Dictionary of variable names to substitution functions. These functions
82 #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
83 #: that an explicit instance of the variable is released with the generic
84 #: ``WIZARD_*`` constant. See also :func:`filename_regex_substitution`.
86 #: Dictionary of file names to a list of resolutions, which are tuples of
87 #: a conflict marker string and a result list. See :mod:`wizard.resolve`
88 #: for more information.
90 #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
91 #: this application requires.
93 #: Name of the database that this application uses, i.e. ``mysql`` or
94 #: ``postgres``. If we end up supporting multiple databases for a single
95 #: application, there should also be a value for this in
96 #: :class:`wizard.deploy.Deployment`; the value here is merely the preferred
99 def __init__(self, name):
103 self._extractors = {}
104 self._substitutions = {}
105 def repository(self, srv_path):
107 Returns the Git repository that would contain this application.
108 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
110 repo = os.path.join(srv_path, self.name + ".git")
111 if not os.path.isdir(repo):
112 repo = os.path.join(srv_path, self.name, ".git")
113 if not os.path.isdir(repo):
114 raise NoRepositoryError(self.name)
116 def makeVersion(self, version):
118 Creates or retrieves the :class:`ApplicationVersion` singleton for the
121 if version not in self.versions:
122 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
123 return self.versions[version]
124 def extract(self, deployment):
126 Extracts wizard variables from a deployment. Default implementation
127 uses :attr:`extractors`.
130 for k,extractor in self.extractors.items():
131 result[k] = extractor(deployment)
132 # XXX: ugh... we have to do quoting
133 for k in self.random_keys:
134 if result[k] is None:
135 result[k] = "'%s'" % ''.join(random.choice(string.letters + string.digits) for i in xrange(30))
137 def dsn(self, deployment):
139 Returns the deployment specific database URL. Uses the override file
140 in :file:`.scripts` if it exists, and otherwise attempt to extract the
141 variables from the source files.
143 Under some cases, the database URL will contain only the database
144 property, and no other values. This indicates that the actual DSN
145 should be determined from the environment.
147 This function might return ``None``.
151 We are allowed to batch these two together, because the full precedence
152 chain for determining the database of an application combines these
153 two together. If this was not the case, we would have to call
154 :meth:`databaseUrlFromOverride` and :meth:`databaseUrlFromExtract` manually.
156 url = self.dsnFromOverride(deployment)
159 return self.dsnFromExtract(deployment)
160 def dsnFromOverride(self, deployment):
162 Extracts database URL from an explicit dsn override file.
165 return sqlalchemy.engine.url.make_url(open(deployment.dsn_file).read().strip())
168 def dsnFromExtract(self, deployment):
170 Extracts database URL from a deployment, and returns them as
171 a :class:`sqlalchemy.engine.url.URL`. Returns ``None`` if we
172 can't figure it out: i.e. the conventional variables are not defined
173 for this application.
175 if not self.database:
177 vars = self.extract(deployment)
178 names = ("WIZARD_DBSERVER", "WIZARD_DBUSER", "WIZARD_DBPASSWORD", "WIZARD_DBNAME")
179 host, user, password, database = (shlex.split(vars[x])[0] if vars[x] is not None else None for x in names)
180 # XXX: You'd have to put support for an explicit different database
182 return sqlalchemy.engine.url.URL(self.database, username=user, password=password, host=host, database=database)
183 def url(self, deployment):
185 Returns the deployment specific web URL. Uses the override file
186 in :file:`.scripts` if it exists, and otherwise attempt to extract
187 the variables from the source files.
189 This function might return ``None``, which indicates we couldn't figure
192 url = self.urlFromOverride(deployment)
195 return self.urlFromExtract(deployment)
196 def urlFromOverride(self, deployment):
198 Extracts URL from explicit url override file.
201 return urlparse.urlparse(open(deployment.url_file).read().strip())
204 def urlFromExtract(self, deployment):
206 Extracts URL from a deployment, and returns ``None`` if we can't
207 figure it out. Default implementation is to fail; we might
208 do something clever with extractable variables in the future.
211 def parametrize(self, deployment, ref_deployment):
213 Takes a generic source checkout and parametrizes it according to the
214 values of ``deployment``. This function operates on the current
215 working directory. ``deployment`` should **not** be the same as the
216 current working directory. Default implementation uses
217 :attr:`parametrized_files` and a simple search and replace on those
220 variables = ref_deployment.extract()
221 for file in self.parametrized_files:
223 contents = open(file, "r").read()
226 for key, value in variables.items():
227 if value is None: continue
228 contents = contents.replace(key, value)
231 def resolveConflicts(self, deployment):
233 Resolves conflicted files in the current working directory. Returns
234 whether or not all conflicted files were resolved or not. Fully
235 resolved files are added to the index, but no commit is made. The
236 default implementation uses :attr:`resolutions`.
240 for status in sh.eval("git", "ls-files", "--unmerged").splitlines():
241 file = status.split()[-1]
242 if file in self.resolutions:
243 contents = open(file, "r").read()
244 for spec, result in self.resolutions[file]:
245 old_contents = contents
246 contents = resolve.resolve(contents, spec, result)
247 if old_contents != contents:
248 logging.info("Did resolution with spec:\n" + spec)
249 open(file, "w").write(contents)
250 if not resolve.is_conflict(contents):
251 sh.call("git", "add", file)
257 def prepareMerge(self, deployment):
259 Performs various edits to files in the current working directory in
260 order to make a merge go more smoothly. This is usually
261 used to fix botched line-endings. If you add new files,
262 you have to 'git add' them; this is not necessary for edits.
263 By default this is a no-op; subclasses should replace this
264 with useful behavior.
267 def prepareConfig(self, deployment):
269 Takes a deployment and replaces any explicit instances
270 of a configuration variable with generic ``WIZARD_*`` constants.
271 The default implementation uses :attr:`substitutions`, and
272 emits warnings when it encounters keys in :attr:`deprecated_keys`.
274 for key, subst in self.substitutions.items():
275 subs = subst(deployment)
276 if not subs and key not in self.deprecated_keys:
277 logging.warning("No substitutions for %s" % key)
278 def install(self, version, options):
280 Run for 'wizard configure' (and, by proxy, 'wizard install') to
281 configure an application. This assumes that the current working
282 directory is a deployment. (Unlike its kin, this function does not
283 take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should
284 provide an implementation.
286 raise NotImplementedError
287 def upgrade(self, deployment, version, options):
289 Run for 'wizard upgrade' to upgrade database schemas and other
290 non-versioned data in an application after the filesystem has been
291 upgraded. This assumes that the current working directory is the
292 deployment. Subclasses should provide an implementation.
294 raise NotImplementedError
295 def backup(self, deployment, outdir, options):
297 Run for 'wizard backup' and upgrades to backup database schemas
298 and other non-versioned data in an application. ``outdir`` is
299 the directory that backup files should be placed. This assumes
300 that the current working directory is the deployment. Subclasses
301 should provide an implementation, even if it is a no-op.
304 Static user files may not need to be backed up, since in
305 many applications upgrades do not modify static files.
307 raise NotImplementedError
308 def restore(self, deployment, backup_dir, options):
310 Run for 'wizard restore' and failed upgrades to restore database
311 and other non-versioned data to a backed up version. This assumes
312 that the current working directory is the deployment. Subclasses
313 should provide an implementation.
315 raise NotImplementedError
316 def remove(self, deployment, options):
318 Run for 'wizard remove' to delete all database and non-local
319 file data. This assumes that the current working directory is
320 the deployment. Subclasses should provide an implementation.
322 raise NotImplementedError
323 def detectVersion(self, deployment):
325 Checks source files to determine the version manually. This assumes
326 that the current working directory is the deployment. Subclasses
327 should provide an implementation.
329 raise NotImplementedError
330 def detectVersionFromFile(self, filename, regex):
332 Helper method that detects a version by using a regular expression
333 from a file. The regexed value is passed through :mod:`shlex`.
334 This assumes that the current working directory is the deployment.
336 contents = open(filename).read()
337 match = regex.search(contents)
338 if not match: return None
339 return distutils.version.LooseVersion(shlex.split(match.group(2))[0])
340 def download(self, version):
342 Returns a URL that can be used to download a tarball of ``version`` of
345 raise NotImplementedError
346 def checkWeb(self, deployment):
348 Checks if the autoinstall is viewable from the web. Subclasses should
349 provide an implementation.
352 Finding a reasonable heuristic that works across skinning
353 choices can be difficult. We've had reasonable success
354 searching for metadata. Be sure that the standard error
355 page does not contain the features you search for. Try
356 not to depend on pages that are not the main page.
358 raise NotImplementedError
359 def checkWebPage(self, deployment, page, output):
361 Checks if a given page of an autoinstall contains a particular string.
363 page = deployment.fetch(page)
364 result = page.find(output) != -1
366 logging.debug("checkWebPage (passed):\n\n" + page)
368 logging.info("checkWebPage (failed):\n\n" + page)
370 def checkConfig(self, deployment):
372 Checks whether or not an autoinstall has been configured/installed
373 for use. Assumes that the current working directory is the deployment.
374 Subclasses should provide an implementation.
376 # XXX: Unfortunately, this doesn't quite work because we package
377 # bogus config files in the -scripts versions of installs. Maybe
378 # we should check a hash or something?
379 raise NotImplementedError
382 """Makes an application, but uses the correct subtype if available."""
384 __import__("wizard.app." + name)
385 return getattr(wizard.app, name).Application(name)
386 except ImportError as error:
387 # XXX ugly hack to check if the import error is from the top level
388 # module we care about or a submodule. should be an archetectural change.
389 if error.args[0].split()[-1]==name:
390 return Application(name)
394 class ApplicationVersion(object):
395 """Represents an abstract notion of a version for an application, where
396 ``version`` is a :class:`distutils.version.LooseVersion` and
397 ``application`` is a :class:`Application`."""
398 #: The :class:`distutils.version.LooseVersion` of this instance.
400 #: The :class:`Application` of this instance.
402 def __init__(self, version, application):
403 self.version = version
404 self.application = application
408 Returns the name of the git describe tag for the commit the user is
409 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
411 return "%s-%s" % (self.application, self.version)
413 def scripts_tag(self):
415 Returns the name of the Git tag for this version.
417 end = str(self.version).partition('-scripts')[2].partition('-')[0]
418 return "%s-scripts%s" % (self.pristine_tag, end)
420 def pristine_tag(self):
422 Returns the name of the Git tag for the pristine version corresponding
425 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
426 def __cmp__(self, y):
427 return cmp(self.version, y.version)
431 Parses a line from the :term:`versions store` and return
432 :class:`ApplicationVersion`.
434 Use this only for cases when speed is of primary importance;
435 the data in version is unreliable and when possible, you should
436 prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
437 the autoinstall itself for information.
439 The `value` to parse will vary. For old style installs, it
442 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
444 For new style installs, it will look like::
448 name = value.split("/")[-1]
450 if name.find("-") != -1:
451 app, _, version = name.partition("-")
453 # kind of poor, maybe should error. Generally this
454 # will actually result in a not found error
458 raise DeploymentParseError(value)
459 return ApplicationVersion.make(app, version)
461 def make(app, version):
463 Makes/retrieves a singleton :class:`ApplicationVersion` from
464 a``app`` and ``version`` string.
467 # defer to the application for version creation to enforce
469 return applications()[app].makeVersion(version)
471 raise NoSuchApplication(app)
475 Takes a tree of values (implement using nested lists) and
476 transforms them into regular expressions.
480 >>> expand_re(['a', 'b'])
482 >>> expand_re(['*', ['b', 'c']])
485 if isinstance(val, str):
486 return re.escape(val)
488 return '(?:' + '|'.join(map(expand_re, val)) + ')'
490 def make_extractors(seed):
492 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into
493 extractor functions (which take a :class:`wizard.deploy.Deployment`
494 and return the value of the second subpattern of ``regex`` when matched
495 with the contents of ``file``).
497 return util.dictmap(lambda a: filename_regex_extractor(*a), seed)
499 def make_substitutions(seed):
501 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into substitution
502 functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
503 of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
505 return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)
507 # The following two functions are *highly* functional, and I recommend
508 # not touching them unless you know what you're doing.
510 def filename_regex_extractor(file, regex):
512 .. highlight:: haskell
514 Given a relative file name ``file``, a regular expression ``regex``, and a
515 :class:`wizard.deploy.Deployment` extracts a value out of the file in that
516 deployment. This function is curried, so you pass just ``file`` and
517 ``regex``, and then pass ``deployment`` to the resulting function.
519 Its Haskell-style type signature would be::
521 Filename -> Regex -> (Deployment -> String)
523 The regular expression requires a very specific form, essentially ``()()()``
524 (with the second subgroup being the value to extract). These enables
525 the regular expression to be used equivalently with filename
527 .. highlight:: python
529 For convenience purposes, we also accept ``[Filename]``, in which case
530 we use the first entry (index 0). Passing an empty list is invalid.
532 >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
533 >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
534 >>> f(deploy.Deployment("."))
536 >>> os.unlink("test-settings.extractor.ini")
539 The first application of ``regex`` and ``file`` is normally performed
540 at compile-time inside a submodule; the second application is
541 performed at runtime.
543 if not isinstance(file, str):
547 contents = deployment.read(file) # cached
550 match = regex.search(contents)
551 if not match: return None
552 # assumes that the second match is the one we want.
553 return match.group(2)
556 def filename_regex_substitution(key, files, regex):
558 .. highlight:: haskell
560 Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
561 regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
562 performs a substitution of the second subpattern of ``regex``
563 with ``key``. Returns the number of replacements made. This function
564 is curried, so you pass just ``key``, ``files`` and ``regex``, and
565 then pass ``deployment`` to the resulting function.
567 Its Haskell-style type signature would be::
569 Key -> ([File], Regex) -> (Deployment -> IO Int)
571 .. highlight:: python
573 For convenience purposes, we also accept ``Filename``, in which case it is treated
574 as a single item list.
576 >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
577 >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
578 >>> f(deploy.Deployment("."))
580 >>> print open("test-settings.substitution.ini", "r").read()
581 config_var = WIZARD_KEY
582 >>> os.unlink("test-settings.substitution.ini")
584 if isinstance(files, str):
587 base = deployment.location
590 file = os.path.join(base, file)
592 contents = open(file, "r").read()
593 contents, n = regex.subn("\\1" + key + "\\3", contents)
595 open(file, "w").write(contents)
601 def backup_database(outdir, deployment):
603 Generic database backup function for MySQL.
605 # XXX: Change this once deployments support multiple dbs
606 if deployment.application.database == "mysql":
607 return backup_mysql_database(outdir, deployment)
609 raise NotImplementedError
611 def backup_mysql_database(outdir, deployment):
613 Database backups for MySQL using the :command:`mysqldump` utility.
616 outfile = os.path.join(outdir, "db.sql")
618 sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment.dsn))
619 sh.call("gzip", "--best", outfile)
620 except shell.CallError as e:
621 shutil.rmtree(outdir)
622 raise BackupFailure(e.stderr)
624 def restore_database(backup_dir, deployment):
626 Generic database restoration function for MySQL.
628 # XXX: see backup_database
629 if deployment.application.database == "mysql":
630 return restore_mysql_database(backup_dir, deployment)
632 raise NotImplementedError
634 def restore_mysql_database(backup_dir, deployment):
636 Database restoration for MySQL by piping SQL commands into :command:`mysql`.
639 if not os.path.exists(backup_dir):
640 raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
641 sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
642 sh.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
644 sh.call("mysql", *get_mysql_args(deployment.dsn), stdin=sql)
647 def remove_database(deployment):
649 Generic database removal function. Actually, not so generic because we
650 go and check if we're on scripts and if we are run a different command.
653 if deployment.dsn.host == "sql.mit.edu":
655 sh.call("/mit/scripts/sql/bin/drop-database", deployment.dsn.database)
657 except shell.CallError:
659 engine = sqlalchemy.create_engine(deployment.dsn)
660 engine.execute("DROP DATABASE `%s`" % deployment.dsn.database)
662 def get_mysql_args(dsn):
664 Extracts arguments that would be passed to the command line mysql utility
669 args += ["-h", dsn.host]
671 args += ["-u", dsn.username]
673 args += ["-p" + dsn.password]
674 args += [dsn.database]
677 class Error(wizard.Error):
678 """Generic error class for this module."""
681 class NoRepositoryError(Error):
683 :class:`Application` does not appear to have a Git repository
684 in the normal location.
686 #: The name of the application that does not have a Git repository.
688 def __init__(self, app):
691 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
693 class DeploymentParseError(Error):
695 Could not parse ``value`` from :term:`versions store`.
697 #: The value that failed to parse.
699 #: The location of the autoinstall that threw this variable.
700 #: This should be set by error handling code when it is available.
702 def __init__(self, value):
705 class NoSuchApplication(Error):
707 You attempted to reference a :class:`Application` named
708 ``app``, which is not recognized by Wizard.
710 #: The name of the application that does not exist.
712 #: The location of the autoinstall that threw this variable.
713 #: This should be set by error handling code when it is availble.
715 def __init__(self, app):
718 class Failure(Error):
720 Represents a failure when performing some double-dispatched operation
721 such as an installation or an upgrade. Failure classes are postfixed
722 with Failure, not Error.
726 class InstallFailure(Error):
727 """Installation failed for unknown reason."""
731 ERROR: Installation failed for unknown reason. You can
732 retry the installation by appending --retry to the installation
735 class RecoverableInstallFailure(InstallFailure):
737 Installation failed, but we were able to determine what the
738 error was, and should give the user a second chance if we were
739 running interactively.
741 #: List of the errors that were found.
743 def __init__(self, errors):
748 ERROR: Installation failed due to the following errors: %s
750 You can retry the installation by appending --retry to the
751 installation command.""" % ", ".join(self.errors)
753 class UpgradeFailure(Failure):
754 """Upgrade script failed."""
755 #: String details of failure (possibly stdout or stderr output)
757 def __init__(self, details):
758 self.details = details
762 ERROR: Upgrade script failed, details:
766 class UpgradeVerificationFailure(Failure):
767 """Upgrade script passed, but website wasn't accessible afterwards"""
771 ERROR: Upgrade script passed, but website wasn't accessible afterwards. Check
772 the debug logs for the contents of the page."""
774 class BackupFailure(Failure):
775 """Backup script failed."""
776 #: String details of failure
778 def __init__(self, details):
779 self.details = details
783 ERROR: Backup script failed, details:
787 class RestoreFailure(Failure):
788 """Restore script failed."""
789 #: String details of failure
791 def __init__(self, details):
792 self.details = details
796 ERROR: Restore script failed, details:
800 class RemoveFailure(Failure):
801 """Remove script failed."""
802 #: String details of failure
804 def __init__(self, details):
805 self.details = details
809 ERROR: Remove script failed, details: