2 Plumbing object model for representing applications we want to
3 install. This module does the heavy lifting, but you probably
4 want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
5 You'll need to know how to overload the :class:`Application` class
6 and use some of the functions in this module in order to specify
9 There are some submodules for programming languages that define common
10 functions and data that may be used by applications in that language. See:
12 * :mod:`wizard.app.php`
19 from wizard import deploy, util
20 from wizard.app import *
25 import distutils.version
37 from wizard import resolve, scripts, shell, util
40 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
41 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
42 # these are technically deprecated
43 "advancedpoll", "gallery",
48 """Hash table for looking up string application name to instance"""
51 _applications = dict([(n,Application.make(n)) for n in _application_list ])
54 def getApplication(appname):
55 """Retrieves application instance given a name"""
56 return applications()[appname]
58 class Application(object):
60 Represents an application, i.e. mediawiki or phpbb.
63 Many of these methods assume a specific working
64 directory; prefer using the corresponding methods
65 in :class:`wizard.deploy.Deployment` and its subclasses.
67 #: String name of the application
69 #: Dictionary of version strings to :class:`ApplicationVersion`.
70 #: See also :meth:`makeVersion`.
72 #: List of files that need to be modified when parametrizing.
73 #: This is a class-wide constant, and should not normally be modified.
74 parametrized_files = []
75 #: Keys that are used in older versions of the application, but
76 #: not for the most recent version.
77 deprecated_keys = set()
78 #: Keys that we can simply generate random strings for if they're missing
80 #: Dictionary of variable names to extractor functions. These functions
81 #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
82 #: the variable, or ``None`` if it could not be found.
83 #: See also :func:`filename_regex_extractor`.
85 #: Dictionary of variable names to substitution functions. These functions
86 #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
87 #: that an explicit instance of the variable is released with the generic
88 #: ``WIZARD_*`` constant. See also :func:`filename_regex_substitution`.
90 #: Dictionary of file names to a list of resolutions, which are tuples of
91 #: a conflict marker string and a result list. See :mod:`wizard.resolve`
92 #: for more information.
94 #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
95 #: this application requires.
97 #: Name of the database that this application uses, i.e. ``mysql`` or
98 #: ``postgres``. If we end up supporting multiple databases for a single
99 #: application, there should also be a value for this in
100 #: :class:`wizard.deploy.Deployment`; the value here is merely the preferred
103 def __init__(self, name):
107 self._extractors = {}
108 self._substitutions = {}
109 def repository(self, srv_path):
111 Returns the Git repository that would contain this application.
112 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
114 repo = os.path.join(srv_path, self.name + ".git")
115 if not os.path.isdir(repo):
116 repo = os.path.join(srv_path, self.name, ".git")
117 if not os.path.isdir(repo):
118 raise NoRepositoryError(self.name)
120 def makeVersion(self, version):
122 Creates or retrieves the :class:`ApplicationVersion` singleton for the
125 if version not in self.versions:
126 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
127 return self.versions[version]
128 def extract(self, deployment):
130 Extracts wizard variables from a deployment. Default implementation
131 uses :attr:`extractors`.
134 for k,extractor in self.extractors.items():
135 result[k] = extractor(deployment)
136 # XXX: ugh... we have to do quoting
137 for k in self.random_keys:
138 if result[k] is None:
139 result[k] = "'%s'" % ''.join(random.choice(string.letters + string.digits) for i in xrange(30))
141 def dsn(self, deployment):
143 Returns the deployment specific database URL. Uses the override file
144 in :file:`.scripts` if it exists, and otherwise attempt to extract the
145 variables from the source files.
147 Under some cases, the database URL will contain only the database
148 property, and no other values. This indicates that the actual DSN
149 should be determined from the environment.
151 This function might return ``None``.
155 We are allowed to batch these two together, because the full precedence
156 chain for determining the database of an application combines these
157 two together. If this was not the case, we would have to call
158 :meth:`databaseUrlFromOverride` and :meth:`databaseUrlFromExtract` manually.
160 url = self.dsnFromOverride(deployment)
163 return self.dsnFromExtract(deployment)
164 def dsnFromOverride(self, deployment):
166 Extracts database URL from an explicit dsn override file.
169 return sqlalchemy.engine.url.make_url(open(deployment.dsn_file).read().strip())
172 def dsnFromExtract(self, deployment):
174 Extracts database URL from a deployment, and returns them as
175 a :class:`sqlalchemy.engine.url.URL`. Returns ``None`` if we
176 can't figure it out: i.e. the conventional variables are not defined
177 for this application.
179 if not self.database:
181 vars = self.extract(deployment)
182 names = ("WIZARD_DBSERVER", "WIZARD_DBUSER", "WIZARD_DBPASSWORD", "WIZARD_DBNAME")
183 host, user, password, database = (shlex.split(vars[x])[0] if vars[x] is not None else None for x in names)
184 # XXX: You'd have to put support for an explicit different database
186 return sqlalchemy.engine.url.URL(self.database, username=user, password=password, host=host, database=database)
187 def url(self, deployment):
189 Returns the deployment specific web URL. Uses the override file
190 in :file:`.scripts` if it exists, and otherwise attempt to extract
191 the variables from the source files.
193 This function might return ``None``, which indicates we couldn't figure
196 url = self.urlFromOverride(deployment)
199 return self.urlFromExtract(deployment)
200 def urlFromOverride(self, deployment):
202 Extracts URL from explicit url override file.
205 return urlparse.urlparse(open(deployment.url_file).read().strip())
208 def urlFromExtract(self, deployment):
210 Extracts URL from a deployment, and returns ``None`` if we can't
211 figure it out. Default implementation is to fail; we might
212 do something clever with extractable variables in the future.
215 def parametrize(self, deployment, ref_deployment):
217 Takes a generic source checkout and parametrizes it according to the
218 values of ``deployment``. This function operates on the current
219 working directory. ``deployment`` should **not** be the same as the
220 current working directory. Default implementation uses
221 :attr:`parametrized_files` and a simple search and replace on those
224 variables = ref_deployment.extract()
225 for file in self.parametrized_files:
227 contents = open(file, "r").read()
230 for key, value in variables.items():
231 if value is None: continue
232 contents = contents.replace(key, value)
235 def resolveConflicts(self, deployment):
237 Resolves conflicted files in the current working directory. Returns
238 whether or not all conflicted files were resolved or not. Fully
239 resolved files are added to the index, but no commit is made. The
240 default implementation uses :attr:`resolutions`.
244 for status in shell.eval("git", "ls-files", "--unmerged").splitlines():
245 files.add(status.split()[-1])
248 if file in self.resolutions:
249 contents = open(file, "r").read()
250 for spec, result in self.resolutions[file]:
251 old_contents = contents
252 contents = resolve.resolve(contents, spec, result)
253 if old_contents != contents:
254 logging.info("Did resolution with spec:\n" + spec)
255 open(file, "w").write(contents)
256 if not resolve.is_conflict(contents):
257 shell.call("git", "add", file)
263 def prepareMerge(self, deployment):
265 Performs various edits to files in the current working directory in
266 order to make a merge go more smoothly. This is usually
267 used to fix botched line-endings. If you add new files,
268 you have to 'git add' them; this is not necessary for edits.
269 By default this is a no-op; subclasses should replace this
270 with useful behavior.
273 def prepareConfig(self, deployment):
275 Takes a deployment and replaces any explicit instances
276 of a configuration variable with generic ``WIZARD_*`` constants.
277 The default implementation uses :attr:`substitutions`, and
278 emits warnings when it encounters keys in :attr:`deprecated_keys`.
280 for key, subst in self.substitutions.items():
281 subs = subst(deployment)
282 if not subs and key not in self.deprecated_keys:
283 logging.warning("No substitutions for %s" % key)
284 def install(self, version, options):
286 Run for 'wizard configure' (and, by proxy, 'wizard install') to
287 configure an application. This assumes that the current working
288 directory is a deployment. (Unlike its kin, this function does not
289 take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should
290 provide an implementation.
292 raise NotImplementedError
293 def upgrade(self, deployment, version, options):
295 Run for 'wizard upgrade' to upgrade database schemas and other
296 non-versioned data in an application after the filesystem has been
297 upgraded. This assumes that the current working directory is the
298 deployment. Subclasses should provide an implementation.
300 raise NotImplementedError
301 def backup(self, deployment, outdir, options):
303 Run for 'wizard backup' and upgrades to backup database schemas
304 and other non-versioned data in an application. ``outdir`` is
305 the directory that backup files should be placed. This assumes
306 that the current working directory is the deployment. Subclasses
307 should provide an implementation, even if it is a no-op.
310 Static user files may not need to be backed up, since in
311 many applications upgrades do not modify static files.
313 raise NotImplementedError
314 def restore(self, deployment, backup_dir, options):
316 Run for 'wizard restore' and failed upgrades to restore database
317 and other non-versioned data to a backed up version. This assumes
318 that the current working directory is the deployment. Subclasses
319 should provide an implementation.
321 raise NotImplementedError
322 def remove(self, deployment, options):
324 Run for 'wizard remove' to delete all database and non-local
325 file data. This assumes that the current working directory is
326 the deployment. Subclasses should provide an implementation.
328 raise NotImplementedError
329 def detectVersion(self, deployment):
331 Checks source files to determine the version manually. This assumes
332 that the current working directory is the deployment. Subclasses
333 should provide an implementation.
335 raise NotImplementedError
336 def detectVersionFromFile(self, filename, regex):
338 Helper method that detects a version by using a regular expression
339 from a file. The regexed value is passed through :mod:`shlex`.
340 This assumes that the current working directory is the deployment.
342 contents = open(filename).read()
343 match = regex.search(contents)
344 if not match: return None
345 return distutils.version.LooseVersion(shlex.split(match.group(2))[0])
346 def download(self, version):
348 Returns a URL that can be used to download a tarball of ``version`` of
351 raise NotImplementedError
352 def checkWeb(self, deployment):
354 Checks if the autoinstall is viewable from the web. Subclasses should
355 provide an implementation.
358 Finding a reasonable heuristic that works across skinning
359 choices can be difficult. We've had reasonable success
360 searching for metadata. Be sure that the standard error
361 page does not contain the features you search for. Try
362 not to depend on pages that are not the main page.
364 raise NotImplementedError
365 def checkWebPage(self, deployment, page, output):
367 Checks if a given page of an autoinstall contains a particular string.
369 page = deployment.fetch(page)
370 result = page.find(output) != -1
372 logging.debug("checkWebPage (passed):\n\n" + page)
374 logging.info("checkWebPage (failed):\n\n" + page)
376 def checkConfig(self, deployment):
378 Checks whether or not an autoinstall has been configured/installed
379 for use. Assumes that the current working directory is the deployment.
380 Subclasses should provide an implementation.
382 # XXX: Unfortunately, this doesn't quite work because we package
383 # bogus config files in the -scripts versions of installs. Maybe
384 # we should check a hash or something?
385 raise NotImplementedError
386 def researchFilter(self, filename, added, deleted):
388 Allows an application to selectively ignore certain diffstat signatures
389 during research; for example, configuration files will have a very
390 specific set of changes, so ignore them; certain installation files
391 may be removed, etc. Return ``True`` if a diffstat signature should be
395 def researchVerbose(self, filename):
397 Allows an application to exclude certain dirty files from the output
398 report; usually this will just be parametrized files, since those are
399 guaranteed to have changes. Return ``True`` if a file should only
400 be displayed in verbose mode.
402 return filename in self.parametrized_files
405 """Makes an application, but uses the correct subtype if available."""
407 __import__("wizard.app." + name)
408 return getattr(wizard.app, name).Application(name)
409 except ImportError as error:
410 # XXX ugly hack to check if the import error is from the top level
411 # module we care about or a submodule. should be an archetectural change.
412 if error.args[0].split()[-1]==name:
413 return Application(name)
417 class ApplicationVersion(object):
418 """Represents an abstract notion of a version for an application, where
419 ``version`` is a :class:`distutils.version.LooseVersion` and
420 ``application`` is a :class:`Application`."""
421 #: The :class:`distutils.version.LooseVersion` of this instance.
423 #: The :class:`Application` of this instance.
425 def __init__(self, version, application):
426 self.version = version
427 self.application = application
431 Returns the name of the git describe tag for the commit the user is
432 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
434 return "%s-%s" % (self.application, self.version)
436 def scripts_tag(self):
438 Returns the name of the Git tag for this version.
440 end = str(self.version).partition('-scripts')[2].partition('-')[0]
441 return "%s-scripts%s" % (self.pristine_tag, end)
443 def pristine_tag(self):
445 Returns the name of the Git tag for the pristine version corresponding
448 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
449 def __cmp__(self, y):
450 return cmp(self.version, y.version)
454 Parses a line from the :term:`versions store` and return
455 :class:`ApplicationVersion`.
457 Use this only for cases when speed is of primary importance;
458 the data in version is unreliable and when possible, you should
459 prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
460 the autoinstall itself for information.
462 The `value` to parse will vary. For old style installs, it
465 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
467 For new style installs, it will look like::
471 name = value.split("/")[-1]
473 if name.find("-") != -1:
474 app, _, version = name.partition("-")
476 # kind of poor, maybe should error. Generally this
477 # will actually result in a not found error
481 raise DeploymentParseError(value)
482 return ApplicationVersion.make(app, version)
484 def make(app, version):
486 Makes/retrieves a singleton :class:`ApplicationVersion` from
487 a``app`` and ``version`` string.
490 # defer to the application for version creation to enforce
492 return applications()[app].makeVersion(version)
494 raise NoSuchApplication(app)
498 Takes a tree of values (implement using nested lists) and
499 transforms them into regular expressions.
503 >>> expand_re(['a', 'b'])
505 >>> expand_re(['*', ['b', 'c']])
508 if isinstance(val, str):
509 return re.escape(val)
511 return '(?:' + '|'.join(map(expand_re, val)) + ')'
513 def make_extractors(seed):
515 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into
516 extractor functions (which take a :class:`wizard.deploy.Deployment`
517 and return the value of the second subpattern of ``regex`` when matched
518 with the contents of ``file``).
520 return util.dictmap(lambda a: filename_regex_extractor(*a), seed)
522 def make_substitutions(seed):
524 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into substitution
525 functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
526 of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
528 return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)
530 # The following two functions are *highly* functional, and I recommend
531 # not touching them unless you know what you're doing.
533 def filename_regex_extractor(file, regex):
535 .. highlight:: haskell
537 Given a relative file name ``file``, a regular expression ``regex``, and a
538 :class:`wizard.deploy.Deployment` extracts a value out of the file in that
539 deployment. This function is curried, so you pass just ``file`` and
540 ``regex``, and then pass ``deployment`` to the resulting function.
542 Its Haskell-style type signature would be::
544 Filename -> Regex -> (Deployment -> String)
546 The regular expression requires a very specific form, essentially ``()()()``
547 (with the second subgroup being the value to extract). These enables
548 the regular expression to be used equivalently with filename
550 .. highlight:: python
552 For convenience purposes, we also accept ``[Filename]``, in which case
553 we use the first entry (index 0). Passing an empty list is invalid.
555 >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
556 >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
557 >>> f(deploy.Deployment("."))
559 >>> os.unlink("test-settings.extractor.ini")
562 The first application of ``regex`` and ``file`` is normally performed
563 at compile-time inside a submodule; the second application is
564 performed at runtime.
566 if not isinstance(file, str):
570 contents = deployment.read(file) # cached
573 match = regex.search(contents)
574 if not match: return None
575 # assumes that the second match is the one we want.
576 return match.group(2)
579 def filename_regex_substitution(key, files, regex):
581 .. highlight:: haskell
583 Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
584 regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
585 performs a substitution of the second subpattern of ``regex``
586 with ``key``. Returns the number of replacements made. This function
587 is curried, so you pass just ``key``, ``files`` and ``regex``, and
588 then pass ``deployment`` to the resulting function.
590 Its Haskell-style type signature would be::
592 Key -> ([File], Regex) -> (Deployment -> IO Int)
594 .. highlight:: python
596 For convenience purposes, we also accept ``Filename``, in which case it is treated
597 as a single item list.
599 >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
600 >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
601 >>> f(deploy.Deployment("."))
603 >>> print open("test-settings.substitution.ini", "r").read()
604 config_var = WIZARD_KEY
605 >>> os.unlink("test-settings.substitution.ini")
607 if isinstance(files, str):
610 base = deployment.location
613 file = os.path.join(base, file)
615 contents = open(file, "r").read()
616 contents, n = regex.subn("\\1" + key + "\\3", contents)
618 open(file, "w").write(contents)
624 def backup_database(outdir, deployment):
626 Generic database backup function for MySQL.
628 # XXX: Change this once deployments support multiple dbs
629 if deployment.application.database == "mysql":
630 return backup_mysql_database(outdir, deployment)
632 raise NotImplementedError
634 def backup_mysql_database(outdir, deployment):
636 Database backups for MySQL using the :command:`mysqldump` utility.
638 outfile = os.path.join(outdir, "db.sql")
640 shell.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment.dsn))
641 shell.call("gzip", "--best", outfile)
642 except shell.CallError as e:
643 raise BackupFailure(e.stderr)
645 def restore_database(backup_dir, deployment):
647 Generic database restoration function for MySQL.
649 # XXX: see backup_database
650 if deployment.application.database == "mysql":
651 return restore_mysql_database(backup_dir, deployment)
653 raise NotImplementedError
655 def restore_mysql_database(backup_dir, deployment):
657 Database restoration for MySQL by piping SQL commands into :command:`mysql`.
659 if not os.path.exists(backup_dir):
660 raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
661 sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
662 shell.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
664 shell.call("mysql", *get_mysql_args(deployment.dsn), stdin=sql)
667 def remove_database(deployment):
669 Generic database removal function. Actually, not so generic because we
670 go and check if we're on scripts and if we are run a different command.
672 if deployment.dsn.host == "sql.mit.edu":
674 shell.call("/mit/scripts/sql/bin/drop-database", deployment.dsn.database)
676 except shell.CallError:
678 engine = sqlalchemy.create_engine(deployment.dsn)
679 engine.execute("DROP DATABASE `%s`" % deployment.dsn.database)
681 def get_mysql_args(dsn):
683 Extracts arguments that would be passed to the command line mysql utility
688 args += ["-h", dsn.host]
690 args += ["-u", dsn.username]
692 args += ["-p" + dsn.password]
693 args += [dsn.database]
696 class Error(wizard.Error):
697 """Generic error class for this module."""
700 class NoRepositoryError(Error):
702 :class:`Application` does not appear to have a Git repository
703 in the normal location.
705 #: The name of the application that does not have a Git repository.
707 def __init__(self, app):
710 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
712 class DeploymentParseError(Error):
714 Could not parse ``value`` from :term:`versions store`.
716 #: The value that failed to parse.
718 #: The location of the autoinstall that threw this variable.
719 #: This should be set by error handling code when it is available.
721 def __init__(self, value):
724 class NoSuchApplication(Error):
726 You attempted to reference a :class:`Application` named
727 ``app``, which is not recognized by Wizard.
729 #: The name of the application that does not exist.
731 #: The location of the autoinstall that threw this variable.
732 #: This should be set by error handling code when it is availble.
734 def __init__(self, app):
737 class Failure(Error):
739 Represents a failure when performing some double-dispatched operation
740 such as an installation or an upgrade. Failure classes are postfixed
741 with Failure, not Error.
745 class InstallFailure(Error):
746 """Installation failed for unknown reason."""
750 ERROR: Installation failed for unknown reason. You can
751 retry the installation by appending --retry to the installation
754 class RecoverableInstallFailure(InstallFailure):
756 Installation failed, but we were able to determine what the
757 error was, and should give the user a second chance if we were
758 running interactively.
760 #: List of the errors that were found.
762 def __init__(self, errors):
767 ERROR: Installation failed due to the following errors: %s
769 You can retry the installation by appending --retry to the
770 installation command.""" % ", ".join(self.errors)
772 class UpgradeFailure(Failure):
773 """Upgrade script failed."""
774 #: String details of failure (possibly stdout or stderr output)
776 def __init__(self, details):
777 self.details = details
781 ERROR: Upgrade script failed, details:
785 class UpgradeVerificationFailure(Failure):
786 """Upgrade script passed, but website wasn't accessible afterwards"""
790 ERROR: Upgrade script passed, but website wasn't accessible afterwards. Check
791 the debug logs for the contents of the page."""
793 class BackupFailure(Failure):
794 """Backup script failed."""
795 #: String details of failure
797 def __init__(self, details):
798 self.details = details
802 ERROR: Backup script failed, details:
806 class RestoreFailure(Failure):
807 """Restore script failed."""
808 #: String details of failure
810 def __init__(self, details):
811 self.details = details
815 ERROR: Restore script failed, details:
819 class RemoveFailure(Failure):
820 """Remove script failed."""
821 #: String details of failure
823 def __init__(self, details):
824 self.details = details
828 ERROR: Remove script failed, details: