2 Plumbing object model for representing applications we want to
3 install. This module does the heavy lifting, but you probably
4 want to use :class:`wizard.deploy.Deployment` which is more user-friendly.
5 You'll need to know how to overload the :class:`Application` class
6 and use some of the functions in this module in order to specify
9 To specify custom applications as plugins, add the following ``entry_points``
13 yourappname = your.module:Application
14 otherappname = your.other.module:Application
18 Wizard will complain loudly if ``yourappname`` conflicts with an
19 application name defined by someone else.
21 There are some submodules for programming languages that define common
22 functions and data that may be used by applications in that language. See:
24 * :mod:`wizard.app.php`
31 from wizard import deploy, util
32 from wizard.app import *
37 import distutils.version
51 from wizard import resolve, scripts, shell, sql, util
54 _scripts_application_list = [
55 "mediawiki", "wordpress", "joomla", "e107", "gallery2",
56 "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
57 # these are technically deprecated
58 "advancedpoll", "gallery",
60 def _scripts_make(name):
61 """Makes an application, but uses the correct subtype if available."""
63 __import__("wizard.app." + name)
64 return getattr(wizard.app, name).Application(name)
65 except ImportError as error:
66 # XXX ugly hack to check if the import error is from the top level
67 # module we care about or a submodule. should be an archetectural change.
68 if error.args[0].split()[-1]==name:
69 return Application(name)
75 """Hash table for looking up string application name to instance"""
79 _applications = dict([(n,_scripts_make(n)) for n in _scripts_application_list ])
81 for dist in pkg_resources.working_set:
82 for appname, appclass in dist.get_entry_map("wizard.app").items():
83 if appname in _applications:
84 newname = dist.key + ":" + appname
85 if newname in _applications:
86 raise Exception("Unrecoverable application name conflict for %s from %s", appname, dist.key)
87 logging.warning("Could not overwrite %s, used %s instead", appname, newname)
89 _applications[appname] = appclass(appname)
92 def getApplication(appname):
93 """Retrieves application instance given a name"""
94 return applications()[appname]
96 class Application(object):
98 Represents an application, i.e. mediawiki or phpbb.
101 Many of these methods assume a specific working
102 directory; prefer using the corresponding methods
103 in :class:`wizard.deploy.Deployment` and its subclasses.
105 #: String name of the application
107 #: Dictionary of version strings to :class:`ApplicationVersion`.
108 #: See also :meth:`makeVersion`.
110 #: List of files that need to be modified when parametrizing.
111 #: This is a class-wide constant, and should not normally be modified.
112 parametrized_files = []
113 #: Keys that are used in older versions of the application, but
114 #: not for the most recent version.
115 deprecated_keys = set()
116 #: Keys that we can simply generate random strings for if they're missing
118 #: Dictionary of variable names to extractor functions. These functions
119 #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of
120 #: the variable, or ``None`` if it could not be found.
121 #: See also :func:`filename_regex_extractor`.
123 #: Dictionary of variable names to substitution functions. These functions
124 #: take a :class:`wizard.deploy.Deployment` as an argument and modify the deployment such
125 #: that an explicit instance of the variable is released with the generic
126 #: ``WIZARD_*`` constant. See also :func:`filename_regex_substitution`.
128 #: Dictionary of file names to a list of resolutions, which are tuples of
129 #: a conflict marker string and a result list. See :mod:`wizard.resolve`
130 #: for more information.
132 #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments
133 #: this application requires.
134 install_schema = None
135 #: Name of the database that this application uses, i.e. ``mysql`` or
136 #: ``postgres``. If we end up supporting multiple databases for a single
137 #: application, there should also be a value for this in
138 #: :class:`wizard.deploy.Deployment`; the value here is merely the preferred
141 def __init__(self, name):
145 self._extractors = {}
146 self._substitutions = {}
147 def repository(self, srv_path):
149 Returns the Git repository that would contain this application.
150 ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
152 repo = os.path.join(srv_path, self.name + ".git")
153 if not os.path.isdir(repo):
154 repo = os.path.join(srv_path, self.name, ".git")
155 if not os.path.isdir(repo):
156 raise NoRepositoryError(self.name)
158 def makeVersion(self, version):
160 Creates or retrieves the :class:`ApplicationVersion` singleton for the
163 if version not in self.versions:
164 self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
165 return self.versions[version]
166 def extract(self, deployment):
168 Extracts wizard variables from a deployment. Default implementation
169 uses :attr:`extractors`.
172 for k,extractor in self.extractors.items():
173 result[k] = extractor(deployment)
174 # XXX: ugh... we have to do quoting
175 for k in self.random_keys:
176 if result[k] is None:
177 result[k] = "'%s'" % ''.join(random.choice(string.letters + string.digits) for i in xrange(30))
179 def dsn(self, deployment):
181 Returns the deployment specific database URL. Uses the override file
182 in :file:`.scripts` if it exists, and otherwise attempt to extract the
183 variables from the source files.
185 Under some cases, the database URL will contain only the database
186 property, and no other values. This indicates that the actual DSN
187 should be determined from the environment.
189 This function might return ``None``.
193 We are allowed to batch these two together, because the full precedence
194 chain for determining the database of an application combines these
195 two together. If this was not the case, we would have to call
196 :meth:`databaseUrlFromOverride` and :meth:`databaseUrlFromExtract` manually.
198 url = self.dsnFromOverride(deployment)
201 return self.dsnFromExtract(deployment)
202 def dsnFromOverride(self, deployment):
204 Extracts database URL from an explicit dsn override file.
207 return sqlalchemy.engine.url.make_url(open(deployment.dsn_file).read().strip())
210 def dsnFromExtract(self, deployment):
212 Extracts database URL from a deployment, and returns them as
213 a :class:`sqlalchemy.engine.url.URL`. Returns ``None`` if we
214 can't figure it out: i.e. the conventional variables are not defined
215 for this application.
217 if not self.database:
219 vars = self.extract(deployment)
220 names = ("WIZARD_DBSERVER", "WIZARD_DBUSER", "WIZARD_DBPASSWORD", "WIZARD_DBNAME")
221 host, user, password, database = (shlex.split(vars[x])[0] if vars[x] is not None else None for x in names)
222 # XXX: You'd have to put support for an explicit different database
224 return sqlalchemy.engine.url.URL(self.database, username=user, password=password, host=host, database=database)
225 def url(self, deployment):
227 Returns the deployment specific web URL. Uses the override file
228 in :file:`.scripts` if it exists, and otherwise attempt to extract
229 the variables from the source files.
231 This function might return ``None``, which indicates we couldn't figure
234 url = self.urlFromOverride(deployment)
237 return self.urlFromExtract(deployment)
238 def urlFromOverride(self, deployment):
240 Extracts URL from explicit url override file.
243 return urlparse.urlparse(open(deployment.url_file).read().strip())
246 def urlFromExtract(self, deployment):
248 Extracts URL from a deployment, and returns ``None`` if we can't
249 figure it out. Default implementation is to fail; we might
250 do something clever with extractable variables in the future.
253 def parametrize(self, deployment, ref_deployment):
255 Takes a generic source checkout and parametrizes it according to the
256 values of ``deployment``. This function operates on the current
257 working directory. ``deployment`` should **not** be the same as the
258 current working directory. Default implementation uses
259 :attr:`parametrized_files` and a simple search and replace on those
262 variables = ref_deployment.extract()
263 for file in self.parametrized_files:
265 contents = open(file, "r").read()
268 for key, value in variables.items():
269 if value is None: continue
270 contents = contents.replace(key, value)
273 def resolveConflicts(self, deployment):
275 Resolves conflicted files in the current working directory. Returns
276 whether or not all conflicted files were resolved or not. Fully
277 resolved files are added to the index, but no commit is made. The
278 default implementation uses :attr:`resolutions`.
283 for status in shell.eval("git", "ls-files", "--unmerged").splitlines():
284 mode, hash, role, name = status.split()
285 files.setdefault(name, set()).add(int(role))
286 for file, roles in files.items():
287 # some automatic resolutions
288 if 1 not in roles and 2 not in roles and 3 in roles:
289 # upstream added a file, but it conflicted for whatever reason
290 shell.call("git", "add", file)
292 elif 1 in roles and 2 not in roles and 3 in roles:
293 # user deleted the file, but upstream changed it
294 shell.call("git", "rm", file)
297 # XXX: this functionality is mostly subsumed by the rerere
299 if file in self.resolutions:
300 contents = open(file, "r").read()
301 for spec, result in self.resolutions[file]:
302 old_contents = contents
303 contents = resolve.resolve(contents, spec, result)
304 if old_contents != contents:
305 logging.info("Did resolution with spec:\n" + spec)
306 open(file, "w").write(contents)
307 if not resolve.is_conflict(contents):
308 shell.call("git", "add", file)
314 def prepareMerge(self, deployment):
316 Performs various edits to files in the current working directory in
317 order to make a merge go more smoothly. This is usually
318 used to fix botched line-endings. If you add new files,
319 you have to 'git add' them; this is not necessary for edits.
320 By default this is a no-op; subclasses should replace this
321 with useful behavior.
324 def prepareConfig(self, deployment):
326 Takes a deployment and replaces any explicit instances
327 of a configuration variable with generic ``WIZARD_*`` constants.
328 The default implementation uses :attr:`substitutions`, and
329 emits warnings when it encounters keys in :attr:`deprecated_keys`.
331 for key, subst in self.substitutions.items():
332 subs = subst(deployment)
333 if not subs and key not in self.deprecated_keys:
334 logging.warning("No substitutions for %s" % key)
335 def install(self, version, options):
337 Run for 'wizard configure' (and, by proxy, 'wizard install') to
338 configure an application. This assumes that the current working
339 directory is a deployment. (Unlike its kin, this function does not
340 take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should
341 provide an implementation.
343 raise NotImplementedError
344 def upgrade(self, deployment, version, options):
346 Run for 'wizard upgrade' to upgrade database schemas and other
347 non-versioned data in an application after the filesystem has been
348 upgraded. This assumes that the current working directory is the
349 deployment. Subclasses should provide an implementation.
351 raise NotImplementedError
352 def backup(self, deployment, outdir, options):
354 Run for 'wizard backup' and upgrades to backup database schemas
355 and other non-versioned data in an application. ``outdir`` is
356 the directory that backup files should be placed. This assumes
357 that the current working directory is the deployment. Subclasses
358 should provide an implementation, even if it is a no-op.
361 Static user files may not need to be backed up, since in
362 many applications upgrades do not modify static files.
364 raise NotImplementedError
365 def restore(self, deployment, backup_dir, options):
367 Run for 'wizard restore' and failed upgrades to restore database
368 and other non-versioned data to a backed up version. This assumes
369 that the current working directory is the deployment. Subclasses
370 should provide an implementation.
372 raise NotImplementedError
373 def remove(self, deployment, options):
375 Run for 'wizard remove' to delete all database and non-local
376 file data. This assumes that the current working directory is
377 the deployment. Subclasses should provide an implementation.
379 raise NotImplementedError
380 def detectVersion(self, deployment):
382 Checks source files to determine the version manually. This assumes
383 that the current working directory is the deployment. Subclasses
384 should provide an implementation.
386 raise NotImplementedError
387 def detectVersionFromFile(self, filename, regex):
389 Helper method that detects a version by using a regular expression
390 from a file. The regexed value is passed through :mod:`shlex`.
391 This assumes that the current working directory is the deployment.
393 contents = open(filename).read()
394 match = regex.search(contents)
395 if not match: return None
396 return distutils.version.LooseVersion(shlex.split(match.group(2))[0])
397 def download(self, version):
399 Returns a URL that can be used to download a tarball of ``version`` of
402 raise NotImplementedError
403 def checkWeb(self, deployment):
405 Checks if the autoinstall is viewable from the web. Subclasses should
406 provide an implementation.
409 Finding a reasonable heuristic that works across skinning
410 choices can be difficult. We've had reasonable success
411 searching for metadata. Be sure that the standard error
412 page does not contain the features you search for. Try
413 not to depend on pages that are not the main page.
415 raise NotImplementedError
416 def checkDatabase(self, deployment):
418 Checks if the database is accessible.
421 sql.connect(deployment.dsn)
423 except sqlalchemy.exc.DBAPIError:
425 def checkWebPage(self, deployment, page, outputs=[], exclude=[]):
427 Checks if a given page of an autoinstall contains a particular string.
429 page = deployment.fetch(page)
431 if page.find(x) != -1:
432 logging.info("checkWebPage (failed due to %s):\n\n%s", x, page)
435 for output in outputs:
436 votes += page.find(output) != -1
437 if votes > len(outputs) / 2:
438 logging.debug("checkWebPage (passed):\n\n" + page)
441 logging.info("checkWebPage (failed):\n\n" + page)
443 def checkConfig(self, deployment):
445 Checks whether or not an autoinstall has been configured/installed
446 for use. Assumes that the current working directory is the deployment.
447 Subclasses should provide an implementation.
449 # XXX: Unfortunately, this doesn't quite work because we package
450 # bogus config files in the -scripts versions of installs. Maybe
451 # we should check a hash or something?
452 raise NotImplementedError
453 def researchFilter(self, filename, added, deleted):
455 Allows an application to selectively ignore certain diffstat signatures
456 during research; for example, configuration files will have a very
457 specific set of changes, so ignore them; certain installation files
458 may be removed, etc. Return ``True`` if a diffstat signature should be
462 def researchVerbose(self, filename):
464 Allows an application to exclude certain dirty files from the output
465 report; usually this will just be parametrized files, since those are
466 guaranteed to have changes. Return ``True`` if a file should only
467 be displayed in verbose mode.
469 return filename in self.parametrized_files
471 class ApplicationVersion(object):
472 """Represents an abstract notion of a version for an application, where
473 ``version`` is a :class:`distutils.version.LooseVersion` and
474 ``application`` is a :class:`Application`."""
475 #: The :class:`distutils.version.LooseVersion` of this instance.
477 #: The :class:`Application` of this instance.
479 def __init__(self, version, application):
480 self.version = version
481 self.application = application
485 Returns the name of the git describe tag for the commit the user is
486 presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
488 return "%s-%s" % (self.application, self.version)
490 def scripts_tag(self):
492 Returns the name of the Git tag for this version.
494 end = str(self.version).partition('-scripts')[2].partition('-')[0]
495 return "%s-scripts%s" % (self.pristine_tag, end)
497 def pristine_tag(self):
499 Returns the name of the Git tag for the pristine version corresponding
502 return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
503 def __cmp__(self, y):
504 return cmp(self.version, y.version)
508 Parses a line from the :term:`versions store` and return
509 :class:`ApplicationVersion`.
511 Use this only for cases when speed is of primary importance;
512 the data in version is unreliable and when possible, you should
513 prefer directly instantiating a :class:`wizard.deploy.Deployment` and having it query
514 the autoinstall itself for information.
516 The `value` to parse will vary. For old style installs, it
519 /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
521 For new style installs, it will look like::
525 name = value.split("/")[-1]
527 if name.find("-") != -1:
528 app, _, version = name.partition("-")
530 # kind of poor, maybe should error. Generally this
531 # will actually result in a not found error
535 raise DeploymentParseError(value)
536 return ApplicationVersion.make(app, version)
538 def make(app, version):
540 Makes/retrieves a singleton :class:`ApplicationVersion` from
541 a``app`` and ``version`` string.
544 # defer to the application for version creation to enforce
546 return applications()[app].makeVersion(version)
548 raise NoSuchApplication(app)
552 Takes a tree of values (implement using nested lists) and
553 transforms them into regular expressions.
557 >>> expand_re(['a', 'b'])
559 >>> expand_re(['*', ['b', 'c']])
562 if isinstance(val, str):
563 return re.escape(val)
565 return '(?:' + '|'.join(map(expand_re, val)) + ')'
567 def make_extractors(seed):
569 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into
570 extractor functions (which take a :class:`wizard.deploy.Deployment`
571 and return the value of the second subpattern of ``regex`` when matched
572 with the contents of ``file``).
574 return util.dictmap(lambda a: filename_regex_extractor(*a), seed)
576 def make_substitutions(seed):
578 Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into substitution
579 functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern
580 of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.)
582 return util.dictkmap(lambda k, v: filename_regex_substitution(k, *v), seed)
584 # The following two functions are *highly* functional, and I recommend
585 # not touching them unless you know what you're doing.
587 def filename_regex_extractor(file, regex):
589 .. highlight:: haskell
591 Given a relative file name ``file``, a regular expression ``regex``, and a
592 :class:`wizard.deploy.Deployment` extracts a value out of the file in that
593 deployment. This function is curried, so you pass just ``file`` and
594 ``regex``, and then pass ``deployment`` to the resulting function.
596 Its Haskell-style type signature would be::
598 Filename -> Regex -> (Deployment -> String)
600 The regular expression requires a very specific form, essentially ``()()()``
601 (with the second subgroup being the value to extract). These enables
602 the regular expression to be used equivalently with filename
604 .. highlight:: python
606 For convenience purposes, we also accept ``[Filename]``, in which case
607 we use the first entry (index 0). Passing an empty list is invalid.
609 >>> open("test-settings.extractor.ini", "w").write("config_var = 3\\n")
610 >>> f = filename_regex_extractor('test-settings.extractor.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
611 >>> f(deploy.Deployment("."))
613 >>> os.unlink("test-settings.extractor.ini")
616 The first application of ``regex`` and ``file`` is normally performed
617 at compile-time inside a submodule; the second application is
618 performed at runtime.
620 if not isinstance(file, str):
624 contents = deployment.read(file) # cached
627 match = regex.search(contents)
628 if not match: return None
629 # assumes that the second match is the one we want.
630 return match.group(2)
633 def filename_regex_substitution(key, files, regex):
635 .. highlight:: haskell
637 Given a Wizard ``key`` (``WIZARD_*``), a list of ``files``, a
638 regular expression ``regex``, and a :class:`wizard.deploy.Deployment`
639 performs a substitution of the second subpattern of ``regex``
640 with ``key``. Returns the number of replacements made. This function
641 is curried, so you pass just ``key``, ``files`` and ``regex``, and
642 then pass ``deployment`` to the resulting function.
644 Its Haskell-style type signature would be::
646 Key -> ([File], Regex) -> (Deployment -> IO Int)
648 .. highlight:: python
650 For convenience purposes, we also accept ``Filename``, in which case it is treated
651 as a single item list.
653 >>> open("test-settings.substitution.ini", "w").write("config_var = 3")
654 >>> f = filename_regex_substitution('WIZARD_KEY', 'test-settings.substitution.ini', re.compile('^(config_var\s*=\s*)(.*)()$'))
655 >>> f(deploy.Deployment("."))
657 >>> print open("test-settings.substitution.ini", "r").read()
658 config_var = WIZARD_KEY
659 >>> os.unlink("test-settings.substitution.ini")
661 if isinstance(files, str):
664 base = deployment.location
667 file = os.path.join(base, file)
669 contents = open(file, "r").read()
670 contents, n = regex.subn("\\1" + key + "\\3", contents)
672 open(file, "w").write(contents)
678 def backup_database(outdir, deployment):
680 Generic database backup function for MySQL.
682 # XXX: Change this once deployments support multiple dbs
683 if deployment.application.database == "mysql":
684 return backup_mysql_database(outdir, deployment)
686 raise NotImplementedError
688 def backup_mysql_database(outdir, deployment):
690 Database backups for MySQL using the :command:`mysqldump` utility.
692 outfile = os.path.join(outdir, "db.sql")
694 shell.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment.dsn))
695 shell.call("gzip", "--best", outfile)
696 except shell.CallError as e:
697 raise BackupFailure(e.stderr)
699 def restore_database(backup_dir, deployment):
701 Generic database restoration function for MySQL.
703 # XXX: see backup_database
704 if deployment.application.database == "mysql":
705 return restore_mysql_database(backup_dir, deployment)
707 raise NotImplementedError
709 def restore_mysql_database(backup_dir, deployment):
711 Database restoration for MySQL by piping SQL commands into :command:`mysql`.
713 if not os.path.exists(backup_dir):
714 raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2])
715 sql = open(os.path.join(backup_dir, "db.sql"), 'w+')
716 shell.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql)
718 shell.call("mysql", *get_mysql_args(deployment.dsn), stdin=sql)
721 def remove_database(deployment):
723 Generic database removal function. Actually, not so generic because we
724 go and check if we're on scripts and if we are run a different command.
726 if deployment.dsn.host == "sql.mit.edu":
728 shell.call("/mit/scripts/sql/bin/drop-database", deployment.dsn.database)
730 except shell.CallError:
732 engine = sqlalchemy.create_engine(deployment.dsn)
733 engine.execute("DROP DATABASE `%s`" % deployment.dsn.database)
735 def get_mysql_args(dsn):
737 Extracts arguments that would be passed to the command line mysql utility
742 args += ["-h", dsn.host]
744 args += ["-u", dsn.username]
746 args += ["-p" + dsn.password]
747 args += [dsn.database]
750 class Error(wizard.Error):
751 """Generic error class for this module."""
754 class NoRepositoryError(Error):
756 :class:`Application` does not appear to have a Git repository
757 in the normal location.
759 #: The name of the application that does not have a Git repository.
761 def __init__(self, app):
764 return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
766 class DeploymentParseError(Error):
768 Could not parse ``value`` from :term:`versions store`.
770 #: The value that failed to parse.
772 #: The location of the autoinstall that threw this variable.
773 #: This should be set by error handling code when it is available.
775 def __init__(self, value):
778 class NoSuchApplication(Error):
780 You attempted to reference a :class:`Application` named
781 ``app``, which is not recognized by Wizard.
783 #: The name of the application that does not exist.
785 #: The location of the autoinstall that threw this variable.
786 #: This should be set by error handling code when it is availble.
788 def __init__(self, app):
791 class Failure(Error):
793 Represents a failure when performing some double-dispatched operation
794 such as an installation or an upgrade. Failure classes are postfixed
795 with Failure, not Error.
799 class InstallFailure(Error):
800 """Installation failed for unknown reason."""
804 ERROR: Installation failed for unknown reason. You can
805 retry the installation by appending --retry to the installation
808 class RecoverableInstallFailure(InstallFailure):
810 Installation failed, but we were able to determine what the
811 error was, and should give the user a second chance if we were
812 running interactively.
814 #: List of the errors that were found.
816 def __init__(self, errors):
821 ERROR: Installation failed due to the following errors: %s
823 You can retry the installation by appending --retry to the
824 installation command.""" % ", ".join(self.errors)
826 class UpgradeFailure(Failure):
827 """Upgrade script failed."""
828 #: String details of failure (possibly stdout or stderr output)
830 def __init__(self, details):
831 self.details = details
835 ERROR: Upgrade script failed, details:
839 class UpgradeVerificationFailure(Failure):
840 """Upgrade script passed, but website wasn't accessible afterwards"""
844 ERROR: Upgrade script passed, but website wasn't accessible afterwards. Check
845 the debug logs for the contents of the page."""
847 class BackupFailure(Failure):
848 """Backup script failed."""
849 #: String details of failure
851 def __init__(self, details):
852 self.details = details
856 ERROR: Backup script failed, details:
860 class RestoreFailure(Failure):
861 """Restore script failed."""
862 #: String details of failure
864 def __init__(self, details):
865 self.details = details
869 ERROR: Restore script failed, details:
873 class RemoveFailure(Failure):
874 """Remove script failed."""
875 #: String details of failure
877 def __init__(self, details):
878 self.details = details
882 ERROR: Remove script failed, details: