X-Git-Url: https://scripts.mit.edu/gitweb/wizard.git/blobdiff_plain/31253eca10ab83a98b0921ac6bbbb22b90ff2af3..5b428fce4566ed627ad61f204935aaa8bc367932:/wizard/app/__init__.py diff --git a/wizard/app/__init__.py b/wizard/app/__init__.py index ab4d008..6a5ee6a 100644 --- a/wizard/app/__init__.py +++ b/wizard/app/__init__.py @@ -6,6 +6,18 @@ You'll need to know how to overload the :class:`Application` class and use some of the functions in this module in order to specify new applications. +To specify custom applications as plugins, add the following ``entry_points`` +configuration:: + + [wizard.app] + yourappname = your.module:Application + otherappname = your.other.module:Application + +.. note:: + + Wizard will complain loudly if ``yourappname`` conflicts with an + application name defined by someone else. + There are some submodules for programming languages that define common functions and data that may be used by applications in that language. See: @@ -21,31 +33,57 @@ functions and data that may be used by applications in that language. See: """ import os.path +import subprocess import re import distutils.version import decorator import shlex import logging import shutil +import sqlalchemy +import sqlalchemy.exc +import string +import urlparse +import tempfile +import pkg_resources +import traceback import wizard -from wizard import resolve, scripts, shell, util - -_application_list = [ - "mediawiki", "wordpress", "joomla", "e107", "gallery2", - "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django", - # these are technically deprecated - "advancedpoll", "gallery", -] -_applications = None +from wizard import plugin, resolve, shell, sql, util +_applications = None def applications(): """Hash table for looking up string application name to instance""" global _applications if not _applications: - _applications = dict([(n,Application.make(n)) for n in _application_list ]) + _applications = dict() + for dist in pkg_resources.working_set: + for appname, entry in dist.get_entry_map("wizard.app").items(): + if appname in _applications: + newname = dist.key + ":" + appname + if newname in _applications: + raise Exception("Unrecoverable application name conflict for %s from %s", appname, dist.key) + logging.warning("Could not overwrite %s, used %s instead", appname, newname) + appname = newname + appclass = entry.load() + _applications[appname] = appclass(appname) + # setup dummy apps + for entry in pkg_resources.iter_entry_points("wizard.dummy_apps"): + appfun = entry.load() + dummy_apps = appfun() + for appname in dummy_apps: + # a dummy app that already exists is not a fatal error + if appname in _applications: + continue + _applications[appname] = Application(appname) return _applications +def getApplication(appname): + """Retrieves application instance given a name""" + try: + return applications()[appname] + except KeyError: + raise NoSuchApplication(appname) class Application(object): """ @@ -66,7 +104,12 @@ class Application(object): parametrized_files = [] #: Keys that are used in older versions of the application, but #: not for the most recent version. - deprecated_keys = [] + deprecated_keys = set() + #: Keys that we can simply generate random strings for if they're missing + random_keys = set() + #: Values that are not sufficiently random for a random key. This can + #: include default values for a random configuration option, + random_blacklist = set() #: Dictionary of variable names to extractor functions. These functions #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of #: the variable, or ``None`` if it could not be found. @@ -84,6 +127,14 @@ class Application(object): #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments #: this application requires. install_schema = None + #: Name of the database that this application uses, i.e. ``mysql`` or + #: ``postgres``. If we end up supporting multiple databases for a single + #: application, there should also be a value for this in + #: :class:`wizard.deploy.Deployment`; the value here is merely the preferred + #: value. + database = None + #: Indicates whether or not a web stub is necessary. + needs_web_stub = False def __init__(self, name): self.name = name self.versions = {} @@ -117,21 +168,111 @@ class Application(object): result = {} for k,extractor in self.extractors.items(): result[k] = extractor(deployment) + # XXX: ugh... we have to do quoting + for k in self.random_keys: + if result[k] is None or result[k] in self.random_blacklist: + result[k] = "'%s'" % util.random_key() return result + def dsn(self, deployment): + """ + Returns the deployment specific database URL. Uses the override file + in :file:`.wizard` if it exists, and otherwise attempt to extract the + variables from the source files. + + Under some cases, the database URL will contain only the database + property, and no other values. This indicates that the actual DSN + should be determined from the environment. + + This function might return ``None``. + + .. note:: + + We are allowed to batch these two together, because the full precedence + chain for determining the database of an application combines these + two together. If this was not the case, we would have to call + :meth:`databaseUrlFromOverride` and :meth:`databaseUrlFromExtract` manually. + """ + url = self.dsnFromOverride(deployment) + if url: + return url + return self.dsnFromExtract(deployment) + def dsnFromOverride(self, deployment): + """ + Extracts database URL from an explicit dsn override file. + """ + try: + return sqlalchemy.engine.url.make_url(open(deployment.dsn_file).read().strip()) + except IOError: + return None + def dsnFromExtract(self, deployment): + """ + Extracts database URL from a deployment, and returns them as + a :class:`sqlalchemy.engine.url.URL`. Returns ``None`` if we + can't figure it out: i.e. the conventional variables are not defined + for this application. + """ + if not self.database: + return None + vars = self.extract(deployment) + names = ("WIZARD_DBSERVER", "WIZARD_DBUSER", "WIZARD_DBPASSWORD", "WIZARD_DBNAME") + host, user, password, database = (shlex.split(vars[x])[0] if vars[x] is not None else None for x in names) + # XXX: You'd have to put support for an explicit different database + # type here + return sqlalchemy.engine.url.URL(self.database, username=user, password=password, host=host, database=database) + def url(self, deployment): + """ + Returns the deployment specific web URL. Uses the override file + in :file:`.wizard` if it exists, and otherwise attempt to extract + the variables from the source files. + + This function might return ``None``, which indicates we couldn't figure + it out. + """ + url = self.urlFromOverride(deployment) + if url: + return url + return self.urlFromExtract(deployment) + def urlFromOverride(self, deployment): + """ + Extracts URL from explicit url override file. + """ + try: + return urlparse.urlparse(open(deployment.url_file).read().strip()) + except IOError: + return None + def urlFromExtract(self, deployment): + """ + Extracts URL from a deployment, and returns ``None`` if we can't + figure it out. Default implementation is to fail; we might + do something clever with extractable variables in the future. + """ + return None def parametrize(self, deployment, ref_deployment): """ Takes a generic source checkout and parametrizes it according to the values of ``deployment``. This function operates on the current working directory. ``deployment`` should **not** be the same as the - current working directory. Default implementation uses - :attr:`parametrized_files` and a simple search and replace on those - files. + current working directory. See :meth:`parametrizeWithVars` for details + on the parametrization. """ + # deployment is not used in this implementation, but note that + # we do have the invariant the current directory matches + # deployment's directory variables = ref_deployment.extract() + self.parametrizeWithVars(variables) + def parametrizeWithVars(self, variables): + """ + Takes a generic source checkout and parametrizes it according to + the values of ``variables``. Default implementation uses + :attr:`parametrized_files` and a simple search and replace on + those files. + """ for file in self.parametrized_files: + logging.debug("Parametrizing file '%s'\n" % (file, )) try: contents = open(file, "r").read() except IOError: + logging.debug("Failed to open file '%s'\n" % (file, )) continue for key, value in variables.items(): if value is None: continue @@ -146,9 +287,24 @@ class Application(object): default implementation uses :attr:`resolutions`. """ resolved = True - sh = shell.Shell() - for status in sh.eval("git", "ls-files", "--unmerged").splitlines(): - file = status.split()[-1] + files = set() + files = {} + for status in shell.eval("git", "ls-files", "--unmerged").splitlines(): + mode, hash, role, name = status.split() + files.setdefault(name, set()).add(int(role)) + for file, roles in files.items(): + # some automatic resolutions + if 1 not in roles and 2 not in roles and 3 in roles: + # upstream added a file, but it conflicted for whatever reason + shell.call("git", "add", file) + continue + elif 1 in roles and 2 not in roles and 3 in roles: + # user deleted the file, but upstream changed it + shell.call("git", "rm", file) + continue + # manual resolutions + # XXX: this functionality is mostly subsumed by the rerere + # tricks we do if file in self.resolutions: contents = open(file, "r").read() for spec, result in self.resolutions[file]: @@ -158,7 +314,7 @@ class Application(object): logging.info("Did resolution with spec:\n" + spec) open(file, "w").write(contents) if not resolve.is_conflict(contents): - sh.call("git", "add", file) + shell.call("git", "add", file) else: resolved = False else: @@ -183,7 +339,7 @@ class Application(object): """ for key, subst in self.substitutions.items(): subs = subst(deployment) - if not subs and key not in self.deprecated_keys: + if not subs and key not in self.deprecated_keys and key not in self.random_keys: logging.warning("No substitutions for %s" % key) def install(self, version, options): """ @@ -223,6 +379,13 @@ class Application(object): should provide an implementation. """ raise NotImplementedError + def remove(self, deployment, options): + """ + Run for 'wizard remove' to delete all database and non-local + file data. This assumes that the current working directory is + the deployment. Subclasses should provide an implementation. + """ + raise NotImplementedError def detectVersion(self, deployment): """ Checks source files to determine the version manually. This assumes @@ -240,19 +403,30 @@ class Application(object): match = regex.search(contents) if not match: return None return distutils.version.LooseVersion(shlex.split(match.group(2))[0]) + # XXX: This signature doesn't really make too much sense... + def detectVersionFromGit(self, tagPattern, preStrip = ''): + """ + Helper method that detects a version by using the most recent tag + in git that matches the specified pattern. + This assumes that the current working directory is the deployment. + """ + sh = wizard.shell.Shell() + cmd = ['git', 'describe', '--tags', '--match', tagPattern, ] + tag = sh.call(*cmd, strip=True) + if tag and len(tag) > len(preStrip) and tag[:len(preStrip)] == preStrip: + tag = tag[len(preStrip):] + if not tag: return None + return distutils.version.LooseVersion(tag) def download(self, version): """ Returns a URL that can be used to download a tarball of ``version`` of this application. """ raise NotImplementedError - def checkWeb(self, deployment, output=None): + def checkWeb(self, deployment): """ - Checks if the autoinstall is viewable from the web. To get - the HTML source that was retrieved, pass a variable containing - an empty list to ``output``; it will be mutated to have its - first element be the output. Subclasses should provide an - implementation. + Checks if the autoinstall is viewable from the web. Subclasses should + provide an implementation. .. note:: Finding a reasonable heuristic that works across skinning @@ -262,6 +436,33 @@ class Application(object): not to depend on pages that are not the main page. """ raise NotImplementedError + def checkDatabase(self, deployment): + """ + Checks if the database is accessible. + """ + try: + sql.connect(deployment.dsn) + return True + except sqlalchemy.exc.DBAPIError: + return False + def checkWebPage(self, deployment, page, outputs=[], exclude=[]): + """ + Checks if a given page of an autoinstall contains a particular string. + """ + page = deployment.fetch(page) + for x in exclude: + if page.find(x) != -1: + logging.info("checkWebPage (failed due to %s):\n\n%s", x, page) + return False + votes = 0 + for output in outputs: + votes += page.find(output) != -1 + if votes > len(outputs) / 2: + logging.debug("checkWebPage (passed):\n\n" + page) + return True + else: + logging.info("checkWebPage (failed):\n\n" + page) + return False def checkConfig(self, deployment): """ Checks whether or not an autoinstall has been configured/installed @@ -269,17 +470,26 @@ class Application(object): Subclasses should provide an implementation. """ # XXX: Unfortunately, this doesn't quite work because we package - # bogus config files in the -scripts versions of installs. Maybe - # we should check a hash or something? + # bogus config files. Maybe we should check a hash or + # something? raise NotImplementedError - @staticmethod - def make(name): - """Makes an application, but uses the correct subtype if available.""" - try: - __import__("wizard.app." + name) - return getattr(wizard.app, name).Application(name) - except ImportError: - return Application(name) + def researchFilter(self, filename, added, deleted): + """ + Allows an application to selectively ignore certain diffstat signatures + during research; for example, configuration files will have a very + specific set of changes, so ignore them; certain installation files + may be removed, etc. Return ``True`` if a diffstat signature should be + ignored, + """ + return False + def researchVerbose(self, filename): + """ + Allows an application to exclude certain dirty files from the output + report; usually this will just be parametrized files, since those are + guaranteed to have changes. Return ``True`` if a file should only + be displayed in verbose mode. + """ + return filename in self.parametrized_files class ApplicationVersion(object): """Represents an abstract notion of a version for an application, where @@ -300,10 +510,11 @@ class ApplicationVersion(object): """ return "%s-%s" % (self.application, self.version) @property - def scripts_tag(self): + def wizard_tag(self): """ Returns the name of the Git tag for this version. """ + # XXX: Scripts specific end = str(self.version).partition('-scripts')[2].partition('-')[0] return "%s-scripts%s" % (self.pristine_tag, end) @property @@ -353,12 +564,9 @@ class ApplicationVersion(object): Makes/retrieves a singleton :class:`ApplicationVersion` from a``app`` and ``version`` string. """ - try: - # defer to the application for version creation to enforce - # singletons - return applications()[app].makeVersion(version) - except KeyError: - raise NoSuchApplication(app) + # defer to the application for version creation to enforce + # singletons + return getApplication(app).makeVersion(version) def expand_re(val): """ @@ -488,53 +696,24 @@ def filename_regex_substitution(key, files, regex): return subs return h -# XXX: rename to show that it's mysql specific -def backup_database(outdir, deployment): +@decorator.decorator +def throws_database_errors(f, self, *args, **kwargs): """ - Generic database backup function for MySQL. Assumes that ``WIZARD_DBNAME`` - is extractable, and that :func:`wizard.scripts.get_sql_credentials` - works. + Decorator that takes database errors from :mod:`wizard.sql` and + converts them into application script failures from + :mod:`wizard.app`. We can't throw application errors directly from + :mod:`wizard.sql` because that would result in a cyclic import; + also, it's cleaner to distinguish between a database error and an + application script failure. """ - sh = shell.Shell() - outfile = os.path.join(outdir, "db.sql") try: - sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment)) - sh.call("gzip", "--best", outfile) - except shell.CallError as e: - shutil.rmtree(outdir) - raise BackupFailure(e.stderr) - -def restore_database(backup_dir, deployment): - """ - Generic database restoration function for MySQL. See :func:`backup_database` - for the assumptions that we make. - """ - sh = shell.Shell() - if not os.path.exists(backup_dir): - raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2]) - sql = open(os.path.join(backup_dir, "db.sql"), 'w+') - sh.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql) - sql.seek(0) - sh.call("mysql", *get_mysql_args(deployment), stdin=sql) - sql.close() - -def get_mysql_args(d): - """ - Extracts arguments that would be passed to the command line mysql utility - from a deployment. - """ - # XXX: add support for getting these out of options - vars = d.extract() - if 'WIZARD_DBNAME' not in vars: - raise BackupFailure("Could not determine database name") - triplet = scripts.get_sql_credentials(vars) - args = [] - if triplet is not None: - server, user, password = triplet - args += ["-h", server, "-u", user, "-p" + password] - name = shlex.split(vars['WIZARD_DBNAME'])[0] - args.append(name) - return args + return f(self, *args, **kwargs) + except sql.BackupDatabaseError: + raise BackupFailure(traceback.format_exc()) + except sql.RestoreDatabaseError: + raise RestoreFailure(traceback.format_exc()) + except sql.RemoveDatabaseError: + raise RemoveFailure(traceback.format_exc()) class Error(wizard.Error): """Generic error class for this module.""" @@ -563,6 +742,8 @@ class DeploymentParseError(Error): location = None def __init__(self, value): self.value = value + def __str__(self): + return "Could not parse '%s' from versions store in '%s'" % (self.value, self.location) class NoSuchApplication(Error): """ @@ -576,6 +757,8 @@ class NoSuchApplication(Error): location = None def __init__(self, app): self.app = app + def __str__(self): + return "Wizard doesn't know about an application named '%s'." % self.app class Failure(Error): """ @@ -587,7 +770,12 @@ class Failure(Error): class InstallFailure(Error): """Installation failed for unknown reason.""" - pass + def __str__(self): + return """ + +ERROR: Installation failed for unknown reason. You can +retry the installation by appending --retry to the installation +command.""" class RecoverableInstallFailure(InstallFailure): """ @@ -600,7 +788,12 @@ class RecoverableInstallFailure(InstallFailure): def __init__(self, errors): self.errors = errors def __str__(self): - return """Installation failed due to the following errors: %s""" % ", ".join(self.errors) + return """ + +ERROR: Installation failed due to the following errors: %s + +You can retry the installation by appending --retry to the +installation command.""" % ", ".join(self.errors) class UpgradeFailure(Failure): """Upgrade script failed.""" @@ -617,16 +810,11 @@ ERROR: Upgrade script failed, details: class UpgradeVerificationFailure(Failure): """Upgrade script passed, but website wasn't accessible afterwards""" - #: String details of failure (possibly stdout or stderr output) - details = None - def __init__(self, details): - self.details = details def __str__(self): return """ -ERROR: Upgrade script passed, but website wasn't accessible afterwards. Details: - -%s""" % self.details +ERROR: Upgrade script passed, but website wasn't accessible afterwards. Check +the debug logs for the contents of the page.""" class BackupFailure(Failure): """Backup script failed.""" @@ -653,3 +841,16 @@ class RestoreFailure(Failure): ERROR: Restore script failed, details: %s""" % self.details + +class RemoveFailure(Failure): + """Remove script failed.""" + #: String details of failure + details = None + def __init__(self, details): + self.details = details + def __str__(self): + return """ + +ERROR: Remove script failed, details: + +%s""" % self.details