X-Git-Url: https://scripts.mit.edu/gitweb/wizard.git/blobdiff_plain/113f2636b978f686013d4350f735398f01f9fcb1..30380c4b5b28df9670ea5952e14bc485d1d34133:/wizard/app/__init__.py diff --git a/wizard/app/__init__.py b/wizard/app/__init__.py index 74d99bd..1d01966 100644 --- a/wizard/app/__init__.py +++ b/wizard/app/__init__.py @@ -6,6 +6,11 @@ You'll need to know how to overload the :class:`Application` class and use some of the functions in this module in order to specify new applications. +There are some submodules for programming languages that define common +functions and data that may be used by applications in that language. See: + +* :mod:`wizard.app.php` + .. testsetup:: * import re @@ -22,6 +27,11 @@ import decorator import shlex import logging import shutil +import sqlalchemy +import random +import string +import urlparse +import tempfile import wizard from wizard import resolve, scripts, shell, util @@ -61,7 +71,9 @@ class Application(object): parametrized_files = [] #: Keys that are used in older versions of the application, but #: not for the most recent version. - deprecated_keys = [] + deprecated_keys = set() + #: Keys that we can simply generate random strings for if they're missing + random_keys = set() #: Dictionary of variable names to extractor functions. These functions #: take a :class:`wizard.deploy.Deployment` as an argument and return the value of #: the variable, or ``None`` if it could not be found. @@ -76,6 +88,15 @@ class Application(object): #: a conflict marker string and a result list. See :mod:`wizard.resolve` #: for more information. resolutions = {} + #: Instance of :class:`wizard.install.ArgSchema` that defines the arguments + #: this application requires. + install_schema = None + #: Name of the database that this application uses, i.e. ``mysql`` or + #: ``postgres``. If we end up supporting multiple databases for a single + #: application, there should also be a value for this in + #: :class:`wizard.deploy.Deployment`; the value here is merely the preferred + #: value. + database = None def __init__(self, name): self.name = name self.versions = {} @@ -109,7 +130,85 @@ class Application(object): result = {} for k,extractor in self.extractors.items(): result[k] = extractor(deployment) + # XXX: ugh... we have to do quoting + for k in self.random_keys: + if result[k] is None: + result[k] = "'%s'" % ''.join(random.choice(string.letters + string.digits) for i in xrange(30)) return result + def dsn(self, deployment): + """ + Returns the deployment specific database URL. Uses the override file + in :file:`.scripts` if it exists, and otherwise attempt to extract the + variables from the source files. + + Under some cases, the database URL will contain only the database + property, and no other values. This indicates that the actual DSN + should be determined from the environment. + + This function might return ``None``. + + .. note:: + + We are allowed to batch these two together, because the full precedence + chain for determining the database of an application combines these + two together. If this was not the case, we would have to call + :meth:`databaseUrlFromOverride` and :meth:`databaseUrlFromExtract` manually. + """ + url = self.dsnFromOverride(deployment) + if url: + return url + return self.dsnFromExtract(deployment) + def dsnFromOverride(self, deployment): + """ + Extracts database URL from an explicit dsn override file. + """ + try: + return sqlalchemy.engine.url.make_url(open(deployment.dsn_file).read().strip()) + except IOError: + return None + def dsnFromExtract(self, deployment): + """ + Extracts database URL from a deployment, and returns them as + a :class:`sqlalchemy.engine.url.URL`. Returns ``None`` if we + can't figure it out: i.e. the conventional variables are not defined + for this application. + """ + if not self.database: + return None + vars = self.extract(deployment) + names = ("WIZARD_DBSERVER", "WIZARD_DBUSER", "WIZARD_DBPASSWORD", "WIZARD_DBNAME") + host, user, password, database = (shlex.split(vars[x])[0] if vars[x] is not None else None for x in names) + # XXX: You'd have to put support for an explicit different database + # type here + return sqlalchemy.engine.url.URL(self.database, username=user, password=password, host=host, database=database) + def url(self, deployment): + """ + Returns the deployment specific web URL. Uses the override file + in :file:`.scripts` if it exists, and otherwise attempt to extract + the variables from the source files. + + This function might return ``None``, which indicates we couldn't figure + it out. + """ + url = self.urlFromOverride(deployment) + if url: + return url + return self.urlFromExtract(deployment) + def urlFromOverride(self, deployment): + """ + Extracts URL from explicit url override file. + """ + try: + return urlparse.urlparse(open(deployment.url_file).read().strip()) + except IOError: + return None + def urlFromExtract(self, deployment): + """ + Extracts URL from a deployment, and returns ``None`` if we can't + figure it out. Default implementation is to fail; we might + do something clever with extractable variables in the future. + """ + return None def parametrize(self, deployment, ref_deployment): """ Takes a generic source checkout and parametrizes it according to the @@ -138,9 +237,60 @@ class Application(object): default implementation uses :attr:`resolutions`. """ resolved = True - sh = shell.Shell() - for status in sh.eval("git", "ls-files", "--unmerged").splitlines(): - file = status.split()[-1] + files = set() + for status in shell.eval("git", "ls-files", "--unmerged").splitlines(): + files.add(status.split()[-1]) + for file in files: + # check for newline mismatch + # HACK: using git diff to tell if files are binary or not + if not len(shell.eval("git", "diff", file).splitlines()) == 1 and util.mixed_newlines(file): + # this code only works on Unix + def get_newline(filename): + f = open(filename, "U") + # for some reason I need two + s = f.readline() + if s != "" and f.newlines is None: + f.readline() + if not isinstance(f.newlines, str): + raise Exception("Assert: expected newlines to be string, instead was %s in %s" % (repr(f.newlines), file)) + return f.newlines + def create_reference(id): + f = tempfile.NamedTemporaryFile(prefix="wizardResolve", delete=False) + shell.call("git", "cat-file", "blob", ":%d:%s" % (id, file), stdout=f) + f.close() + return get_newline(f.name), f.name + def convert(filename, dest_nl): + contents = open(filename, "U").read().replace("\n", dest_nl) + open(filename, "wb").write(contents) + logging.info("Mixed newlines detected in %s", file) + common_nl, common_file = create_reference(1) + our_nl, our_file = create_reference(2) + their_nl, their_file = create_reference(3) + remerge = False + if common_nl != their_nl: + # upstream can't keep their newlines straight + logging.info("Converting common file (1) from %s to %s newlines", repr(common_nl), repr(their_nl)) + convert(common_file, their_nl) + remerge = True + if our_nl != their_nl: + # common case + logging.info("Converting our file (2) from %s to %s newlines", repr(our_nl), repr(their_nl)) + convert(our_file, their_nl) + remerge = True + if remerge: + logging.info("Remerging %s", file) + with open(file, "wb") as f: + try: + shell.call("git", "merge-file", "--stdout", our_file, common_file, their_file, stdout=f) + logging.info("New merge was clean") + shell.call("git", "add", file) + continue + except shell.CallError: + pass + logging.info("Merge was still unclean") + else: + logging.warning("Mixed newlines detected in %s, but no remerge possible", file) + # manual resolutions if file in self.resolutions: contents = open(file, "r").read() for spec, result in self.resolutions[file]: @@ -150,7 +300,7 @@ class Application(object): logging.info("Did resolution with spec:\n" + spec) open(file, "w").write(contents) if not resolve.is_conflict(contents): - sh.call("git", "add", file) + shell.call("git", "add", file) else: resolved = False else: @@ -185,7 +335,7 @@ class Application(object): take a :class:`wizard.deploy.Deployment` as a parameter.) Subclasses should provide an implementation. """ - raise NotImplemented + raise NotImplementedError def upgrade(self, deployment, version, options): """ Run for 'wizard upgrade' to upgrade database schemas and other @@ -193,7 +343,7 @@ class Application(object): upgraded. This assumes that the current working directory is the deployment. Subclasses should provide an implementation. """ - raise NotImplemented + raise NotImplementedError def backup(self, deployment, outdir, options): """ Run for 'wizard backup' and upgrades to backup database schemas @@ -206,7 +356,7 @@ class Application(object): Static user files may not need to be backed up, since in many applications upgrades do not modify static files. """ - raise NotImplemented + raise NotImplementedError def restore(self, deployment, backup_dir, options): """ Run for 'wizard restore' and failed upgrades to restore database @@ -214,21 +364,41 @@ class Application(object): that the current working directory is the deployment. Subclasses should provide an implementation. """ - raise NotImplemented + raise NotImplementedError + def remove(self, deployment, options): + """ + Run for 'wizard remove' to delete all database and non-local + file data. This assumes that the current working directory is + the deployment. Subclasses should provide an implementation. + """ + raise NotImplementedError def detectVersion(self, deployment): """ Checks source files to determine the version manually. This assumes that the current working directory is the deployment. Subclasses should provide an implementation. """ - raise NotImplemented - def checkWeb(self, deployment, output=None): + raise NotImplementedError + def detectVersionFromFile(self, filename, regex): + """ + Helper method that detects a version by using a regular expression + from a file. The regexed value is passed through :mod:`shlex`. + This assumes that the current working directory is the deployment. + """ + contents = open(filename).read() + match = regex.search(contents) + if not match: return None + return distutils.version.LooseVersion(shlex.split(match.group(2))[0]) + def download(self, version): + """ + Returns a URL that can be used to download a tarball of ``version`` of + this application. + """ + raise NotImplementedError + def checkWeb(self, deployment): """ - Checks if the autoinstall is viewable from the web. To get - the HTML source that was retrieved, pass a variable containing - an empty list to ``output``; it will be mutated to have its - first element be the output. Subclasses should provide an - implementation. + Checks if the autoinstall is viewable from the web. Subclasses should + provide an implementation. .. note:: Finding a reasonable heuristic that works across skinning @@ -237,22 +407,58 @@ class Application(object): page does not contain the features you search for. Try not to depend on pages that are not the main page. """ - raise NotImplemented + raise NotImplementedError + def checkWebPage(self, deployment, page, output): + """ + Checks if a given page of an autoinstall contains a particular string. + """ + page = deployment.fetch(page) + result = page.find(output) != -1 + if result: + logging.debug("checkWebPage (passed):\n\n" + page) + else: + logging.info("checkWebPage (failed):\n\n" + page) + return result def checkConfig(self, deployment): """ Checks whether or not an autoinstall has been configured/installed for use. Assumes that the current working directory is the deployment. Subclasses should provide an implementation. """ - raise NotImplemented + # XXX: Unfortunately, this doesn't quite work because we package + # bogus config files in the -scripts versions of installs. Maybe + # we should check a hash or something? + raise NotImplementedError + def researchFilter(self, filename, added, deleted): + """ + Allows an application to selectively ignore certain diffstat signatures + during research; for example, configuration files will have a very + specific set of changes, so ignore them; certain installation files + may be removed, etc. Return ``True`` if a diffstat signature should be + ignored, + """ + return False + def researchVerbose(self, filename): + """ + Allows an application to exclude certain dirty files from the output + report; usually this will just be parametrized files, since those are + guaranteed to have changes. Return ``True`` if a file should only + be displayed in verbose mode. + """ + return filename in self.parametrized_files @staticmethod def make(name): """Makes an application, but uses the correct subtype if available.""" try: __import__("wizard.app." + name) return getattr(wizard.app, name).Application(name) - except ImportError: - return Application(name) + except ImportError as error: + # XXX ugly hack to check if the import error is from the top level + # module we care about or a submodule. should be an archetectural change. + if error.args[0].split()[-1]==name: + return Application(name) + else: + raise class ApplicationVersion(object): """Represents an abstract notion of a version for an application, where @@ -352,7 +558,7 @@ def expand_re(val): def make_extractors(seed): """ - Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into + Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into extractor functions (which take a :class:`wizard.deploy.Deployment` and return the value of the second subpattern of ``regex`` when matched with the contents of ``file``). @@ -361,7 +567,7 @@ def make_extractors(seed): def make_substitutions(seed): """ - Take a dictionary of ``key``s to ``(file, regex)`` tuples and convert them into substitution + Take a dictionary of ``key`` to ``(file, regex)`` tuples and convert them into substitution functions (which take a :class:`wizard.deploy.Deployment`, replace the second subpattern of ``regex`` with ``key`` in ``file``, and returns the number of substitutions made.) """ @@ -463,41 +669,74 @@ def filename_regex_substitution(key, files, regex): def backup_database(outdir, deployment): """ - Generic database backup function. Assumes that ``WIZARD_DBNAME`` - is extractable, and that :func:`wizard.scripts.get_sql_credentials` - works. + Generic database backup function for MySQL. + """ + # XXX: Change this once deployments support multiple dbs + if deployment.application.database == "mysql": + return backup_mysql_database(outdir, deployment) + else: + raise NotImplementedError + +def backup_mysql_database(outdir, deployment): + """ + Database backups for MySQL using the :command:`mysqldump` utility. """ - sh = shell.Shell() outfile = os.path.join(outdir, "db.sql") try: - sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment)) - sh.call("gzip", "--best", outfile) + shell.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment.dsn)) + shell.call("gzip", "--best", outfile) except shell.CallError as e: - shutil.rmtree(outdir) raise BackupFailure(e.stderr) def restore_database(backup_dir, deployment): - sh = shell.Shell() + """ + Generic database restoration function for MySQL. + """ + # XXX: see backup_database + if deployment.application.database == "mysql": + return restore_mysql_database(backup_dir, deployment) + else: + raise NotImplementedError + +def restore_mysql_database(backup_dir, deployment): + """ + Database restoration for MySQL by piping SQL commands into :command:`mysql`. + """ if not os.path.exists(backup_dir): raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2]) sql = open(os.path.join(backup_dir, "db.sql"), 'w+') - sh.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql) + shell.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql) sql.seek(0) - sh.call("mysql", *get_mysql_args(deployment), stdin=sql) + shell.call("mysql", *get_mysql_args(deployment.dsn), stdin=sql) sql.close() -def get_mysql_args(d): - # XXX: add support for getting these out of options - vars = d.extract() - if 'WIZARD_DBNAME' not in vars: - raise app.BackupFailure("Could not determine database name") - triplet = scripts.get_sql_credentials(vars) +def remove_database(deployment): + """ + Generic database removal function. Actually, not so generic because we + go and check if we're on scripts and if we are run a different command. + """ + if deployment.dsn.host == "sql.mit.edu": + try: + shell.call("/mit/scripts/sql/bin/drop-database", deployment.dsn.database) + return + except shell.CallError: + pass + engine = sqlalchemy.create_engine(deployment.dsn) + engine.execute("DROP DATABASE `%s`" % deployment.dsn.database) + +def get_mysql_args(dsn): + """ + Extracts arguments that would be passed to the command line mysql utility + from a deployment. + """ args = [] - if triplet is not None: - server, user, password = triplet - args += ["-h", server, "-u", user, "-p" + password] - name = shlex.split(vars['WIZARD_DBNAME'])[0] - args.append(name) + if dsn.host: + args += ["-h", dsn.host] + if dsn.username: + args += ["-u", dsn.username] + if dsn.password: + args += ["-p" + dsn.password] + args += [dsn.database] return args class Error(wizard.Error): @@ -541,7 +780,42 @@ class NoSuchApplication(Error): def __init__(self, app): self.app = app -class UpgradeFailure(Error): +class Failure(Error): + """ + Represents a failure when performing some double-dispatched operation + such as an installation or an upgrade. Failure classes are postfixed + with Failure, not Error. + """ + pass + +class InstallFailure(Error): + """Installation failed for unknown reason.""" + def __str__(self): + return """ + +ERROR: Installation failed for unknown reason. You can +retry the installation by appending --retry to the installation +command.""" + +class RecoverableInstallFailure(InstallFailure): + """ + Installation failed, but we were able to determine what the + error was, and should give the user a second chance if we were + running interactively. + """ + #: List of the errors that were found. + errors = None + def __init__(self, errors): + self.errors = errors + def __str__(self): + return """ + +ERROR: Installation failed due to the following errors: %s + +You can retry the installation by appending --retry to the +installation command.""" % ", ".join(self.errors) + +class UpgradeFailure(Failure): """Upgrade script failed.""" #: String details of failure (possibly stdout or stderr output) details = None @@ -554,21 +828,29 @@ ERROR: Upgrade script failed, details: %s""" % self.details -class UpgradeVerificationFailure(Error): +class UpgradeVerificationFailure(Failure): """Upgrade script passed, but website wasn't accessible afterwards""" - #: String details of failure (possibly stdout or stderr output) + def __str__(self): + return """ + +ERROR: Upgrade script passed, but website wasn't accessible afterwards. Check +the debug logs for the contents of the page.""" + +class BackupFailure(Failure): + """Backup script failed.""" + #: String details of failure details = None def __init__(self, details): self.details = details def __str__(self): return """ -ERROR: Upgrade script passed, but website wasn't accessible afterwards. Details: +ERROR: Backup script failed, details: %s""" % self.details -class BackupFailure(Error): - """Backup script failed.""" +class RestoreFailure(Failure): + """Restore script failed.""" #: String details of failure details = None def __init__(self, details): @@ -576,12 +858,12 @@ class BackupFailure(Error): def __str__(self): return """ -ERROR: Backup script failed, details: +ERROR: Restore script failed, details: %s""" % self.details -class RestoreFailure(Error): - """Restore script failed.""" +class RemoveFailure(Failure): + """Remove script failed.""" #: String details of failure details = None def __init__(self, details): @@ -589,6 +871,6 @@ class RestoreFailure(Error): def __str__(self): return """ -ERROR: Restore script failed, details: +ERROR: Remove script failed, details: %s""" % self.details