X-Git-Url: https://scripts.mit.edu/gitweb/wizard.git/blobdiff_plain/03c6b34d4878e828dda99514941fb586ed886a3a..db709749a3e3b1cba85a6869c87af925ae33d441:/wizard/deploy.py diff --git a/wizard/deploy.py b/wizard/deploy.py index 6197a9a..6d3ec67 100644 --- a/wizard/deploy.py +++ b/wizard/deploy.py @@ -1,19 +1,22 @@ """ Object model for querying information and manipulating deployments -of autoinstalls. Every :class:`Deployment` has an :class:`ApplicationVersion` -which in turn has an :class:`Application`. +of autoinstalls. Every :class:`Deployment` has an :class:`app.ApplicationVersion` +which in turn has an :class:`app.Application`. """ import os.path import fileinput -import dateutil.parser -import distutils.version -import tempfile import logging +import decorator +import datetime +import tempfile +import time +import traceback import shutil +import errno import wizard -from wizard import git, old_log, scripts, shell, util +from wizard import app, git, old_log, scripts, shell, sql, util ## -- Global Functions -- @@ -38,7 +41,7 @@ def parse_install_lines(show, versions_store, yield_errors = False, user = None) log output. """ if not show: - show = applications() + show = app.applications() elif isinstance(show, str): # otherwise, frozenset will treat string as an iterable show = frozenset([show]) @@ -49,7 +52,12 @@ def parse_install_lines(show, versions_store, yield_errors = False, user = None) try: d = Deployment.parse(line) name = d.application.name - except NoSuchApplication as e: + except app.NoSuchApplication as e: + if not e.location: + try: + e.location = line.split(':')[0] + except IndexError: + e.location = line if yield_errors: yield e continue @@ -67,13 +75,26 @@ def parse_install_lines(show, versions_store, yield_errors = False, user = None) ## -- Model Objects -- +@decorator.decorator +def chdir_to_location(f, self, *args, **kwargs): + """ + Decorator for making a function have working directory + :attr:`Deployment.location`. + """ + with util.ChangeDirectory(self.location): + return f(self, *args, **kwargs) + class Deployment(object): """ - Represents a deployment of an autoinstall, e.g. directory in a user's - web_scripts that has ``.scripts`` directory or ``.scripts-version`` + Represents a deployment of an autoinstall, e.g. directory + that has ``.scripts`` directory or ``.scripts-version`` file in it. Supply ``version`` with an :class:`ApplicationVersion` only if you were reading from the :term:`versions store` and care about speed (data from there can be stale). + + The Deployment interface is somewhat neutered, so you may + want to use :class:`WorkingCopy` or :class:`ProductionCopy` for + more powerful operations. """ #: Absolute path to the deployment location = None @@ -83,6 +104,16 @@ class Deployment(object): # some cache variables self._read_cache = {} self._old_log = None + self._dsn = None + self._url = None + def invalidateCache(self): + """ + Invalidates all cached variables. This currently applies to + :attr:`app_version`, :attr:`old_log` and :meth:`read`. + """ + self._app_version = None + self._read_cache = {} + self._old_log = None def read(self, file, force = False): """ Reads a file's contents, possibly from cache unless ``force`` @@ -101,45 +132,6 @@ class Deployment(object): and local configuration. """ return self.application.extract(self) - def parametrize(self, dir): - """ - Edits files in ``dir`` to replace WIZARD_* variables with literal - instances. This is used for constructing virtual merge bases, and - as such dir will generally not equal :attr:`location`. - """ - return self.application.parametrize(self, dir) - def upgrade(self, version, options): - """ - Performs an upgrae of database schemas and other non-versioned data. - """ - with util.ChangeDirectory(self.location): - return self.application.upgrade(self, version, options) - def backup(self, options): - """ - Performs a backup of database schemas and other non-versioned data. - """ - with util.ChangeDirectory(self.location): - return self.application.backup(self, options) - def restore(self, backup, options): - """ - Restores a backup. Destroys state, so be careful! Also, this does - NOT restore the file-level backup, which is what 'wizard restore' - does, so you probably do NOT want to call this elsewhere unless - you know what you're doing. - """ - with util.ChangeDirectory(self.location): - return self.application.restore(self, backup, options) - def prepareConfig(self): - """ - Edits files in the deployment such that any user-specific configuration - is replaced with generic WIZARD_* variables. - """ - return self.application.prepareConfig(self) - def checkConfig(self, deployment): - """ - Checks if the application is configured. - """ - raise NotImplemented def verify(self): """ @@ -156,6 +148,8 @@ class Deployment(object): elif not has_git and not has_scripts: if os.path.isfile(".scripts-version"): raise NotMigratedError(self.location) + else: + raise NotAutoinstallError(self.location) def verifyTag(self, srv_path): """ @@ -164,7 +158,7 @@ class Deployment(object): """ repo = self.application.repository(srv_path) try: - shell.Shell().eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--') + shell.eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--') except shell.CallError: raise NoTagError(self.app_version.scripts_tag) @@ -176,13 +170,12 @@ class Deployment(object): corresponds to the one in the remote repository. """ with util.ChangeDirectory(self.location): - sh = shell.Shell() repo = self.application.repository(srv_path) def repo_rev_parse(tag): - return sh.eval("git", "--git-dir", repo, "rev-parse", tag) + return shell.eval("git", "--git-dir", repo, "rev-parse", tag) def self_rev_parse(tag): try: - return sh.safeCall("git", "rev-parse", tag, strip=True) + return shell.safeCall("git", "rev-parse", tag, strip=True) except shell.CallError: raise NoLocalTagError(tag) def compare_tags(tag): @@ -192,7 +185,7 @@ class Deployment(object): if not compare_tags(self.app_version.scripts_tag): raise InconsistentScriptsTagError(self.app_version.scripts_tag) parent = repo_rev_parse(self.app_version.scripts_tag) - merge_base = sh.safeCall("git", "merge-base", parent, "HEAD", strip=True) + merge_base = shell.safeCall("git", "merge-base", parent, "HEAD", strip=True) if merge_base != parent: raise HeadNotDescendantError(self.app_version.scripts_tag) @@ -203,36 +196,31 @@ class Deployment(object): if not self.configured: raise NotConfiguredError(self.location) + @chdir_to_location def verifyVersion(self): """ Checks if our version and the version number recorded in a file are consistent. """ - real = self.application.detectVersion(self) - if not real: - raise VersionDetectionError - elif not str(real) == self.app_version.pristine_tag.partition('-')[2]: + real = self.detectVersion() + if not str(real) == self.app_version.pristine_tag.partition('-')[2]: raise VersionMismatchError(real, self.version) - def verifyWeb(self): - """ - Checks if the autoinstall is viewable from the web. + @chdir_to_location + def detectVersion(self): """ - out = [] - if not self.application.checkWeb(self, out): - raise WebVerificationError(out[0]) + Returns the real version, based on filesystem, of install. - def fetch(self, path, post=None): + Throws a :class:`VersionDetectionError` if we couldn't figure out + what the real version was. """ - Performs a HTTP request on the website. - """ - try: - host, basepath = scripts.get_web_host_and_path(self.location) - except (ValueError, TypeError): - raise UnknownWebPath - return util.fetch(host, basepath, path, post) + real = self.application.detectVersion(self) + if not real: + raise VersionDetectionError + return real @property + @chdir_to_location def configured(self): """Whether or not an autoinstall has been configured/installed for use.""" return self.application.checkConfig(self) @@ -260,8 +248,16 @@ class Deployment(object): """The absolute path of the ``.scripts/version`` file.""" return os.path.join(self.scripts_dir, 'version') @property + def dsn_file(self): + """The absolute path of the :file:`.scripts/dsn` override file.""" + return os.path.join(self.scripts_dir, 'dsn') + @property + def url_file(self): + """The absolute path of the :file:`.scripts/url` override file.""" + return os.path.join(self.scripts_dir, 'url') + @property def application(self): - """The :class:`Application` of this deployment.""" + """The :class:`app.Application` of this deployment.""" return self.app_version.application @property def old_log(self): @@ -281,18 +277,44 @@ class Deployment(object): return self.app_version.version @property def app_version(self): - """The :class:`ApplicationVersion` of this deployment.""" + """The :class:`app.ApplicationVersion` of this deployment.""" if not self._app_version: if os.path.isdir(os.path.join(self.location, ".git")): try: with util.ChangeDirectory(self.location): appname, _, version = git.describe().partition('-') - self._app_version = ApplicationVersion.make(appname, version) + self._app_version = app.ApplicationVersion.make(appname, version) except shell.CallError: pass if not self._app_version: - self._app_version = self.old_log[-1].version + try: + self._app_version = self.old_log[-1].version + except old_log.ScriptsVersionNoSuchFile: + pass + if not self._app_version: + appname = shell.eval("git", "config", "remote.origin.url").rpartition("/")[2].partition(".")[0] + self._app_version = app.ApplicationVersion.make(appname, "unknown") return self._app_version + @property + def dsn(self): + """The :class:`sqlalchemy.engine.url.URL` for this deployment.""" + if not self._dsn: + self._dsn = sql.fill_url(self.application.dsn(self)) + return self._dsn + @property + def url(self): + """The :class:`urlparse.ParseResult` for this deployment.""" + if not self._url: + self._url = scripts.fill_url(self.location, self.application.url(self)) + if not self._url: + raise UnknownWebPath + return self._url + def enableOldStyleUrls(self): + """ + Switches to using http://user.scripts.mit.edu/~user/app URLs. + No effect if they have an explicit .scripts/url override. + """ + self._url = scripts.fill_url(self.location, self.application.url(self), old_style = True) @staticmethod def parse(line): """ @@ -308,235 +330,149 @@ class Deployment(object): try: location, deploydir = line.split(":") except ValueError: - return Deployment(line) # lazy loaded version + return ProductionCopy(line) # lazy loaded version try: - return Deployment(location, version=ApplicationVersion.parse(deploydir)) + return ProductionCopy(location, version=app.ApplicationVersion.parse(deploydir)) except Error as e: e.location = location raise e -class Application(object): - """Represents an application, i.e. mediawiki or phpbb.""" - #: String name of the application - name = None - #: Dictionary of version strings to :class:`ApplicationVersion`. - #: See also :meth:`makeVersion`. - versions = None - #: List of files that need to be modified when parametrizing. - #: This is a class-wide constant, and should not normally be modified. - parametrized_files = [] - def __init__(self, name): - self.name = name - self.versions = {} - # cache variables - self._extractors = {} - self._substitutions = {} - def repository(self, srv_path): - """ - Returns the Git repository that would contain this application. - ``srv_path`` corresponds to ``options.srv_path`` from the global baton. - """ - repo = os.path.join(srv_path, self.name + ".git") - if not os.path.isdir(repo): - repo = os.path.join(srv_path, self.name, ".git") - if not os.path.isdir(repo): - raise NoRepositoryError(self.name) - return repo - def makeVersion(self, version): - """ - Creates or retrieves the :class:`ApplicationVersion` singleton for the - specified version. - """ - if version not in self.versions: - self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self) - return self.versions[version] - def extract(self, deployment): - """Extracts wizard variables from a deployment.""" - result = {} - for k,extractor in self.extractors.items(): - result[k] = extractor(deployment) - return result - def parametrize(self, deployment, dir): - """ - Takes a generic source checkout at dir and parametrizes - it according to the values of deployment. - """ - variables = deployment.extract() - for file in self.parametrized_files: - fullpath = os.path.join(dir, file) - try: - contents = open(fullpath, "r").read() - except IOError: - continue - for key, value in variables.items(): - if value is None: continue - contents = contents.replace(key, value) - f = open(fullpath, "w") - f.write(contents) - def resolveConflicts(self, dir): - """ - Takes a directory with conflicted files and attempts to - resolve them. Returns whether or not all conflicted - files were resolved or not. Fully resolved files are - added to the index, but no commit is made. - """ - return False - def prepareConfig(self, deployment): - """ - Takes a deployment and replaces any explicit instances - of a configuration variable with generic WIZARD_* constants. - There is a sane default implementation built on substitutions; - you can override this method to provide arbitrary extra - behavior. - """ - for key, subst in self.substitutions.items(): - subs = subst(deployment) - if not subs and key not in self.deprecated_keys: - logging.warning("No substitutions for %s" % key) - def install(self, version, options): - """ - Run for 'wizard configure' (and, by proxy, 'wizard install') - to configure an application. This assumes that the current - working directory is a deployment. - """ - raise NotImplemented - def upgrade(self, deployment, version, options): +class ProductionCopy(Deployment): + """ + Represents the production copy of a deployment. This copy + is canonical, and is the only one guaranteed to be accessible + via web, have a database, etc. + """ + @chdir_to_location + def upgrade(self, version, options): """ - Run for 'wizard upgrade' to upgrade database schemas and other - non-versioned data in an application. This assumes that - the current working directory is the deployment. + Performs an upgrade of database schemas and other non-versioned data. """ - raise NotImplemented - def backup(self, deployment, options): + return self.application.upgrade(self, version, options) + @chdir_to_location + def backup(self, options): """ - Run for 'wizard backup' and upgrades to backup database schemas - and other non-versioned data in an application. This assumes - that the current working directory is the deployment. + Performs a backup of database schemas and other non-versioned data. """ - raise NotImplemented - def restore(self, deployment, backup, options): + # There are retarded amounts of race-safety in this function, + # because we do NOT want to claim to have made a backup, when + # actually something weird happened to it. + backupdir = os.path.join(self.scripts_dir, "backups") + if not os.path.exists(backupdir): + try: + os.mkdir(backupdir) + except OSError as e: + if e.errno == errno.EEXIST: + pass + else: + raise + tmpdir = tempfile.mkdtemp() # actually will be kept around + try: + self.application.backup(self, tmpdir, options) + except app.BackupFailure: + # the backup is bogus, don't let it show up + shutil.rmtree(tmpdir) + raise + backup = None + with util.LockDirectory(os.path.join(backupdir, "lock")): + while 1: + backup = str(self.version) + "-" + datetime.datetime.today().strftime("%Y-%m-%dT%H%M%S") + outdir = os.path.join(backupdir, backup) + if os.path.exists(outdir): + logging.warning("Backup: A backup occurred in the last second. Trying again in a second...") + time.sleep(1) + continue + try: + shutil.move(tmpdir, outdir) + except: + # don't leave half-baked stuff lying around + try: + shutil.rmtree(outdir) + except OSError: + pass + raise + break + return backup + @chdir_to_location + def restore(self, backup, options): """ - Run for 'wizard restore' and failed upgrades to restore database - and other non-versioned data to a backed up version. This assumes - that the current working directory is the deployment. + Restores a backup. Destroys state, so be careful! Also, this does + NOT restore the file-level backup, which is what 'wizard restore' + does, so you probably do NOT want to call this elsewhere unless + you know what you're doing (call 'wizard restore' instead). """ - raise NotImplemented - def detectVersion(self, deployment): + backup_dir = os.path.join(".scripts", "backups", backup) + return self.application.restore(self, backup_dir, options) + @chdir_to_location + def remove(self, options): """ - Checks source files to determine the version manually. + Deletes all non-local or non-filesystem data (such as databases) that + this application uses. """ - return None - def checkWeb(self, deployment, output=None): + self.application.remove(self, options) + def verifyDatabase(self): """ - Checks if the autoinstall is viewable from the web. Output - should be an empty list that will get mutated by this function. + Checks if the autoinstall has a properly configured database. """ - raise NotImplemented - @property - def extractors(self): + if not self.application.checkDatabase(self): + raise DatabaseVerificationError + def verifyWeb(self): """ - Dictionary of variable names to extractor functions. These functions - take a :class:`Deployment` as an argument and return the value of - the variable, or ``None`` if it could not be found. - See also :func:`wizard.app.filename_regex_extractor`. + Checks if the autoinstall is viewable from the web. """ - return {} - @property - def substitutions(self): + if not self.application.checkWeb(self): + raise WebVerificationError + def fetch(self, path, post=None): """ - Dictionary of variable names to substitution functions. These functions - take a :class:`Deployment` as an argument and modify the deployment such - that an explicit instance of the variable is released with the generic - WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`. + Performs a HTTP request on the website. """ - return {} - @staticmethod - def make(name): - """Makes an application, but uses the correct subtype if available.""" - try: - __import__("wizard.app." + name) - return getattr(wizard.app, name).Application(name) - except ImportError: - return Application(name) - -class ApplicationVersion(object): - """Represents an abstract notion of a version for an application, where - ``version`` is a :class:`distutils.version.LooseVersion` and - ``application`` is a :class:`Application`.""" - #: The :class:`distutils.version.LooseVersion` of this instance. - version = None - #: The :class:`Application` of this instance. - application = None - def __init__(self, version, application): - self.version = version - self.application = application - @property - def tag(self): + return util.fetch(self.url.netloc, self.url.path, path, post) # pylint: disable-msg=E1103 + +class WorkingCopy(Deployment): + """ + Represents a temporary clone of a deployment that we can make + modifications to without fear of interfering with a production + deployment. More operations are permitted on these copies. + """ + def setAppVersion(self, app_version): """ - Returns the name of the git describe tag for the commit the user is - presently on, something like mediawiki-1.2.3-scripts-4-g123abcd + Manually resets the application version; useful if the working + copy is off in space (i.e. not anchored to something we can + git describe off of.) """ - return "%s-%s" % (self.application, self.version) - @property - def scripts_tag(self): + self._app_version = app_version + @chdir_to_location + def parametrize(self, deployment): """ - Returns the name of the Git tag for this version. + Edits files in ``dir`` to replace WIZARD_* variables with literal + instances based on ``deployment``. This is used for constructing + virtual merge bases, and as such ``deployment`` will generally not + equal ``self``. """ - end = str(self.version).partition('-scripts')[2].partition('-')[0] - return "%s-scripts%s" % (self.pristine_tag, end) - @property - def pristine_tag(self): + return self.application.parametrize(self, deployment) + @chdir_to_location + def prepareConfig(self): """ - Returns the name of the Git tag for the pristine version corresponding - to this version. + Edits files in the deployment such that any user-specific configuration + is replaced with generic WIZARD_* variables. """ - return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0]) - def __cmp__(x, y): - return cmp(x.version, y.version) - @staticmethod - def parse(value): + return self.application.prepareConfig(self) + @chdir_to_location + def resolveConflicts(self): """ - Parses a line from the :term:`versions store` and return - :class:`ApplicationVersion`. - - Use this only for cases when speed is of primary importance; - the data in version is unreliable and when possible, you should - prefer directly instantiating a Deployment and having it query - the autoinstall itself for information. - - The `value` to parse will vary. For old style installs, it - will look like:: - - /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z - - For new style installs, it will look like:: - - APP-x.y.z-scripts + Resolves conflicted files in this working copy. Returns whether or + not all conflicted files were resolved or not. Fully resolved + files are added to the index, but no commit is made. """ - name = value.split("/")[-1] - try: - if name.find("-") != -1: - app, _, version = name.partition("-") - else: - # kind of poor, maybe should error. Generally this - # will actually result in a not found error - app = name - version = "trunk" - except ValueError: - raise DeploymentParseError(deploydir) - return ApplicationVersion.make(app, version) - @staticmethod - def make(app, version): + return self.application.resolveConflicts(self) + @chdir_to_location + def prepareMerge(self): """ - Makes/retrieves a singleton :class:`ApplicationVersion` from - a``app`` and ``version`` string. + Performs various edits to files in the current working directory in + order to make a merge go more smoothly. This is usually + used to fix botched line-endings. """ - try: - # defer to the application for version creation to enforce - # singletons - return applications()[app].makeVersion(version) - except KeyError: - raise NoSuchApplication(app) + return self.application.prepareMerge(self) ## -- Exceptions -- @@ -544,43 +480,6 @@ class Error(wizard.Error): """Base error class for this module""" pass -class NoSuchApplication(Error): - """ - You attempted to reference a :class:`Application` named - ``app``, which is not recognized by Wizard. - """ - #: The name of the application that does not exist. - app = None - #: The location of the autoinstall that threw this variable. - #: This should be set by error handling code when it is availble. - location = None - def __init__(self, app): - self.app = app - -class DeploymentParseError(Error): - """ - Could not parse ``value`` from :term:`versions store`. - """ - #: The value that failed to parse. - value = None - #: The location of the autoinstall that threw this variable. - #: This should be set by error handling code when it is available. - location = None - def __init__(self, value): - self.value = value - -class NoRepositoryError(Error): - """ - :class:`Application` does not appear to have a Git repository - in the normal location. - """ - #: The name of the application that does not have a Git repository. - app = None - def __init__(self, app): - self.app = app - def __str__(self): - return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app - class NotMigratedError(Error): """ The deployment contains a .scripts-version file, but no .git @@ -632,17 +531,21 @@ ERROR: Directory contains a .scripts directory, but not a .git directory.""" class NotAutoinstallError(Error): - """The directory was not an autoinstall, due to missing .scripts-version file.""" - #: Directory in question + """Application is not an autoinstall.""" + #: Directory of the not autoinstall dir = None def __init__(self, dir): self.dir = dir def __str__(self): return """ -ERROR: Could not find .scripts-version file. Are you sure -this is an autoinstalled application? -""" +ERROR: The directory + + %s + +does not appear to be an autoinstall. If you are in a +subdirectory of an autoinstall, you need to use the root +directory for the autoinstall.""" % self.dir class NoTagError(Error): """Deployment has a tag that does not have an equivalent in upstream repository.""" @@ -727,19 +630,22 @@ version %s.""" % (self.real_version, self.git_version) class WebVerificationError(Error): """Could not access the application on the web""" - #: Contents of web page access - contents = None - def __init__(self, contents): - self.contents = contents def __str__(self): return """ ERROR: We were not able to access the application on the web. This may indicate that the website is behind -authentication on the htaccess level. The contents -of the page were: +authentication on the htaccess level. You can find +the contents of the page from the debug backtraces.""" + +class DatabaseVerificationError(Error): + """Could not access the database""" + def __str__(self): + return """ -%s""" % self.contents +ERROR: We were not able to access the database for +this application; this probably means that your database +configuration is misconfigured.""" class UnknownWebPath(Error): """Could not determine application's web path.""" @@ -751,19 +657,3 @@ host and path were in order to perform a web request on the application. You can specify this manually using the WIZARD_WEB_HOST and WIZARD_WEB_PATH environment variables.""" - -_application_list = [ - "mediawiki", "wordpress", "joomla", "e107", "gallery2", - "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django", - # these are technically deprecated - "advancedpoll", "gallery", -] -_applications = None - -def applications(): - """Hash table for looking up string application name to instance""" - global _applications - if not _applications: - _applications = dict([(n,Application.make(n)) for n in _application_list ]) - return _applications -