X-Git-Url: https://scripts.mit.edu/gitweb/wizard.git/blobdiff_plain/dfaede6bb50a8bce330f87c1eaadf09fcee28be1..319f82896fb658d9bbaf8d01d6c2c7410eb967c7:/wizard/deploy.py diff --git a/wizard/deploy.py b/wizard/deploy.py index cc43d85..12a2781 100644 --- a/wizard/deploy.py +++ b/wizard/deploy.py @@ -9,9 +9,16 @@ import fileinput import logging import decorator import datetime +import tempfile +import time +import traceback +import shutil +import errno +import pkg_resources +import urlparse import wizard -from wizard import app, git, old_log, scripts, shell, util +from wizard import app, git, old_log, shell, sql, util ## -- Global Functions -- @@ -48,6 +55,11 @@ def parse_install_lines(show, versions_store, yield_errors = False, user = None) d = Deployment.parse(line) name = d.application.name except app.NoSuchApplication as e: + if not e.location: + try: + e.location = line.split(':')[0] + except IndexError: + e.location = line if yield_errors: yield e continue @@ -56,13 +68,51 @@ def parse_install_lines(show, versions_store, yield_errors = False, user = None) logging.warning("Error with '%s'" % line.rstrip()) continue # filter - if name + "-" + str(d.version) in show or name in show: + if name + "-" + util.truncate(str(d.version)) in show or name in show: pass else: continue # yield yield d +def web(dir, url=None): + """ + Attempts to determine the URL a directory would be web-accessible at. + If ``url`` is specified, automatically use it. Returns a generator which + produces a list of candidate urls. + + This function implements a plugin interface named :ref:`wizard.deploy.web`. + """ + if url: + if isinstance(url, str): + url = urlparse.urlparse(url) + logging.info("wizard.deploy.web: Using default URL %s", url) + yield url + return + + for entry in pkg_resources.iter_entry_points("wizard.deploy.web"): + logging.debug("wizard.deploy.web: Processing %s", entry) + f = entry.load() + for r in f(dir): + if isinstance(r, str): + r = urlparse.urlparse(r) + logging.info("wizard.deploy.web: Using plugin-supplied URL %s", r) + yield r + + # try the environment + host = os.getenv("WIZARD_WEB_HOST") + path = os.getenv("WIZARD_WEB_PATH") + if host is not None and path is not None: + r = urlparse.ParseResult( + "http", + host, + path.rstrip('/'), + "", "", "") + logging.info("wizard.deploy.web: Using environment URL %s", r) + yield r + + logging.info("wizard.deploy.web: Exhausted URLs") + ## -- Model Objects -- @decorator.decorator @@ -76,15 +126,20 @@ def chdir_to_location(f, self, *args, **kwargs): class Deployment(object): """ - Represents a deployment of an autoinstall, e.g. directory - that has ``.scripts`` directory or ``.scripts-version`` - file in it. Supply ``version`` with an :class:`ApplicationVersion` only if - you were reading from the :term:`versions store` and care about - speed (data from there can be stale). + Represents a deployment of an autoinstall, e.g. directory that has a + ``.wizard`` directory in it. Supply ``version`` with an + :class:`ApplicationVersion` only if you were reading from the + :term:`versions store` and care about speed (data from there can be + stale). The Deployment interface is somewhat neutered, so you may want to use :class:`WorkingCopy` or :class:`ProductionCopy` for more powerful operations. + + .. note:: + + For legacy purposes, deployments can also be marked by a + ``.scripts`` directory or a ``.scripts-version`` file. """ #: Absolute path to the deployment location = None @@ -94,6 +149,10 @@ class Deployment(object): # some cache variables self._read_cache = {} self._old_log = None + self._dsn = None + self._url = None + self._urlGen = None + self._wizard_dir = None def invalidateCache(self): """ Invalidates all cached variables. This currently applies to @@ -102,6 +161,13 @@ class Deployment(object): self._app_version = None self._read_cache = {} self._old_log = None + def setAppVersion(self, app_version): + """ + Manually resets the application version; useful if the working + copy is off in space (i.e. not anchored to something we can + git describe off of) or there is no metadata to be heard of. + """ + self._app_version = app_version def read(self, file, force = False): """ Reads a file's contents, possibly from cache unless ``force`` @@ -121,21 +187,29 @@ class Deployment(object): """ return self.application.extract(self) - def verify(self): + def verify(self, no_touch=False): """ Checks if this is an autoinstall, throws an exception if there - are problems. + are problems. If ``no_touch`` is ``True``, it will not attempt + edit the installation. """ with util.ChangeDirectory(self.location): has_git = os.path.isdir(".git") - has_scripts = os.path.isdir(".scripts") - if not has_git and has_scripts: + has_wizard = os.path.isdir(".wizard") + if not has_wizard and os.path.isdir(".scripts"): + # LEGACY + os.symlink(".scripts", ".wizard") + has_wizard = True + if not has_git and has_wizard: raise CorruptedAutoinstallError(self.location) - elif has_git and not has_scripts: + elif has_git and not has_wizard: raise AlreadyVersionedError(self.location) - elif not has_git and not has_scripts: + # LEGACY + elif not has_git and not has_wizard: if os.path.isfile(".scripts-version"): raise NotMigratedError(self.location) + else: + raise NotAutoinstallError(self.location) def verifyTag(self, srv_path): """ @@ -144,9 +218,9 @@ class Deployment(object): """ repo = self.application.repository(srv_path) try: - shell.Shell().eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--') + shell.eval("git", "--git-dir", repo, "rev-parse", self.app_version.wizard_tag, '--') except shell.CallError: - raise NoTagError(self.app_version.scripts_tag) + raise NoTagError(self.app_version.wizard_tag) def verifyGit(self, srv_path): """ @@ -156,25 +230,24 @@ class Deployment(object): corresponds to the one in the remote repository. """ with util.ChangeDirectory(self.location): - sh = shell.Shell() repo = self.application.repository(srv_path) def repo_rev_parse(tag): - return sh.eval("git", "--git-dir", repo, "rev-parse", tag) + return shell.eval("git", "--git-dir", repo, "rev-parse", tag) def self_rev_parse(tag): try: - return sh.safeCall("git", "rev-parse", tag, strip=True) + return shell.safeCall("git", "rev-parse", tag, strip=True) except shell.CallError: raise NoLocalTagError(tag) def compare_tags(tag): return repo_rev_parse(tag) == self_rev_parse(tag) if not compare_tags(self.app_version.pristine_tag): raise InconsistentPristineTagError(self.app_version.pristine_tag) - if not compare_tags(self.app_version.scripts_tag): - raise InconsistentScriptsTagError(self.app_version.scripts_tag) - parent = repo_rev_parse(self.app_version.scripts_tag) - merge_base = sh.safeCall("git", "merge-base", parent, "HEAD", strip=True) + if not compare_tags(self.app_version.wizard_tag): + raise InconsistentWizardTagError(self.app_version.wizard_tag) + parent = repo_rev_parse(self.app_version.wizard_tag) + merge_base = shell.safeCall("git", "merge-base", parent, "HEAD", strip=True) if merge_base != parent: - raise HeadNotDescendantError(self.app_version.scripts_tag) + raise HeadNotDescendantError(self.app_version.wizard_tag) def verifyConfigured(self): """ @@ -189,11 +262,22 @@ class Deployment(object): Checks if our version and the version number recorded in a file are consistent. """ + real = self.detectVersion() + if not str(real) == self.app_version.pristine_tag.partition('-')[2]: + raise VersionMismatchError(real, self.version) + + @chdir_to_location + def detectVersion(self): + """ + Returns the real version, based on filesystem, of install. + + Throws a :class:`VersionDetectionError` if we couldn't figure out + what the real version was. + """ real = self.application.detectVersion(self) if not real: raise VersionDetectionError - elif not str(real) == self.app_version.pristine_tag.partition('-')[2]: - raise VersionMismatchError(real, self.version) + return real @property @chdir_to_location @@ -203,26 +287,42 @@ class Deployment(object): @property def migrated(self): """Whether or not the autoinstalls has been migrated.""" - return os.path.isdir(self.scripts_dir) + return os.path.isdir(self.wizard_dir) + @property + def wizard_dir(self): + """The absolute path of the Wizard directory.""" + return os.path.join(self.location, ".wizard") @property - def scripts_dir(self): - """The absolute path of the ``.scripts`` directory.""" - return os.path.join(self.location, '.scripts') + def backup_dir(self): + """The absolute path to ``.wizard/backups``.""" + return os.path.join(self.wizard_dir, "backups") + # LEGACY @property def old_version_file(self): """ - The absolute path of either ``.scripts-version`` (for unmigrated - installs) or ``.scripts/version``. - - .. note:: - - Use of this is discouraged for migrated installs. + The absolute path of either ``.scripts-version``. """ return os.path.join(self.location, '.scripts-version') @property + def blacklisted_file(self): + """The absolute path of the ``.wizard/blacklisted`` file.""" + return os.path.join(self.wizard_dir, 'blacklisted') + @property + def pending_file(self): + """The absolute path of the ``.wizard/pending`` file.""" + return os.path.join(self.wizard_dir, 'pending') + @property def version_file(self): - """The absolute path of the ``.scripts/version`` file.""" - return os.path.join(self.scripts_dir, 'version') + """The absolute path of the ``.wizard/version`` file.""" + return os.path.join(self.wizard_dir, 'version') + @property + def dsn_file(self): + """The absolute path of the :file:`.wizard/dsn` override file.""" + return os.path.join(self.wizard_dir, 'dsn') + @property + def url_file(self): + """The absolute path of the :file:`.wizard/url` override file.""" + return os.path.join(self.wizard_dir, 'url') @property def application(self): """The :class:`app.Application` of this deployment.""" @@ -255,8 +355,40 @@ class Deployment(object): except shell.CallError: pass if not self._app_version: - self._app_version = self.old_log[-1].version + # LEGACY + try: + self._app_version = self.old_log[-1].version + except old_log.ScriptsVersionNoSuchFile: + pass + if not self._app_version: + appname = shell.eval("git", "config", "remote.origin.url").rpartition("/")[2].partition(".")[0] + self._app_version = app.ApplicationVersion.make(appname, "unknown") return self._app_version + @property + def dsn(self): + """The :class:`sqlalchemy.engine.url.URL` for this deployment.""" + if not self._dsn: + self._dsn = sql.auth(self.application.dsn(self)) + return self._dsn + @property + def url(self): + """The :class:`urlparse.ParseResult` for this deployment.""" + if not self._url: + self.nextUrl() + return self._url + def nextUrl(self): + """ + Initializes :attr:`url` with a possible URL the web application may + be located at. It may be called again to switch to another possible + URL, usually in the event of a web access failure. + """ + if not self._urlGen: + self._urlGen = web(self.location, self.application.url(self)) + try: + self._url = self._urlGen.next() # pylint: disable-msg=E1101 + return self._url + except StopIteration: + raise UnknownWebPath @staticmethod def parse(line): """ @@ -296,15 +428,43 @@ class ProductionCopy(Deployment): """ Performs a backup of database schemas and other non-versioned data. """ - backupdir = os.path.join(self.location, ".scripts", "backups") - backup = str(self.version) + "-" + datetime.date.today().isoformat() - outdir = os.path.join(backupdir, backup) - if not os.path.exists(backupdir): - os.mkdir(backupdir) - if os.path.exists(outdir): - util.safe_unlink(outdir) - os.mkdir(outdir) - self.application.backup(self, outdir, options) + # There are retarded amounts of race-safety in this function, + # because we do NOT want to claim to have made a backup, when + # actually something weird happened to it. + if not os.path.exists(self.backup_dir): + try: + os.mkdir(self.backup_dir) + except OSError as e: + if e.errno == errno.EEXIST: + pass + else: + raise + tmpdir = tempfile.mkdtemp() # actually will be kept around + try: + self.application.backup(self, tmpdir, options) + except app.BackupFailure: + # the backup is bogus, don't let it show up + shutil.rmtree(tmpdir) + raise + backup = None + with util.LockDirectory(os.path.join(self.backup_dir, "lock")): + while 1: + backup = str(self.version) + "-" + datetime.datetime.today().strftime("%Y-%m-%dT%H%M%S") + outdir = os.path.join(self.backup_dir, backup) + if os.path.exists(outdir): + logging.warning("Backup: A backup occurred in the last second. Trying again in a second...") + time.sleep(1) + continue + try: + shutil.move(tmpdir, outdir) + except: + # don't leave half-baked stuff lying around + try: + shutil.rmtree(outdir) + except OSError: + pass + raise + break return backup @chdir_to_location def restore(self, backup, options): @@ -314,24 +474,39 @@ class ProductionCopy(Deployment): does, so you probably do NOT want to call this elsewhere unless you know what you're doing (call 'wizard restore' instead). """ - backup_dir = os.path.join(".scripts", "backups", backup) - return self.application.restore(self, backup_dir, options) + return self.application.restore(self, os.path.join(self.backup_dir, backup), options) + @chdir_to_location + def remove(self, options): + """ + Deletes all non-local or non-filesystem data (such as databases) that + this application uses. + """ + self.application.remove(self, options) + def verifyDatabase(self): + """ + Checks if the autoinstall has a properly configured database. + """ + if not self.application.checkDatabase(self): + raise DatabaseVerificationError def verifyWeb(self): """ - Checks if the autoinstall is viewable from the web. + Checks if the autoinstall is viewable from the web. If you do not run + this, there is no guarantee that the url returned by this application + is the correct one. """ - out = [] - if not self.application.checkWeb(self, out): - raise WebVerificationError(out[0]) + while True: + if not self.application.checkWeb(self): + try: + self.nextUrl() + except UnknownWebPath: + raise WebVerificationError + else: + break def fetch(self, path, post=None): """ Performs a HTTP request on the website. """ - try: - host, basepath = scripts.get_web_host_and_path(self.location) - except (ValueError, TypeError): - raise UnknownWebPath - return util.fetch(host, basepath, path, post) + return util.fetch(self.url.netloc, self.url.path, path, post) # pylint: disable-msg=E1103 class WorkingCopy(Deployment): """ @@ -378,10 +553,11 @@ class Error(wizard.Error): """Base error class for this module""" pass +# LEGACY class NotMigratedError(Error): """ The deployment contains a .scripts-version file, but no .git - or .scripts directory. + or .wizard directory. """ #: Directory of deployment dir = None @@ -391,7 +567,7 @@ class NotMigratedError(Error): return """This installation was not migrated""" class AlreadyVersionedError(Error): - """The deployment contained a .git directory but no .scripts directory.""" + """The deployment contained a .git directory but no .wizard directory.""" #: Directory of deployment dir = None def __init__(self, dir): @@ -400,7 +576,7 @@ class AlreadyVersionedError(Error): return """ ERROR: Directory contains a .git directory, but not -a .scripts directory. If this is not a corrupt +a .wizard directory. If this is not a corrupt migration, this means that the user was versioning their install using Git.""" @@ -417,7 +593,7 @@ ERROR: The install was well-formed, but not configured (essential configuration files were not found.)""" class CorruptedAutoinstallError(Error): - """The install was missing a .git directory, but had a .scripts directory.""" + """The install was missing a .git directory, but had a .wizard directory.""" #: Directory of the corrupted install dir = None def __init__(self, dir): @@ -425,21 +601,25 @@ class CorruptedAutoinstallError(Error): def __str__(self): return """ -ERROR: Directory contains a .scripts directory, +ERROR: Directory contains a .wizard directory, but not a .git directory.""" class NotAutoinstallError(Error): - """The directory was not an autoinstall, due to missing .scripts-version file.""" - #: Directory in question + """Application is not an autoinstall.""" + #: Directory of the not autoinstall dir = None def __init__(self, dir): self.dir = dir def __str__(self): return """ -ERROR: Could not find .scripts-version file. Are you sure -this is an autoinstalled application? -""" +ERROR: The directory + + %s + +does not appear to be an autoinstall. If you are in a +subdirectory of an autoinstall, you need to use the root +directory for the autoinstall.""" % self.dir class NoTagError(Error): """Deployment has a tag that does not have an equivalent in upstream repository.""" @@ -475,8 +655,8 @@ class InconsistentPristineTagError(Error): ERROR: Local pristine tag %s did not match repository's. This probably means an upstream rebase occured.""" % self.tag -class InconsistentScriptsTagError(Error): - """Scripts tag commit ID does not match upstream scripts tag commit ID.""" +class InconsistentWizardTagError(Error): + """Wizard tag commit ID does not match upstream wizard tag commit ID.""" #: Inconsistent tag tag = None def __init__(self, tag): @@ -484,7 +664,7 @@ class InconsistentScriptsTagError(Error): def __str__(self): return """ -ERROR: Local scripts tag %s did not match repository's. This +ERROR: Local wizard tag %s did not match repository's. This probably means an upstream rebase occurred.""" % self.tag class HeadNotDescendantError(Error): @@ -524,19 +704,22 @@ version %s.""" % (self.real_version, self.git_version) class WebVerificationError(Error): """Could not access the application on the web""" - #: Contents of web page access - contents = None - def __init__(self, contents): - self.contents = contents def __str__(self): return """ ERROR: We were not able to access the application on the web. This may indicate that the website is behind -authentication on the htaccess level. The contents -of the page were: +authentication on the htaccess level. You can find +the contents of the page from the debug backtraces.""" -%s""" % self.contents +class DatabaseVerificationError(Error): + """Could not access the database""" + def __str__(self): + return """ + +ERROR: We were not able to access the database for +this application; this probably means that your database +configuration is misconfigured.""" class UnknownWebPath(Error): """Could not determine application's web path.""" @@ -548,4 +731,3 @@ host and path were in order to perform a web request on the application. You can specify this manually using the WIZARD_WEB_HOST and WIZARD_WEB_PATH environment variables.""" -