X-Git-Url: https://scripts.mit.edu/gitweb/wizard.git/blobdiff_plain/33c58b0b437fb89e60c98185d937f8df64e205b1..30380c4b5b28df9670ea5952e14bc485d1d34133:/wizard/deploy.py diff --git a/wizard/deploy.py b/wizard/deploy.py index e6ab0cf..67c4118 100644 --- a/wizard/deploy.py +++ b/wizard/deploy.py @@ -1,29 +1,58 @@ +""" +Object model for querying information and manipulating deployments +of autoinstalls. Every :class:`Deployment` has an :class:`app.ApplicationVersion` +which in turn has an :class:`app.Application`. +""" + import os.path import fileinput -import dateutil.parser -import distutils.version +import logging +import decorator +import datetime +import tempfile +import time +import traceback +import shutil +import errno import wizard -from wizard import log +from wizard import app, git, old_log, scripts, shell, sql, util ## -- Global Functions -- -def getInstallLines(vs): - """Retrieves a list of lines from the version directory that - can be passed to Deployment.parse()""" - if os.path.isfile(vs): - return fileinput.input([vs]) - return fileinput.input([vs + "/" + f for f in os.listdir(vs)]) - -def parse_install_lines(show, options, yield_errors = False): - if not show: show = applications() - show = frozenset(show) - for line in getInstallLines(options.versions_path): +def get_install_lines(versions_store, user=None): + """ + Low level function that retrieves a list of lines from the + :term:`versions store` that can be passed to :meth:`Deployment.parse`. + """ + if os.path.isfile(versions_store): + return fileinput.input([versions_store]) + if user: + return fileinput.input([versions_store + "/" + user]) + return fileinput.input([versions_store + "/" + f for f in sorted(os.listdir(versions_store))]) + +def parse_install_lines(show, versions_store, yield_errors = False, user = None): + """ + Generator function for iterating through all autoinstalls. + Each item is an instance of :class:`Deployment`, or possibly + a :class:`wizard.deploy.Error` if ``yield_errors`` is ``True``. You can + filter out applications and versions by specifying ``app`` + or ``app-1.2.3`` in ``show``. This function may generate + log output. + """ + if not show: + show = app.applications() + elif isinstance(show, str): + # otherwise, frozenset will treat string as an iterable + show = frozenset([show]) + else: + show = frozenset(show) + for line in get_install_lines(versions_store, user): # construction try: d = Deployment.parse(line) name = d.application.name - except NoSuchApplication as e: + except app.NoSuchApplication as e: if yield_errors: yield e continue @@ -41,196 +70,541 @@ def parse_install_lines(show, options, yield_errors = False): ## -- Model Objects -- +@decorator.decorator +def chdir_to_location(f, self, *args, **kwargs): + """ + Decorator for making a function have working directory + :attr:`Deployment.location`. + """ + with util.ChangeDirectory(self.location): + return f(self, *args, **kwargs) + class Deployment(object): - """Represents a deployment of an autoinstall; i.e. a concrete - directory in web_scripts that has .scripts-version in it.""" - def __init__(self, location, log=None, version=None): - """ `location` Location of the deployment - `version` ApplicationVersion of the app (this is cached info) - `log` DeployLog of the app""" - self.location = location + """ + Represents a deployment of an autoinstall, e.g. directory + that has ``.scripts`` directory or ``.scripts-version`` + file in it. Supply ``version`` with an :class:`ApplicationVersion` only if + you were reading from the :term:`versions store` and care about + speed (data from there can be stale). + + The Deployment interface is somewhat neutered, so you may + want to use :class:`WorkingCopy` or :class:`ProductionCopy` for + more powerful operations. + """ + #: Absolute path to the deployment + location = None + def __init__(self, location, version=None): + self.location = os.path.abspath(location) self._app_version = version - self._log = log + # some cache variables + self._read_cache = {} + self._old_log = None + self._dsn = None + self._url = None + def invalidateCache(self): + """ + Invalidates all cached variables. This currently applies to + :attr:`app_version`, :attr:`old_log` and :meth:`read`. + """ + self._app_version = None self._read_cache = {} + self._old_log = None def read(self, file, force = False): - """Reads a file's contents and stuffs it in a cache""" + """ + Reads a file's contents, possibly from cache unless ``force`` + is ``True``. + """ if force or file not in self._read_cache: f = open(os.path.join(self.location, file)) self._read_cache[file] = f.read() f.close() return self._read_cache[file] def extract(self): + """ + Extracts all the values of all variables from deployment. + These variables may be used for parametrizing generic parent + commits and include things such as database access credentials + and local configuration. + """ return self.application.extract(self) - def updateVersion(self, version=None): - """`version` Version string to update to, or leave out to simply - force the creation of .scripts/version file""" - if not version: - version = str(self.version) - else: - self._app_version = self.application.makeVersion(version) - f = open(os.path.join(self.scripts_dir, 'version'), 'w') - f.write(self.application.name + '-' + version + "\n") - f.close() + + def verify(self): + """ + Checks if this is an autoinstall, throws an exception if there + are problems. + """ + with util.ChangeDirectory(self.location): + has_git = os.path.isdir(".git") + has_scripts = os.path.isdir(".scripts") + if not has_git and has_scripts: + raise CorruptedAutoinstallError(self.location) + elif has_git and not has_scripts: + raise AlreadyVersionedError(self.location) + elif not has_git and not has_scripts: + if os.path.isfile(".scripts-version"): + raise NotMigratedError(self.location) + + def verifyTag(self, srv_path): + """ + Checks if the purported version has a corresponding tag + in the upstream repository. + """ + repo = self.application.repository(srv_path) + try: + shell.eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--') + except shell.CallError: + raise NoTagError(self.app_version.scripts_tag) + + def verifyGit(self, srv_path): + """ + Checks if the autoinstall's Git repository makes sense, + checking if the tag is parseable and corresponds to + a real application, and if the tag in this repository + corresponds to the one in the remote repository. + """ + with util.ChangeDirectory(self.location): + repo = self.application.repository(srv_path) + def repo_rev_parse(tag): + return shell.eval("git", "--git-dir", repo, "rev-parse", tag) + def self_rev_parse(tag): + try: + return shell.safeCall("git", "rev-parse", tag, strip=True) + except shell.CallError: + raise NoLocalTagError(tag) + def compare_tags(tag): + return repo_rev_parse(tag) == self_rev_parse(tag) + if not compare_tags(self.app_version.pristine_tag): + raise InconsistentPristineTagError(self.app_version.pristine_tag) + if not compare_tags(self.app_version.scripts_tag): + raise InconsistentScriptsTagError(self.app_version.scripts_tag) + parent = repo_rev_parse(self.app_version.scripts_tag) + merge_base = shell.safeCall("git", "merge-base", parent, "HEAD", strip=True) + if merge_base != parent: + raise HeadNotDescendantError(self.app_version.scripts_tag) + + def verifyConfigured(self): + """ + Checks if the autoinstall is configured running. + """ + if not self.configured: + raise NotConfiguredError(self.location) + + @chdir_to_location + def verifyVersion(self): + """ + Checks if our version and the version number recorded in a file + are consistent. + """ + real = self.application.detectVersion(self) + if not real: + raise VersionDetectionError + elif not str(real) == self.app_version.pristine_tag.partition('-')[2]: + raise VersionMismatchError(real, self.version) + + @property + @chdir_to_location + def configured(self): + """Whether or not an autoinstall has been configured/installed for use.""" + return self.application.checkConfig(self) + @property + def migrated(self): + """Whether or not the autoinstalls has been migrated.""" + return os.path.isdir(self.scripts_dir) @property def scripts_dir(self): + """The absolute path of the ``.scripts`` directory.""" return os.path.join(self.location, '.scripts') @property - def version_file(self): + def old_version_file(self): + """ + The absolute path of either ``.scripts-version`` (for unmigrated + installs) or ``.scripts/version``. + + .. note:: + + Use of this is discouraged for migrated installs. + """ return os.path.join(self.location, '.scripts-version') @property + def version_file(self): + """The absolute path of the ``.scripts/version`` file.""" + return os.path.join(self.scripts_dir, 'version') + @property + def dsn_file(self): + """The absolute path of the :file:`.scripts/dsn` override file.""" + return os.path.join(self.scripts_dir, 'dsn') + @property + def url_file(self): + """The absolute path of the :file:`.scripts/url` override file.""" + return os.path.join(self.scripts_dir, 'url') + @property def application(self): + """The :class:`app.Application` of this deployment.""" return self.app_version.application @property - def log(self): - if not self._log: - self._log = log.DeployLog.load(self.version_file) - return self._log + def old_log(self): + """ + The :class:`wizard.old_log.Log` of this deployment. This + is only applicable to un-migrated autoinstalls. + """ + if not self._old_log: + self._old_log = old_log.DeployLog.load(self) + return self._old_log @property def version(self): - """Returns the distutils Version of the deployment""" + """ + The :class:`distutils.version.LooseVersion` of this + deployment. + """ return self.app_version.version @property - def app_version(self, force = False): - """Returns the ApplicationVersion of the deployment""" - if self._app_version and not force: return self._app_version - else: return self.log[-1].version + def app_version(self): + """The :class:`app.ApplicationVersion` of this deployment.""" + if not self._app_version: + if os.path.isdir(os.path.join(self.location, ".git")): + try: + with util.ChangeDirectory(self.location): + appname, _, version = git.describe().partition('-') + self._app_version = app.ApplicationVersion.make(appname, version) + except shell.CallError: + pass + if not self._app_version: + try: + self._app_version = self.old_log[-1].version + except old_log.ScriptsVersionNoSuchFile: + pass + if not self._app_version: + appname = shell.eval("git", "config", "remote.origin.url").rpartition("/")[2].partition(".")[0] + self._app_version = app.ApplicationVersion.make(appname, "unknown") + return self._app_version + @property + def dsn(self): + """The :class:`sqlalchemy.engine.url.URL` for this deployment.""" + if not self._dsn: + self._dsn = sql.fill_url(self.application.dsn(self)) + return self._dsn + @property + def url(self): + """The :class:`urlparse.ParseResult` for this deployment.""" + if not self._url: + self._url = scripts.fill_url(self.location, self.application.url(self)) + if not self._url: + raise UnknownWebPath + return self._url @staticmethod def parse(line): - """Parses a line from the results of parallel-find.pl. - This will work out of the box with fileinput, see - getInstallLines()""" + """ + Parses a line from the :term:`versions store`. + + .. note:: + + Use this method only when speed is of the utmost + importance. You should prefer to directly create a deployment + with only a ``location`` when possible. + """ line = line.rstrip() try: location, deploydir = line.split(":") except ValueError: - return Deployment(line) # lazy loaded version - return Deployment(location, version=ApplicationVersion.parse(deploydir, location)) - -class Application(object): - """Represents the generic notion of an application, i.e. - mediawiki or phpbb.""" - def __init__(self, name): - self.name = name - self.versions = {} - self._extractors = {} - @property - def repository(self): - """Returns the Git repository that would contain this application.""" - repo = os.path.join("/afs/athena.mit.edu/contrib/scripts/git/autoinstalls", self.name + ".git") - if not os.path.isdir(repo): - raise NoRepositoryError(app) - return repo - def makeVersion(self, version): - if version not in self.versions: - self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self) - return self.versions[version] - def extract(self, deployment): - """Extracts wizard variables from a deployment.""" - result = {} - for k,extractor in self.extractors.items(): - result[k] = extractor(deployment) - return result - @property - def extractors(self): - return {} - @staticmethod - def make(name): - """Makes an application, but uses the correct subtype if available.""" - try: - __import__("wizard.app." + name) - return getattr(wizard.app, name).Application(name) - except ImportError: - return Application(name) - -class ApplicationVersion(object): - """Represents an abstract notion of a version for an application""" - def __init__(self, version, application): - """ `version` Instance of distutils.LooseVersion - `application` Instance of Application - WARNING: Please don't call this directly; instead, use getVersion() - on the application you want, so that this version gets registered.""" - self.version = version - self.application = application - @property - def scripts_tag(self): - """Returns the name of the Git tag for this version""" - # XXX: This assumes that there's only a -scripts version - # which will not be true in the future. Unfortunately, finding - # the "true" latest version is computationally expensive - return "v%s-scripts" % self.version - def __cmp__(x, y): - return cmp(x.version, y.version) - @staticmethod - def parse(deploydir,location,applookup=None): - # The version of the deployment, will be: - # /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z for old style installs - name = deploydir.split("/")[-1] + return ProductionCopy(line) # lazy loaded version try: - if name.find(" ") != -1: - raw_app, raw_version = name.split(" ") - version = raw_version[1:] # remove leading v - app, _ = raw_app.split(".") # remove trailing .git - elif name.find("-") != -1: - app, _, version = name.partition("-") - else: - app = name - version = "trunk" - except ValueError: # mostly from the a, b = foo.split(' ') - raise DeploymentParseError(deploydir, location) - if not applookup: applookup = applications() + return ProductionCopy(location, version=app.ApplicationVersion.parse(deploydir)) + except Error as e: + e.location = location + raise e + +class ProductionCopy(Deployment): + """ + Represents the production copy of a deployment. This copy + is canonical, and is the only one guaranteed to be accessible + via web, have a database, etc. + """ + @chdir_to_location + def upgrade(self, version, options): + """ + Performs an upgrade of database schemas and other non-versioned data. + """ + return self.application.upgrade(self, version, options) + @chdir_to_location + def backup(self, options): + """ + Performs a backup of database schemas and other non-versioned data. + """ + # There are retarded amounts of race-safety in this function, + # because we do NOT want to claim to have made a backup, when + # actually something weird happened to it. + backupdir = os.path.join(self.scripts_dir, "backups") + if not os.path.exists(backupdir): + try: + os.mkdir(backupdir) + except OSError as e: + if e.errno == errno.EEXIST: + pass + else: + raise + tmpdir = tempfile.mkdtemp() # actually will be kept around try: - # defer to the application for version creation - return applookup[app].makeVersion(version) - except KeyError: - raise NoSuchApplication(app, location) + self.application.backup(self, tmpdir, options) + except app.BackupFailure: + # the backup is bogus, don't let it show up + shutil.rmtree(tmpdir) + raise + backup = None + with util.LockDirectory(os.path.join(backupdir, "lock")): + while 1: + backup = str(self.version) + "-" + datetime.datetime.today().strftime("%Y-%m-%dT%H%M%S") + outdir = os.path.join(backupdir, backup) + if os.path.exists(outdir): + logging.warning("Backup: A backup occurred in the last second. Trying again in a second...") + time.sleep(1) + continue + try: + shutil.move(tmpdir, outdir) + except: + # don't leave half-baked stuff lying around + try: + shutil.rmtree(outdir) + except OSError: + pass + raise + break + return backup + @chdir_to_location + def restore(self, backup, options): + """ + Restores a backup. Destroys state, so be careful! Also, this does + NOT restore the file-level backup, which is what 'wizard restore' + does, so you probably do NOT want to call this elsewhere unless + you know what you're doing (call 'wizard restore' instead). + """ + backup_dir = os.path.join(".scripts", "backups", backup) + return self.application.restore(self, backup_dir, options) + @chdir_to_location + def remove(self, options): + """ + Deletes all non-local or non-filesystem data (such as databases) that + this application uses. + """ + self.application.remove(self, options) + def verifyWeb(self): + """ + Checks if the autoinstall is viewable from the web. + """ + if not self.application.checkWeb(self): + raise WebVerificationError + def fetch(self, path, post=None): + """ + Performs a HTTP request on the website. + """ + return util.fetch(self.url.netloc, self.url.path, path, post) # pylint: disable-msg=E1103 + +class WorkingCopy(Deployment): + """ + Represents a temporary clone of a deployment that we can make + modifications to without fear of interfering with a production + deployment. More operations are permitted on these copies. + """ + @chdir_to_location + def parametrize(self, deployment): + """ + Edits files in ``dir`` to replace WIZARD_* variables with literal + instances based on ``deployment``. This is used for constructing + virtual merge bases, and as such ``deployment`` will generally not + equal ``self``. + """ + return self.application.parametrize(self, deployment) + @chdir_to_location + def prepareConfig(self): + """ + Edits files in the deployment such that any user-specific configuration + is replaced with generic WIZARD_* variables. + """ + return self.application.prepareConfig(self) + @chdir_to_location + def resolveConflicts(self): + """ + Resolves conflicted files in this working copy. Returns whether or + not all conflicted files were resolved or not. Fully resolved + files are added to the index, but no commit is made. + """ + return self.application.resolveConflicts(self) + @chdir_to_location + def prepareMerge(self): + """ + Performs various edits to files in the current working directory in + order to make a merge go more smoothly. This is usually + used to fix botched line-endings. + """ + return self.application.prepareMerge(self) ## -- Exceptions -- -class Error(Exception): +class Error(wizard.Error): """Base error class for this module""" pass -class NoSuchApplication(Error): - def __init__(self, name, location): - self.name = name - self.location = location +class NotMigratedError(Error): + """ + The deployment contains a .scripts-version file, but no .git + or .scripts directory. + """ + #: Directory of deployment + dir = None + def __init__(self, dir): + self.dir = dir + def __str__(self): + return """This installation was not migrated""" + +class AlreadyVersionedError(Error): + """The deployment contained a .git directory but no .scripts directory.""" + #: Directory of deployment + dir = None + def __init__(self, dir): + self.dir = dir + def __str__(self): + return """ + +ERROR: Directory contains a .git directory, but not +a .scripts directory. If this is not a corrupt +migration, this means that the user was versioning their +install using Git.""" + +class NotConfiguredError(Error): + """The install was missing essential configuration.""" + #: Directory of unconfigured install + dir = None + def __init__(self, dir): + self.dir = dir def __str__(self): - return "ERROR: Unrecognized app '%s' at %s" % (self.name, self.location) + return """ + +ERROR: The install was well-formed, but not configured +(essential configuration files were not found.)""" + +class CorruptedAutoinstallError(Error): + """The install was missing a .git directory, but had a .scripts directory.""" + #: Directory of the corrupted install + dir = None + def __init__(self, dir): + self.dir = dir + def __str__(self): + return """ + +ERROR: Directory contains a .scripts directory, +but not a .git directory.""" + +class NotAutoinstallError(Error): + """The directory was not an autoinstall, due to missing .scripts-version file.""" + #: Directory in question + dir = None + def __init__(self, dir): + self.dir = dir + def __str__(self): + return """ + +ERROR: Could not find .scripts-version file. Are you sure +this is an autoinstalled application? +""" + +class NoTagError(Error): + """Deployment has a tag that does not have an equivalent in upstream repository.""" + #: Missing tag + tag = None + def __init__(self, tag): + self.tag = tag + def __str__(self): + return """ + +ERROR: Could not find tag %s in repository.""" % self.tag + +class NoLocalTagError(Error): + """Could not find tag in local repository.""" + #: Missing tag + tag = None + def __init__(self, tag): + self.tag = tag + def __str__(self): + return """ + +ERROR: Could not find tag %s in local repository.""" % self.tag + +class InconsistentPristineTagError(Error): + """Pristine tag commit ID does not match upstream pristine tag commit ID.""" + #: Inconsistent tag + tag = None + def __init__(self, tag): + self.tag = tag + def __str__(self): + return """ + +ERROR: Local pristine tag %s did not match repository's. This +probably means an upstream rebase occured.""" % self.tag + +class InconsistentScriptsTagError(Error): + """Scripts tag commit ID does not match upstream scripts tag commit ID.""" + #: Inconsistent tag + tag = None + def __init__(self, tag): + self.tag = tag + def __str__(self): + return """ + +ERROR: Local scripts tag %s did not match repository's. This +probably means an upstream rebase occurred.""" % self.tag + +class HeadNotDescendantError(Error): + """HEAD is not connected to tag.""" + #: Tag that HEAD should have been descendant of. + tag = None + def __init__(self, tag): + self.tag = tag + def __str__(self): + return """ + +ERROR: HEAD is not a descendant of %s. This probably +means that an upstream rebase occurred, and new tags were +pulled, but local user commits were never rebased.""" % self.tag -class DeploymentParseError(Error): - def __init__(self, malformed, location): - self.malformed = malformed - self.location = location +class VersionDetectionError(Error): + """Could not detect real version of application.""" def __str__(self): - return """ERROR: Unparseable '%s' at %s""" % (self.malformed, self.location) + return """ + +ERROR: Could not detect the real version of the application.""" + +class VersionMismatchError(Error): + """Git version of application does not match detected version.""" + #: Detected version + real_version = None + #: Version from Git + git_version = None + def __init__(self, real_version, git_version): + self.real_version = real_version + self.git_version = git_version + def __str__(self): + return """ + +ERROR: The detected version %s did not match the Git +version %s.""" % (self.real_version, self.git_version) + +class WebVerificationError(Error): + """Could not access the application on the web""" + def __str__(self): + return """ + +ERROR: We were not able to access the application on the +web. This may indicate that the website is behind +authentication on the htaccess level. You can find +the contents of the page from the debug backtraces.""" -class NoRepositoryError(Error): - def __init__(self, app): - self.app = app - self.location = "unknown" +class UnknownWebPath(Error): + """Could not determine application's web path.""" def __str__(self): return """ -ERROR: Could not find repository for this application. Have -you converted the repository over? Is the name %s -the same as the name of the .git folder? -""" % self.app - -# If you want, you can wrap this up into a registry and access things -# through that, but it's not really necessary - -application_list = [ - "mediawiki", "wordpress", "joomla", "e107", "gallery2", - "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django", - # these are technically deprecated - "advancedpoll", "gallery", -] -_applications = None - -def applications(): - """Hash table for looking up string application name to instance""" - global _applications - if not _applications: - _applications = dict([(n,Application.make(n)) for n in application_list ]) - return _applications +ERROR: We were not able to determine what the application's +host and path were in order to perform a web request +on the application. You can specify this manually using +the WIZARD_WEB_HOST and WIZARD_WEB_PATH environment +variables."""