"""
Object model for querying information and manipulating deployments
-of autoinstalls. Every :class:`Deployment` has an :class:`ApplicationVersion`
-which in turn has an :class:`Application`.
+of autoinstalls. Every :class:`Deployment` has an :class:`app.ApplicationVersion`
+which in turn has an :class:`app.Application`.
"""
import os.path
import fileinput
-import dateutil.parser
-import distutils.version
-import tempfile
import logging
+import decorator
+import datetime
+import tempfile
+import time
+import traceback
import shutil
+import errno
import wizard
-from wizard import git, old_log, scripts, shell, util
+from wizard import app, git, old_log, scripts, shell, sql, util
## -- Global Functions --
log output.
"""
if not show:
- show = applications()
+ show = app.applications()
elif isinstance(show, str):
# otherwise, frozenset will treat string as an iterable
show = frozenset([show])
try:
d = Deployment.parse(line)
name = d.application.name
- except NoSuchApplication as e:
+ except app.NoSuchApplication as e:
if yield_errors:
yield e
continue
## -- Model Objects --
+@decorator.decorator
+def chdir_to_location(f, self, *args, **kwargs):
+ """
+ Decorator for making a function have working directory
+ :attr:`Deployment.location`.
+ """
+ with util.ChangeDirectory(self.location):
+ return f(self, *args, **kwargs)
+
class Deployment(object):
"""
- Represents a deployment of an autoinstall, e.g. directory in a user's
- web_scripts that has ``.scripts`` directory or ``.scripts-version``
+ Represents a deployment of an autoinstall, e.g. directory
+ that has ``.scripts`` directory or ``.scripts-version``
file in it. Supply ``version`` with an :class:`ApplicationVersion` only if
you were reading from the :term:`versions store` and care about
speed (data from there can be stale).
+
+ The Deployment interface is somewhat neutered, so you may
+ want to use :class:`WorkingCopy` or :class:`ProductionCopy` for
+ more powerful operations.
"""
#: Absolute path to the deployment
location = None
# some cache variables
self._read_cache = {}
self._old_log = None
+ self._dsn = None
+ self._url = None
+ def invalidateCache(self):
+ """
+ Invalidates all cached variables. This currently applies to
+ :attr:`app_version`, :attr:`old_log` and :meth:`read`.
+ """
+ self._app_version = None
+ self._read_cache = {}
+ self._old_log = None
def read(self, file, force = False):
"""
Reads a file's contents, possibly from cache unless ``force``
and local configuration.
"""
return self.application.extract(self)
- def parametrize(self, dir):
- """
- Edits files in ``dir`` to replace WIZARD_* variables with literal
- instances. This is used for constructing virtual merge bases, and
- as such dir will generally not equal :attr:`location`.
- """
- return self.application.parametrize(self, dir)
- def upgrade(self, version, options):
- """
- Performs an upgrae of database schemas and other non-versioned data.
- """
- with util.ChangeDirectory(self.location):
- return self.application.upgrade(self, version, options)
- def backup(self, options):
- """
- Performs a backup of database schemas and other non-versioned data.
- """
- with util.ChangeDirectory(self.location):
- return self.application.backup(self, options)
- def restore(self, backup, options):
- """
- Restores a backup. Destroys state, so be careful! Also, this does
- NOT restore the file-level backup, which is what 'wizard restore'
- does, so you probably do NOT want to call this elsewhere unless
- you know what you're doing.
- """
- with util.ChangeDirectory(self.location):
- return self.application.restore(self, backup, options)
- def prepareConfig(self):
- """
- Edits files in the deployment such that any user-specific configuration
- is replaced with generic WIZARD_* variables.
- """
- return self.application.prepareConfig(self)
- def checkConfig(self, deployment):
- """
- Checks if the application is configured.
- """
- raise NotImplemented
def verify(self):
"""
"""
repo = self.application.repository(srv_path)
try:
- shell.Shell().eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--')
+ shell.eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--')
except shell.CallError:
raise NoTagError(self.app_version.scripts_tag)
corresponds to the one in the remote repository.
"""
with util.ChangeDirectory(self.location):
- sh = shell.Shell()
repo = self.application.repository(srv_path)
def repo_rev_parse(tag):
- return sh.eval("git", "--git-dir", repo, "rev-parse", tag)
+ return shell.eval("git", "--git-dir", repo, "rev-parse", tag)
def self_rev_parse(tag):
try:
- return sh.safeCall("git", "rev-parse", tag, strip=True)
+ return shell.safeCall("git", "rev-parse", tag, strip=True)
except shell.CallError:
raise NoLocalTagError(tag)
def compare_tags(tag):
if not compare_tags(self.app_version.scripts_tag):
raise InconsistentScriptsTagError(self.app_version.scripts_tag)
parent = repo_rev_parse(self.app_version.scripts_tag)
- merge_base = sh.safeCall("git", "merge-base", parent, "HEAD", strip=True)
+ merge_base = shell.safeCall("git", "merge-base", parent, "HEAD", strip=True)
if merge_base != parent:
raise HeadNotDescendantError(self.app_version.scripts_tag)
if not self.configured:
raise NotConfiguredError(self.location)
+ @chdir_to_location
def verifyVersion(self):
"""
Checks if our version and the version number recorded in a file
elif not str(real) == self.app_version.pristine_tag.partition('-')[2]:
raise VersionMismatchError(real, self.version)
- def verifyWeb(self):
- """
- Checks if the autoinstall is viewable from the web.
- """
- out = []
- if not self.application.checkWeb(self, out):
- raise WebVerificationError(out[0])
-
- def fetch(self, path, post=None):
- """
- Performs a HTTP request on the website.
- """
- try:
- host, basepath = scripts.get_web_host_and_path(self.location)
- except ValueError:
- raise UnknownWebPath
- return util.fetch(host, basepath, path, post)
-
@property
+ @chdir_to_location
def configured(self):
"""Whether or not an autoinstall has been configured/installed for use."""
return self.application.checkConfig(self)
"""The absolute path of the ``.scripts/version`` file."""
return os.path.join(self.scripts_dir, 'version')
@property
+ def dsn_file(self):
+ """The absolute path of the :file:`.scripts/dsn` override file."""
+ return os.path.join(self.scripts_dir, 'dsn')
+ @property
+ def url_file(self):
+ """The absolute path of the :file:`.scripts/url` override file."""
+ return os.path.join(self.scripts_dir, 'url')
+ @property
def application(self):
- """The :class:`Application` of this deployment."""
+ """The :class:`app.Application` of this deployment."""
return self.app_version.application
@property
def old_log(self):
return self.app_version.version
@property
def app_version(self):
- """The :class:`ApplicationVersion` of this deployment."""
+ """The :class:`app.ApplicationVersion` of this deployment."""
if not self._app_version:
if os.path.isdir(os.path.join(self.location, ".git")):
try:
with util.ChangeDirectory(self.location):
appname, _, version = git.describe().partition('-')
- self._app_version = ApplicationVersion.make(appname, version)
+ self._app_version = app.ApplicationVersion.make(appname, version)
except shell.CallError:
pass
if not self._app_version:
- self._app_version = self.old_log[-1].version
+ try:
+ self._app_version = self.old_log[-1].version
+ except old_log.ScriptsVersionNoSuchFile:
+ pass
+ if not self._app_version:
+ appname = shell.eval("git", "config", "remote.origin.url").rpartition("/")[2].partition(".")[0]
+ self._app_version = app.ApplicationVersion.make(appname, "unknown")
return self._app_version
+ @property
+ def dsn(self):
+ """The :class:`sqlalchemy.engine.url.URL` for this deployment."""
+ if not self._dsn:
+ self._dsn = sql.fill_url(self.application.dsn(self))
+ return self._dsn
+ @property
+ def url(self):
+ """The :class:`urlparse.ParseResult` for this deployment."""
+ if not self._url:
+ self._url = scripts.fill_url(self.location, self.application.url(self))
+ if not self._url:
+ raise UnknownWebPath
+ return self._url
@staticmethod
def parse(line):
"""
try:
location, deploydir = line.split(":")
except ValueError:
- return Deployment(line) # lazy loaded version
+ return ProductionCopy(line) # lazy loaded version
try:
- return Deployment(location, version=ApplicationVersion.parse(deploydir))
+ return ProductionCopy(location, version=app.ApplicationVersion.parse(deploydir))
except Error as e:
e.location = location
raise e
-class Application(object):
- """Represents an application, i.e. mediawiki or phpbb."""
- #: String name of the application
- name = None
- #: Dictionary of version strings to :class:`ApplicationVersion`.
- #: See also :meth:`makeVersion`.
- versions = None
- #: List of files that need to be modified when parametrizing.
- #: This is a class-wide constant, and should not normally be modified.
- parametrized_files = []
- def __init__(self, name):
- self.name = name
- self.versions = {}
- # cache variables
- self._extractors = {}
- self._substitutions = {}
- def repository(self, srv_path):
- """
- Returns the Git repository that would contain this application.
- ``srv_path`` corresponds to ``options.srv_path`` from the global baton.
- """
- repo = os.path.join(srv_path, self.name + ".git")
- if not os.path.isdir(repo):
- repo = os.path.join(srv_path, self.name, ".git")
- if not os.path.isdir(repo):
- raise NoRepositoryError(self.name)
- return repo
- def makeVersion(self, version):
- """
- Creates or retrieves the :class:`ApplicationVersion` singleton for the
- specified version.
- """
- if version not in self.versions:
- self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
- return self.versions[version]
- def extract(self, deployment):
- """Extracts wizard variables from a deployment."""
- result = {}
- for k,extractor in self.extractors.items():
- result[k] = extractor(deployment)
- return result
- def parametrize(self, deployment, dir):
- """
- Takes a generic source checkout at dir and parametrizes
- it according to the values of deployment.
- """
- variables = deployment.extract()
- for file in self.parametrized_files:
- fullpath = os.path.join(dir, file)
- try:
- contents = open(fullpath, "r").read()
- except IOError:
- continue
- for key, value in variables.items():
- if value is None: continue
- contents = contents.replace(key, value)
- f = open(fullpath, "w")
- f.write(contents)
- def resolveConflicts(self, dir):
- """
- Takes a directory with conflicted files and attempts to
- resolve them. Returns whether or not all conflicted
- files were resolved or not. Fully resolved files are
- added to the index, but no commit is made.
- """
- return False
- def prepareConfig(self, deployment):
- """
- Takes a deployment and replaces any explicit instances
- of a configuration variable with generic WIZARD_* constants.
- There is a sane default implementation built on substitutions;
- you can override this method to provide arbitrary extra
- behavior.
- """
- for key, subst in self.substitutions.items():
- subs = subst(deployment)
- if not subs and key not in self.deprecated_keys:
- logging.warning("No substitutions for %s" % key)
- def install(self, version, options):
- """
- Run for 'wizard configure' (and, by proxy, 'wizard install')
- to configure an application. This assumes that the current
- working directory is a deployment.
- """
- raise NotImplemented
- def upgrade(self, deployment, version, options):
- """
- Run for 'wizard upgrade' to upgrade database schemas and other
- non-versioned data in an application. This assumes that
- the current working directory is the deployment.
- """
- raise NotImplemented
- def backup(self, deployment, options):
- """
- Run for 'wizard backup' and upgrades to backup database schemas
- and other non-versioned data in an application. This assumes
- that the current working directory is the deployment.
- """
- raise NotImplemented
- def restore(self, deployment, backup, options):
+class ProductionCopy(Deployment):
+ """
+ Represents the production copy of a deployment. This copy
+ is canonical, and is the only one guaranteed to be accessible
+ via web, have a database, etc.
+ """
+ @chdir_to_location
+ def upgrade(self, version, options):
"""
- Run for 'wizard restore' and failed upgrades to restore database
- and other non-versioned data to a backed up version. This assumes
- that the current working directory is the deployment.
+ Performs an upgrade of database schemas and other non-versioned data.
"""
- raise NotImplemented
- def detectVersion(self, deployment):
+ return self.application.upgrade(self, version, options)
+ @chdir_to_location
+ def backup(self, options):
"""
- Checks source files to determine the version manually.
+ Performs a backup of database schemas and other non-versioned data.
"""
- return None
- def checkWeb(self, deployment, output=None):
+ # There are retarded amounts of race-safety in this function,
+ # because we do NOT want to claim to have made a backup, when
+ # actually something weird happened to it.
+ backupdir = os.path.join(self.scripts_dir, "backups")
+ if not os.path.exists(backupdir):
+ try:
+ os.mkdir(backupdir)
+ except OSError as e:
+ if e.errno == errno.EEXIST:
+ pass
+ else:
+ raise
+ tmpdir = tempfile.mkdtemp() # actually will be kept around
+ try:
+ self.application.backup(self, tmpdir, options)
+ except app.BackupFailure:
+ # the backup is bogus, don't let it show up
+ shutil.rmtree(tmpdir)
+ raise
+ backup = None
+ with util.LockDirectory(os.path.join(backupdir, "lock")):
+ while 1:
+ backup = str(self.version) + "-" + datetime.datetime.today().strftime("%Y-%m-%dT%H%M%S")
+ outdir = os.path.join(backupdir, backup)
+ if os.path.exists(outdir):
+ logging.warning("Backup: A backup occurred in the last second. Trying again in a second...")
+ time.sleep(1)
+ continue
+ try:
+ shutil.move(tmpdir, outdir)
+ except:
+ # don't leave half-baked stuff lying around
+ try:
+ shutil.rmtree(outdir)
+ except OSError:
+ pass
+ raise
+ break
+ return backup
+ @chdir_to_location
+ def restore(self, backup, options):
"""
- Checks if the autoinstall is viewable from the web. Output
- should be an empty list that will get mutated by this function.
+ Restores a backup. Destroys state, so be careful! Also, this does
+ NOT restore the file-level backup, which is what 'wizard restore'
+ does, so you probably do NOT want to call this elsewhere unless
+ you know what you're doing (call 'wizard restore' instead).
"""
- raise NotImplemented
- @property
- def extractors(self):
+ backup_dir = os.path.join(".scripts", "backups", backup)
+ return self.application.restore(self, backup_dir, options)
+ @chdir_to_location
+ def remove(self, options):
"""
- Dictionary of variable names to extractor functions. These functions
- take a :class:`Deployment` as an argument and return the value of
- the variable, or ``None`` if it could not be found.
- See also :func:`wizard.app.filename_regex_extractor`.
+ Deletes all non-local or non-filesystem data (such as databases) that
+ this application uses.
"""
- return {}
- @property
- def substitutions(self):
+ self.application.remove(self, options)
+ def verifyWeb(self):
"""
- Dictionary of variable names to substitution functions. These functions
- take a :class:`Deployment` as an argument and modify the deployment such
- that an explicit instance of the variable is released with the generic
- WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`.
+ Checks if the autoinstall is viewable from the web.
"""
- return {}
- @staticmethod
- def make(name):
- """Makes an application, but uses the correct subtype if available."""
- try:
- __import__("wizard.app." + name)
- return getattr(wizard.app, name).Application(name)
- except ImportError:
- return Application(name)
-
-class ApplicationVersion(object):
- """Represents an abstract notion of a version for an application, where
- ``version`` is a :class:`distutils.version.LooseVersion` and
- ``application`` is a :class:`Application`."""
- #: The :class:`distutils.version.LooseVersion` of this instance.
- version = None
- #: The :class:`Application` of this instance.
- application = None
- def __init__(self, version, application):
- self.version = version
- self.application = application
- @property
- def tag(self):
+ if not self.application.checkWeb(self):
+ raise WebVerificationError
+ def fetch(self, path, post=None):
"""
- Returns the name of the git describe tag for the commit the user is
- presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
+ Performs a HTTP request on the website.
"""
- return "%s-%s" % (self.application, self.version)
- @property
- def scripts_tag(self):
+ return util.fetch(self.url.netloc, self.url.path, path, post) # pylint: disable-msg=E1103
+
+class WorkingCopy(Deployment):
+ """
+ Represents a temporary clone of a deployment that we can make
+ modifications to without fear of interfering with a production
+ deployment. More operations are permitted on these copies.
+ """
+ @chdir_to_location
+ def parametrize(self, deployment):
"""
- Returns the name of the Git tag for this version.
+ Edits files in ``dir`` to replace WIZARD_* variables with literal
+ instances based on ``deployment``. This is used for constructing
+ virtual merge bases, and as such ``deployment`` will generally not
+ equal ``self``.
"""
- end = str(self.version).partition('-scripts')[2].partition('-')[0]
- return "%s-scripts%s" % (self.pristine_tag, end)
- @property
- def pristine_tag(self):
+ return self.application.parametrize(self, deployment)
+ @chdir_to_location
+ def prepareConfig(self):
"""
- Returns the name of the Git tag for the pristine version corresponding
- to this version.
+ Edits files in the deployment such that any user-specific configuration
+ is replaced with generic WIZARD_* variables.
"""
- return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
- def __cmp__(x, y):
- return cmp(x.version, y.version)
- @staticmethod
- def parse(value):
+ return self.application.prepareConfig(self)
+ @chdir_to_location
+ def resolveConflicts(self):
"""
- Parses a line from the :term:`versions store` and return
- :class:`ApplicationVersion`.
-
- Use this only for cases when speed is of primary importance;
- the data in version is unreliable and when possible, you should
- prefer directly instantiating a Deployment and having it query
- the autoinstall itself for information.
-
- The `value` to parse will vary. For old style installs, it
- will look like::
-
- /afs/athena.mit.edu/contrib/scripts/deploy/APP-x.y.z
-
- For new style installs, it will look like::
-
- APP-x.y.z-scripts
+ Resolves conflicted files in this working copy. Returns whether or
+ not all conflicted files were resolved or not. Fully resolved
+ files are added to the index, but no commit is made.
"""
- name = value.split("/")[-1]
- try:
- if name.find("-") != -1:
- app, _, version = name.partition("-")
- else:
- # kind of poor, maybe should error. Generally this
- # will actually result in a not found error
- app = name
- version = "trunk"
- except ValueError:
- raise DeploymentParseError(deploydir)
- return ApplicationVersion.make(app, version)
- @staticmethod
- def make(app, version):
+ return self.application.resolveConflicts(self)
+ @chdir_to_location
+ def prepareMerge(self):
"""
- Makes/retrieves a singleton :class:`ApplicationVersion` from
- a``app`` and ``version`` string.
+ Performs various edits to files in the current working directory in
+ order to make a merge go more smoothly. This is usually
+ used to fix botched line-endings.
"""
- try:
- # defer to the application for version creation to enforce
- # singletons
- return applications()[app].makeVersion(version)
- except KeyError:
- raise NoSuchApplication(app)
+ return self.application.prepareMerge(self)
## -- Exceptions --
"""Base error class for this module"""
pass
-class NoSuchApplication(Error):
- """
- You attempted to reference a :class:`Application` named
- ``app``, which is not recognized by Wizard.
- """
- #: The name of the application that does not exist.
- app = None
- #: The location of the autoinstall that threw this variable.
- #: This should be set by error handling code when it is availble.
- location = None
- def __init__(self, app):
- self.app = app
-
-class DeploymentParseError(Error):
- """
- Could not parse ``value`` from :term:`versions store`.
- """
- #: The value that failed to parse.
- value = None
- #: The location of the autoinstall that threw this variable.
- #: This should be set by error handling code when it is available.
- location = None
- def __init__(self, value):
- self.value = value
-
-class NoRepositoryError(Error):
- """
- :class:`Application` does not appear to have a Git repository
- in the normal location.
- """
- #: The name of the application that does not have a Git repository.
- app = None
- def __init__(self, app):
- self.app = app
- def __str__(self):
- return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
-
class NotMigratedError(Error):
"""
The deployment contains a .scripts-version file, but no .git
class WebVerificationError(Error):
"""Could not access the application on the web"""
- #: Contents of web page access
- contents = None
- def __init__(self, contents):
- self.contents = contents
def __str__(self):
return """
ERROR: We were not able to access the application on the
web. This may indicate that the website is behind
-authentication on the htaccess level. The contents
-of the page were:
-
-%s""" % self.contents
+authentication on the htaccess level. You can find
+the contents of the page from the debug backtraces."""
class UnknownWebPath(Error):
"""Could not determine application's web path."""
the WIZARD_WEB_HOST and WIZARD_WEB_PATH environment
variables."""
-_application_list = [
- "mediawiki", "wordpress", "joomla", "e107", "gallery2",
- "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
- # these are technically deprecated
- "advancedpoll", "gallery",
-]
-_applications = None
-
-def applications():
- """Hash table for looking up string application name to instance"""
- global _applications
- if not _applications:
- _applications = dict([(n,Application.make(n)) for n in _application_list ])
- return _applications
-
scripts.mit.edu / wizard.git / blobdiff