import dateutil.parser
import distutils.version
import tempfile
+import logging
+import shutil
import wizard
-from wizard import git, log, util
+from wizard import git, old_log, scripts, shell, util
## -- Global Functions --
-def get_install_lines(versions_store):
+def get_install_lines(versions_store, user=None):
"""
Low level function that retrieves a list of lines from the
:term:`versions store` that can be passed to :meth:`Deployment.parse`.
"""
if os.path.isfile(versions_store):
return fileinput.input([versions_store])
- return fileinput.input([versions_store + "/" + f for f in os.listdir(versions_store)])
+ if user:
+ return fileinput.input([versions_store + "/" + user])
+ return fileinput.input([versions_store + "/" + f for f in sorted(os.listdir(versions_store))])
-def parse_install_lines(show, versions_store, yield_errors = False):
+def parse_install_lines(show, versions_store, yield_errors = False, user = None):
"""
Generator function for iterating through all autoinstalls.
Each item is an instance of :class:`Deployment`, or possibly
or ``app-1.2.3`` in ``show``. This function may generate
log output.
"""
- if not show: show = applications()
- show = frozenset(show)
- for line in get_install_lines(versions_store):
+ if not show:
+ show = applications()
+ elif isinstance(show, str):
+ # otherwise, frozenset will treat string as an iterable
+ show = frozenset([show])
+ else:
+ show = frozenset(show)
+ for line in get_install_lines(versions_store, user):
# construction
try:
d = Deployment.parse(line)
#: Absolute path to the deployment
location = None
def __init__(self, location, version=None):
- self.location = os.path.realpath(location)
+ self.location = os.path.abspath(location)
self._app_version = version
# some cache variables
self._read_cache = {}
- self._log = None
+ self._old_log = None
def read(self, file, force = False):
"""
Reads a file's contents, possibly from cache unless ``force``
as such dir will generally not equal :attr:`location`.
"""
return self.application.parametrize(self, dir)
- def updateVersion(self, version):
+ def upgrade(self, version, options):
"""
- Update the version of this deployment.
+ Performs an upgrae of database schemas and other non-versioned data.
+ """
+ with util.ChangeDirectory(self.location):
+ return self.application.upgrade(self, version, options)
+ def backup(self, options):
+ """
+ Performs a backup of database schemas and other non-versioned data.
+ """
+ with util.ChangeDirectory(self.location):
+ return self.application.backup(self, options)
+ def restore(self, backup, options):
+ """
+ Restores a backup. Destroys state, so be careful! Also, this does
+ NOT restore the file-level backup, which is what 'wizard restore'
+ does, so you probably do NOT want to call this elsewhere unless
+ you know what you're doing.
+ """
+ with util.ChangeDirectory(self.location):
+ return self.application.restore(self, backup, options)
+ def prepareConfig(self):
+ """
+ Edits files in the deployment such that any user-specific configuration
+ is replaced with generic WIZARD_* variables.
+ """
+ return self.application.prepareConfig(self)
+ def checkConfig(self, deployment):
+ """
+ Checks if the application is configured.
+ """
+ raise NotImplemented
- This method will update the version of this deployment in memory
- and on disk. It doesn't actually do an upgrade. The version
- string you pass here should have ``-scripts`` as a suffix.
+ def verify(self):
"""
- self._app_version = self.application.makeVersion(version)
- f = open(os.path.join(self.scripts_dir, 'version'), 'w')
- f.write(self.application.name + '-' + version + "\n")
- f.close()
- def scriptsifyVersion(self):
+ Checks if this is an autoinstall, throws an exception if there
+ are problems.
"""
- Converts from ``v1.0`` to ``v1.0-scripts``; use at end of migration.
+ with util.ChangeDirectory(self.location):
+ has_git = os.path.isdir(".git")
+ has_scripts = os.path.isdir(".scripts")
+ if not has_git and has_scripts:
+ raise CorruptedAutoinstallError(self.location)
+ elif has_git and not has_scripts:
+ raise AlreadyVersionedError(self.location)
+ elif not has_git and not has_scripts:
+ if os.path.isfile(".scripts-version"):
+ raise NotMigratedError(self.location)
- .. note::
+ def verifyTag(self, srv_path):
+ """
+ Checks if the purported version has a corresponding tag
+ in the upstream repository.
+ """
+ repo = self.application.repository(srv_path)
+ try:
+ shell.Shell().eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--')
+ except shell.CallError:
+ raise NoTagError(self.app_version.scripts_tag)
- This makes the assumption that a migration will be to
- a ``-scripts`` tag and not a ``-scripts2`` tag. If you botch
- migration, blow away the tag and try again.
+ def verifyGit(self, srv_path):
+ """
+ Checks if the autoinstall's Git repository makes sense,
+ checking if the tag is parseable and corresponds to
+ a real application, and if the tag in this repository
+ corresponds to the one in the remote repository.
+ """
+ with util.ChangeDirectory(self.location):
+ sh = shell.Shell()
+ repo = self.application.repository(srv_path)
+ def repo_rev_parse(tag):
+ return sh.eval("git", "--git-dir", repo, "rev-parse", tag)
+ def self_rev_parse(tag):
+ try:
+ return sh.safeCall("git", "rev-parse", tag, strip=True)
+ except shell.CallError:
+ raise NoLocalTagError(tag)
+ def compare_tags(tag):
+ return repo_rev_parse(tag) == self_rev_parse(tag)
+ if not compare_tags(self.app_version.pristine_tag):
+ raise InconsistentPristineTagError(self.app_version.pristine_tag)
+ if not compare_tags(self.app_version.scripts_tag):
+ raise InconsistentScriptsTagError(self.app_version.scripts_tag)
+ parent = repo_rev_parse(self.app_version.scripts_tag)
+ merge_base = sh.safeCall("git", "merge-base", parent, "HEAD", strip=True)
+ if merge_base != parent:
+ raise HeadNotDescendantError(self.app_version.scripts_tag)
+
+ def verifyConfigured(self):
+ """
+ Checks if the autoinstall is configured running.
+ """
+ if not self.configured:
+ raise NotConfiguredError(self.location)
+
+ def verifyVersion(self):
+ """
+ Checks if our version and the version number recorded in a file
+ are consistent.
+ """
+ real = self.application.detectVersion(self)
+ if not real:
+ raise VersionDetectionError
+ elif not str(real) == self.app_version.pristine_tag.partition('-')[2]:
+ raise VersionMismatchError(real, self.version)
+
+ def verifyWeb(self):
+ """
+ Checks if the autoinstall is viewable from the web.
+ """
+ out = []
+ if not self.application.checkWeb(self, out):
+ raise WebVerificationError(out[0])
+
+ def fetch(self, path, post=None):
+ """
+ Performs a HTTP request on the website.
"""
- self.updateVersion(self.app_version.scripts_tag)
+ try:
+ host, basepath = scripts.get_web_host_and_path(self.location)
+ except ValueError:
+ raise UnknownWebPath
+ return util.fetch(host, basepath, path, post)
+
+ @property
+ def configured(self):
+ """Whether or not an autoinstall has been configured/installed for use."""
+ return self.application.checkConfig(self)
@property
def migrated(self):
"""Whether or not the autoinstalls has been migrated."""
Use of this is discouraged for migrated installs.
"""
- if self.migrated:
- return os.path.join(self.scripts_dir, 'old-version')
- else:
- return os.path.join(self.location, '.scripts-version')
+ return os.path.join(self.location, '.scripts-version')
@property
def version_file(self):
"""The absolute path of the ``.scripts/version`` file."""
"""The :class:`Application` of this deployment."""
return self.app_version.application
@property
- def log(self):
- """The :class:`wizard.deploy.Log` of this deployment."""
- if not self._log:
- self._log = log.DeployLog.load(self)
- return self._log
+ def old_log(self):
+ """
+ The :class:`wizard.old_log.Log` of this deployment. This
+ is only applicable to un-migrated autoinstalls.
+ """
+ if not self._old_log:
+ self._old_log = old_log.DeployLog.load(self)
+ return self._old_log
@property
def version(self):
"""
"""The :class:`ApplicationVersion` of this deployment."""
if not self._app_version:
if os.path.isdir(os.path.join(self.location, ".git")):
- with util.ChangeDirectory(self.location):
- appname, _, version = git.describe().partition('-')
- self._app_version = ApplicationVersion.make(appname, version)
- else:
- self._app_version = self.log[-1].version
+ try:
+ with util.ChangeDirectory(self.location):
+ appname, _, version = git.describe().partition('-')
+ self._app_version = ApplicationVersion.make(appname, version)
+ except shell.CallError:
+ pass
+ if not self._app_version:
+ self._app_version = self.old_log[-1].version
return self._app_version
@staticmethod
def parse(line):
self.versions = {}
# cache variables
self._extractors = {}
- self._parametrizers = {}
+ self._substitutions = {}
def repository(self, srv_path):
"""
Returns the Git repository that would contain this application.
``srv_path`` corresponds to ``options.srv_path`` from the global baton.
- Throws :exc:`NoRepositoryError` if the calculated path does not
- exist.
"""
repo = os.path.join(srv_path, self.name + ".git")
if not os.path.isdir(repo):
variables = deployment.extract()
for file in self.parametrized_files:
fullpath = os.path.join(dir, file)
- f = open(fullpath, "r")
- contents = f.read()
- f.close()
+ try:
+ contents = open(fullpath, "r").read()
+ except IOError:
+ continue
for key, value in variables.items():
if value is None: continue
contents = contents.replace(key, value)
- tmp = tempfile.NamedTemporaryFile(delete=False)
- tmp.write(contents)
- tmp.close()
- os.rename(tmp.name, fullpath)
+ f = open(fullpath, "w")
+ f.write(contents)
+ def resolveConflicts(self, dir):
+ """
+ Takes a directory with conflicted files and attempts to
+ resolve them. Returns whether or not all conflicted
+ files were resolved or not. Fully resolved files are
+ added to the index, but no commit is made.
+ """
+ return False
+ def prepareConfig(self, deployment):
+ """
+ Takes a deployment and replaces any explicit instances
+ of a configuration variable with generic WIZARD_* constants.
+ There is a sane default implementation built on substitutions;
+ you can override this method to provide arbitrary extra
+ behavior.
+ """
+ for key, subst in self.substitutions.items():
+ subs = subst(deployment)
+ if not subs and key not in self.deprecated_keys:
+ logging.warning("No substitutions for %s" % key)
+ def install(self, version, options):
+ """
+ Run for 'wizard configure' (and, by proxy, 'wizard install')
+ to configure an application. This assumes that the current
+ working directory is a deployment.
+ """
+ raise NotImplemented
+ def upgrade(self, deployment, version, options):
+ """
+ Run for 'wizard upgrade' to upgrade database schemas and other
+ non-versioned data in an application. This assumes that
+ the current working directory is the deployment.
+ """
+ raise NotImplemented
+ def backup(self, deployment, options):
+ """
+ Run for 'wizard backup' and upgrades to backup database schemas
+ and other non-versioned data in an application. This assumes
+ that the current working directory is the deployment.
+ """
+ raise NotImplemented
+ def restore(self, deployment, backup, options):
+ """
+ Run for 'wizard restore' and failed upgrades to restore database
+ and other non-versioned data to a backed up version. This assumes
+ that the current working directory is the deployment.
+ """
+ raise NotImplemented
+ def detectVersion(self, deployment):
+ """
+ Checks source files to determine the version manually.
+ """
+ return None
+ def checkWeb(self, deployment, output=None):
+ """
+ Checks if the autoinstall is viewable from the web. Output
+ should be an empty list that will get mutated by this function.
+ """
+ raise NotImplemented
@property
def extractors(self):
"""
See also :func:`wizard.app.filename_regex_extractor`.
"""
return {}
+ @property
+ def substitutions(self):
+ """
+ Dictionary of variable names to substitution functions. These functions
+ take a :class:`Deployment` as an argument and modify the deployment such
+ that an explicit instance of the variable is released with the generic
+ WIZARD_* constant. See also :func:`wizard.app.filename_regex_substitution`.
+ """
+ return {}
@staticmethod
def make(name):
"""Makes an application, but uses the correct subtype if available."""
self.version = version
self.application = application
@property
+ def tag(self):
+ """
+ Returns the name of the git describe tag for the commit the user is
+ presently on, something like mediawiki-1.2.3-scripts-4-g123abcd
+ """
+ return "%s-%s" % (self.application, self.version)
+ @property
def scripts_tag(self):
"""
Returns the name of the Git tag for this version.
-
- .. note::
-
- Use this function only during migration, as it does
- not account for the existence of ``-scripts2``.
"""
- return "%s-scripts" % self.pristine_tag
+ end = str(self.version).partition('-scripts')[2].partition('-')[0]
+ return "%s-scripts%s" % (self.pristine_tag, end)
@property
def pristine_tag(self):
"""
Returns the name of the Git tag for the pristine version corresponding
to this version.
"""
- return "%s-%s" % (self.application.name, self.version)
+ return "%s-%s" % (self.application.name, str(self.version).partition('-scripts')[0])
def __cmp__(x, y):
return cmp(x.version, y.version)
@staticmethod
def __str__(self):
return """Could not find Git repository for '%s'. If you would like to use a local version, try specifying --srv-path or WIZARD_SRV_PATH.""" % self.app
-# If you want, you can wrap this up into a registry and access things
-# through that, but it's not really necessary
+class NotMigratedError(Error):
+ """
+ The deployment contains a .scripts-version file, but no .git
+ or .scripts directory.
+ """
+ #: Directory of deployment
+ dir = None
+ def __init__(self, dir):
+ self.dir = dir
+ def __str__(self):
+ return """This installation was not migrated"""
+
+class AlreadyVersionedError(Error):
+ """The deployment contained a .git directory but no .scripts directory."""
+ #: Directory of deployment
+ dir = None
+ def __init__(self, dir):
+ self.dir = dir
+ def __str__(self):
+ return """
+
+ERROR: Directory contains a .git directory, but not
+a .scripts directory. If this is not a corrupt
+migration, this means that the user was versioning their
+install using Git."""
+
+class NotConfiguredError(Error):
+ """The install was missing essential configuration."""
+ #: Directory of unconfigured install
+ dir = None
+ def __init__(self, dir):
+ self.dir = dir
+ def __str__(self):
+ return """
+
+ERROR: The install was well-formed, but not configured
+(essential configuration files were not found.)"""
+
+class CorruptedAutoinstallError(Error):
+ """The install was missing a .git directory, but had a .scripts directory."""
+ #: Directory of the corrupted install
+ dir = None
+ def __init__(self, dir):
+ self.dir = dir
+ def __str__(self):
+ return """
+
+ERROR: Directory contains a .scripts directory,
+but not a .git directory."""
+
+class NotAutoinstallError(Error):
+ """The directory was not an autoinstall, due to missing .scripts-version file."""
+ #: Directory in question
+ dir = None
+ def __init__(self, dir):
+ self.dir = dir
+ def __str__(self):
+ return """
+
+ERROR: Could not find .scripts-version file. Are you sure
+this is an autoinstalled application?
+"""
+
+class NoTagError(Error):
+ """Deployment has a tag that does not have an equivalent in upstream repository."""
+ #: Missing tag
+ tag = None
+ def __init__(self, tag):
+ self.tag = tag
+ def __str__(self):
+ return """
+
+ERROR: Could not find tag %s in repository.""" % self.tag
+
+class NoLocalTagError(Error):
+ """Could not find tag in local repository."""
+ #: Missing tag
+ tag = None
+ def __init__(self, tag):
+ self.tag = tag
+ def __str__(self):
+ return """
+
+ERROR: Could not find tag %s in local repository.""" % self.tag
+
+class InconsistentPristineTagError(Error):
+ """Pristine tag commit ID does not match upstream pristine tag commit ID."""
+ #: Inconsistent tag
+ tag = None
+ def __init__(self, tag):
+ self.tag = tag
+ def __str__(self):
+ return """
+
+ERROR: Local pristine tag %s did not match repository's. This
+probably means an upstream rebase occured.""" % self.tag
+
+class InconsistentScriptsTagError(Error):
+ """Scripts tag commit ID does not match upstream scripts tag commit ID."""
+ #: Inconsistent tag
+ tag = None
+ def __init__(self, tag):
+ self.tag = tag
+ def __str__(self):
+ return """
+
+ERROR: Local scripts tag %s did not match repository's. This
+probably means an upstream rebase occurred.""" % self.tag
+
+class HeadNotDescendantError(Error):
+ """HEAD is not connected to tag."""
+ #: Tag that HEAD should have been descendant of.
+ tag = None
+ def __init__(self, tag):
+ self.tag = tag
+ def __str__(self):
+ return """
+
+ERROR: HEAD is not a descendant of %s. This probably
+means that an upstream rebase occurred, and new tags were
+pulled, but local user commits were never rebased.""" % self.tag
+
+class VersionDetectionError(Error):
+ """Could not detect real version of application."""
+ def __str__(self):
+ return """
+
+ERROR: Could not detect the real version of the application."""
+
+class VersionMismatchError(Error):
+ """Git version of application does not match detected version."""
+ #: Detected version
+ real_version = None
+ #: Version from Git
+ git_version = None
+ def __init__(self, real_version, git_version):
+ self.real_version = real_version
+ self.git_version = git_version
+ def __str__(self):
+ return """
+
+ERROR: The detected version %s did not match the Git
+version %s.""" % (self.real_version, self.git_version)
+
+class WebVerificationError(Error):
+ """Could not access the application on the web"""
+ #: Contents of web page access
+ contents = None
+ def __init__(self, contents):
+ self.contents = contents
+ def __str__(self):
+ return """
+
+ERROR: We were not able to access the application on the
+web. This may indicate that the website is behind
+authentication on the htaccess level. The contents
+of the page were:
+
+%s""" % self.contents
+
+class UnknownWebPath(Error):
+ """Could not determine application's web path."""
+ def __str__(self):
+ return """
+
+ERROR: We were not able to determine what the application's
+host and path were in order to perform a web request
+on the application. You can specify this manually using
+the WIZARD_WEB_HOST and WIZARD_WEB_PATH environment
+variables."""
_application_list = [
"mediawiki", "wordpress", "joomla", "e107", "gallery2",