module/wizard.merge
module/wizard.prompt
module/wizard.resolve
- module/wizard.scripts
module/wizard.shell
module/wizard.sql
module/wizard.sset
.. autofunction:: get_install_lines
.. autofunction:: parse_install_lines
.. autofunction:: chdir_to_location
+.. autofunction:: web
Exceptions
----------
+++ /dev/null
-:mod:`wizard.scripts`
-=====================
-
-.. automodule:: wizard.scripts
-
-Functions
----------
-.. autofunction:: fill_url
-.. autofunction:: get_quota_usage_and_limit
-.. autofunction:: get_disk_usage
-
-Exceptions
-----------
-.. autoexception:: QuotaParseError
-
.. autofunction:: mixed_newlines
.. autofunction:: random_key
.. autofunction:: makedirs
+.. autofunction:: disk_usage
Exceptions
----------
.. todo::
- The below plugins are not implemented by Wizard yet.
+ Plugins marked with prototype todo messages are not complete yet.
``wizard.strategy``
-------------------
.. todo:: Prototype
+``wizard.deploy.web``
+---------------------
+
+Used to fill in a user's web URL if it is not able to be determined from
+an application's configuration files. Plugin should be a function that
+takes a single required argument ``dir``, which is the directory to
+determint the web URL(s) for, and return a list or generator of
+:class:`urlparse.ParseResult` objects or URL strings of possible web locations
+for an application.
+
``wizard.user.email``
---------------------
alternative ``passwd`` lookup.
.. todo:: Prototype
+
+``wizard.user.quota``
+---------------------
+
+Plugin for :func:`wizard.user.quota`. Wizard has safeguards for
+avoiding exceeding user quotas when an automated upgrade is being
+performed. Unfortunately, methods for enforcing quotas are frequently
+highly system dependent. Use this hook to implement quota usage and
+limit reporting. Plugin should be a function that takes a single
+required argument ``dir``, which is the directory to determine the quota
+for, and return a tuple ``(quota usage, quota limit)`` or ``(0, None)``
+if it could not determine quota.
--- /dev/null
+wizard_scripts.egg-info
--- /dev/null
+import setuptools
+
+setuptools.setup(
+ name = 'wizard-scripts',
+ version = '0.1.dev',
+ author = 'The Scripts Team',
+ author_email = 'scripts-team@mit.edu',
+ description = ('Customizations to Wizard for scripts.mit.edu'),
+ license = 'MIT',
+ url = 'http://scripts.mit.edu/',
+ packages = setuptools.find_packages(),
+ entry_points = {
+ 'wizard.user.quota': 'scripts = wizard.scripts:user_quota',
+ 'wizard.deploy.web': 'scripts = wizard.scripts:deploy_web',
+ }
+)
import wizard
from wizard import shell, util
-def fill_url(dir, url=None, old_style=False):
- """
- Attempts to determine the URL a directory would be web-accessible at.
- If ``url`` is specified, automatically use it.
- """
- if url:
- return url
-
- # hook hook
-
+def deploy_web()
# try the directory
homedir, _, web_path = dir.partition("/web_scripts")
if web_path:
- if old_style:
- return urlparse.ParseResult(
- "http",
- "scripts.mit.edu",
- "/~" + util.get_dir_owner(homedir) + web_path.rstrip('/'),
- "", "", "")
- else:
- return urlparse.ParseResult(
- "http",
- util.get_dir_owner(homedir) + ".scripts.mit.edu",
- web_path.rstrip('/'),
- "", "", "")
-
- # try the environment
- host = os.getenv("WIZARD_WEB_HOST")
- path = os.getenv("WIZARD_WEB_PATH")
- if host is not None and path is not None:
- return urlparse.ParseResult(
+ yield urlparse.ParseResult(
"http",
- host,
- path.rstrip('/'),
+ util.get_dir_owner(homedir) + ".scripts.mit.edu",
+ web_path.rstrip('/'),
+ "", "", "")
+ yield urlparse.ParseResult(
+ "http",
+ "scripts.mit.edu",
+ "/~" + util.get_dir_owner(homedir) + web_path.rstrip('/'),
"", "", "")
- return None
-
-def get_quota_usage_and_limit(dir=None):
+def user_quota(dir=None):
"""
Returns a tuple (quota usage, quota limit). Works only for scripts
servers. Values are in KiB. Returns ``(0, None)`` if we couldn't figure it out.
# sometimes the volume is busy; so we try several times
for i in range(0, end + 1):
try:
- return _get_quota_usage_and_limit(dir)
+ return _user_quota(dir)
except QuotaParseError as e:
if i == end:
raise e
time.sleep(3) # give it a chance to unbusy
assert False # should not get here
-def _get_quota_usage_and_limit(dir=None):
+def _user_quota(dir=None):
# XXX: The correct way is to implement Python modules implementing
# bindings for all the appropriate interfaces
+ unknown = (0, None)
def parse_last_quote(ret):
return ret.rstrip('\'').rpartition('\'')[2]
if dir is None:
try:
cell = parse_last_quote(sh.eval("fs", "whichcell", "-path", dir))
except shell.CallError:
- return (0, None)
+ return unknown
except OSError as e:
if e.errno == errno.ENOENT:
- return (0, None)
+ return unknown
raise
mount = None
while dir:
dir = os.path.dirname(dir)
except OSError as e:
if e.errno == errno.ENOENT:
- return (0, None)
+ return unknown
raise
- if not volume: return (0, None)
+ if not volume: return unknown
try:
result = sh.eval("vos", "examine", "-id", volume, "-cell", cell).splitlines()
except OSError:
try:
result = sh.eval("/usr/sbin/vos", "examine", "-id", volume, "-cell", cell).splitlines()
except OSError:
- return (0, None)
+ return unknown
except shell.CallError:
- return (0, None)
+ return unknown
try:
usage = int(result[0].split()[3])
limit = int(result[3].split()[1]) # XXX: FRAGILE
raise QuotaParseError("vos examine output was:\n\n" + "\n".join(result))
return (usage, limit)
-# XXX: Possibly in the wrong module
-def get_disk_usage(dir=None, excluded_dir=".git"):
- """
- Recursively determines the disk usage of a directory, excluding
- .git directories. Value is in bytes.
- """
- if dir is None: dir = os.getcwd()
- sum_sizes = 0
- for root, _, files in os.walk(dir):
- for name in files:
- if not os.path.join(root, name).startswith(dir + excluded_dir):
- file = os.path.join(root, name)
- try:
- if os.path.islink(file): continue
- sum_sizes += os.path.getsize(file)
- except OSError as e:
- if e.errno == errno.ENOENT:
- logging.warning("%s disappeared before we could stat", file)
- else:
- raise
- return sum_sizes
-
class QuotaParseError(wizard.Error):
"""Could not parse quota information."""
def __init__(self, msg):
cd /mit/scripts/wizard
git pull
python setup.py egg_info
+cd plugins/scripts
+python setup.py egg_info
# crlf-ify the install
FROB="RELEASE-NOTES"
-unix2dos "$TESTDIR/$FROB"
+unix2dos "$TESTDIR/$FROB" || todos "$TESTDIR/$FROB"
wizard upgrade "$TESTDIR" --non-interactive
-WIZARD_TITLE="TestApp" wizard install "mediawiki-$VERSION-scripts" "$TESTDIR" --non-interactive -- --title="TestApp"
+wizard install "mediawiki-$VERSION-scripts" "$TESTDIR" --non-interactive -- --title="TestApp"
import pkg_resources
import wizard
-from wizard import resolve, scripts, shell, sql, util
+from wizard import resolve, shell, sql, util
# SCRIPTS SPECIFIC
_scripts_application_list = [
def checkConfig(self, deployment):
return os.path.isfile("wp-config.php")
def checkWeb(self, deployment):
- # XXX: this sucks pretty hard
- def doCheck():
- return self.checkWebPage(deployment, "",
- outputs=["<html", "WordPress", "feed"],
- exclude=["Error establishing a database connection"])
- if not doCheck():
- deployment.enableOldStyleUrls()
- return doCheck()
- else:
- return True
+ return self.checkWebPage(deployment, "",
+ outputs=["<html", "WordPress", "feed"],
+ exclude=["Error establishing a database connection"])
def detectVersion(self, deployment):
return self.detectVersionFromFile("wp-includes/version.php", php.re_var("wp_version"))
def install(self, version, options):
import shutil
import errno
-from wizard import deploy, report, scripts, shell, sset, command
+from wizard import deploy, report, shell, sset, command
from wizard.command import upgrade
def main(argv, baton):
import os.path
import sys
-from wizard import command, scripts
+from wizard import command, user
def main(argv, baton):
options, args = parse_args(argv, baton)
dir = os.path.abspath(args[0]) if args else os.getcwd()
- r = scripts.get_quota_usage_and_limit(dir)
+ r = user.quota(dir)
if r[0] is None or r[1] is None:
sys.exit(1)
print "%d %d" % r
import time
import errno
-from wizard import app, command, deploy, merge, scripts, shell, util
+from wizard import app, command, deploy, merge, shell, user, util
-kib_buffer = 1024 * 30 # 30 MiB we will always leave available
+buffer = 1024 * 1024 * 30 # 30 MiB we will always leave available
errno_blacklisted = 64
def main(argv, baton):
"""Restore :attr:`prod` attribute, and check if the production copy has drifted."""
self.prod = deploy.ProductionCopy(".")
try:
- shell.call("git", "status")
- raise LocalChangesError()
+ r1 = shell.eval("git", "diff-files", "--name-only").strip()
+ r2 = shell.eval("git", "diff-index", "--name-only", "HEAD").strip()
+ if r1 or r2:
+ raise LocalChangesError()
except shell.CallError:
pass
# Working copy is not anchored anywhere useful for git describe,
sys.stderr.write("Traceback:\n (n/a)\nAlreadyUpgraded\n")
sys.exit(2)
def preflightQuota(self):
- kib_usage, kib_limit = scripts.get_quota_usage_and_limit()
- if kib_limit is not None and (kib_limit - kib_usage) < kib_buffer:
+ usage, limit = user.quota()
+ if limit is not None and (limit - usage) < buffer:
raise QuotaTooLow
def merge(self):
# yeah yeah no trailing newline whatever
open(".git/WIZARD_UPGRADE_VERSION", "w").write(self.version)
open(".git/WIZARD_PARENTS", "w").write("%s\n%s" % (self.user_commit, self.next_commit))
- open(".git/WIZARD_SIZE", "w").write(str(scripts.get_disk_usage()))
+ open(".git/WIZARD_SIZE", "w").write(str(util.disk_usage()))
if self.options.log_file:
open(".git/WIZARD_LOG_FILE", "w").write(self.options.log_file)
def mergePerform(self):
# Ok, now we have to do a crazy complicated dance to see if we're
# going to have enough quota to finish what we need
pre_size = int(open(os.path.join(self.temp_wc_dir, ".git/WIZARD_SIZE"), "r").read())
- post_size = scripts.get_disk_usage(self.temp_wc_dir)
+ post_size = util.disk_usage(self.temp_wc_dir)
backup = self.prod.backup(self.options)
- kib_usage, kib_limit = scripts.get_quota_usage_and_limit()
- if kib_limit is not None and (kib_limit - kib_usage) - (post_size - pre_size) / 1024 < kib_buffer:
+ usage, limit = user.quota()
+ if limit is not None and (limit - usage) - (post_size - pre_size) < buffer:
shutil.rmtree(os.path.join(".scripts/backups", shell.eval("wizard", "restore").splitlines()[0]))
raise QuotaTooLow
return backup
import traceback
import shutil
import errno
+import pkg_resources
+import urlparse
import wizard
-from wizard import app, git, old_log, scripts, shell, sql, util
+from wizard import app, git, old_log, shell, sql, util
## -- Global Functions --
# yield
yield d
+def web(dir, url=None):
+ """
+ Attempts to determine the URL a directory would be web-accessible at.
+ If ``url`` is specified, automatically use it. Returns a generator whic
+ produces a list of candidate urls.
+ """
+ if url:
+ yield url
+ return
+
+ for f in pkg_resources.iter_entry_points("deploy.web"):
+ for r in f(dir):
+ if isinstance(r, str):
+ r = urlparse.urlparse(r)
+ yield r
+
+ # try the environment
+ host = os.getenv("WIZARD_WEB_HOST")
+ path = os.getenv("WIZARD_WEB_PATH")
+ if host is not None and path is not None:
+ yield urlparse.ParseResult(
+ "http",
+ host,
+ path.rstrip('/'),
+ "", "", "")
+
## -- Model Objects --
@decorator.decorator
self._old_log = None
self._dsn = None
self._url = None
+ self._urlGen = None
def invalidateCache(self):
"""
Invalidates all cached variables. This currently applies to
@property
def url(self):
"""The :class:`urlparse.ParseResult` for this deployment."""
- if not self._url:
- self._url = scripts.fill_url(self.location, self.application.url(self))
- if not self._url:
- raise UnknownWebPath
+ if not self._urlGen:
+ self._urlGen = web(self.location, self.application.url(self))
+ self.nextUrl()
return self._url
+ def nextUrl(self):
+ """"""
+ try:
+ self._url = self._urlGen.next() # pylint: disable-msg=E1101
+ return self._url
+ except StopIteration:
+ raise UnknownWebPath
def enableOldStyleUrls(self):
"""
Switches to using http://user.scripts.mit.edu/~user/app URLs.
No effect if they have an explicit .scripts/url override.
"""
+ # XXX: This is pretty scripts specific
self._url = scripts.fill_url(self.location, self.application.url(self), old_style = True)
@staticmethod
def parse(line):
raise DatabaseVerificationError
def verifyWeb(self):
"""
- Checks if the autoinstall is viewable from the web.
+ Checks if the autoinstall is viewable from the web. If you do not run
+ this, there is no guarantee that the url returned by this application
+ is the correct one.
"""
- if not self.application.checkWeb(self):
- raise WebVerificationError
+ while True:
+ if not self.application.checkWeb(self):
+ try:
+ self.nextUrl()
+ except UnknownWebPath:
+ raise WebVerificationError
+ else:
+ break
def fetch(self, path, post=None):
"""
Performs a HTTP request on the website.
import warnings
import wizard
-from wizard import scripts, shell, sql, util
+from wizard import deploy, shell, sql, util
def dsn_callback(options):
if not isinstance(options.dsn, sqlalchemy.engine.url.URL):
class ScriptsWebStrategy(Strategy):
"""Performs scripts specific guesses for web variables."""
+ # XXX: This actually isn't too scripts specific
provides = frozenset(["web_host", "web_path"])
def __init__(self, dir):
self.dir = dir
def prepare(self):
- """Uses :func:`wizard.scripts.get_web_host_and_path`."""
+ """Uses :func:`deploy.web`."""
if self.dir is None:
raise StrategyFailed
- self._url = scripts.fill_url(self.dir, None)
- if not self._url:
+ urls = deploy.web(self.dir, None)
+ if not urls:
+ raise StrategyFailed
+ try:
+ self._url = urls.next()
+ except StopIteration:
raise StrategyFailed
def execute(self, options):
"""No-op."""
self.application = application
self.dir = dir
def prepare(self):
- """Uses :func:`wizard.scripts.get_sql_credentials`"""
+ """Uses the SQL programs in the scripts locker"""
if self.application.database != "mysql":
raise StrategyFailed
try:
+++ /dev/null
-import os
-import unittest
-
-from wizard import tests
-from wizard.scripts import *
-
-class GetDiskUsageTest(unittest.TestCase):
- def basicTest(self):
- self.assertEqual(get_disk_usage(tests.getTestFile("scripts_test"), "ignore_me"), 7)
with LockDirectory(lockfile):
with LockDirectory(lockfile, expiry = 0):
pass
+
+def test_disk_usage():
+ assert disk_usage(tests.getTestFile("disk_usage_test"), "ignore_me") == 7
--- /dev/null
+"""
+Module for querying information about users. This mostly asks plugins for
+the extra information, and falls back to using a default that should work
+on most systems (but by no means all systems.)
+"""
+
+import pkg_resources
+import os
+
+def quota(dir=None):
+ """
+ Returns a tuple (quota usage, quota limit). Returns ``(0, None)`` if
+ the quota usage is unknown. If ``dir`` is omitted, the current
+ working directory is assumed. Value returned is in bytes.
+ """
+ if dir is None:
+ dir = os.getcwd()
+ unknown = (0, None)
+ for func in pkg_resources.iter_entry_points("wizard.user.quota"):
+ r = func(dir)
+ if r != unknown:
+ return r
+ return unknown
f.close() # just to be safe
return ret
+def disk_usage(dir=None, excluded_dir=".git"):
+ """
+ Recursively determines the disk usage of a directory, excluding
+ .git directories. Value is in bytes. If ``dir`` is omitted, the
+ current working directory is assumed.
+ """
+ if dir is None: dir = os.getcwd()
+ sum_sizes = 0
+ for root, _, files in os.walk(dir):
+ for name in files:
+ if not os.path.join(root, name).startswith(os.path.join(dir, excluded_dir)):
+ file = os.path.join(root, name)
+ try:
+ if os.path.islink(file): continue
+ sum_sizes += os.path.getsize(file)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ logging.warning("%s disappeared before we could stat", file)
+ else:
+ raise
+ return sum_sizes
+
def random_key(length=30):
"""Generates a random alphanumeric key of ``length`` size."""
return ''.join(random.choice(string.letters + string.digits) for i in xrange(length))
scripts.mit.edu / wizard.git / commitdiff
