#!/usr/bin/env python
-"""
-This script generates basic statistics about our autoinstalls.
-"""
-
import os
import optparse
-import fileinput
-import math
import sys
-from distutils.version import LooseVersion as Version
+import logging
+import traceback
-class NoSuchApplication(Exception):
- pass
+# import some non-standard modules to make it fail out early
+import decorator
-class DeploymentParseError(Exception):
- pass
+sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
-class Deployment(object):
- def __init__(self, location, version):
- self.location = location
- self.version = version
- self.application = version.application
- @staticmethod
- def parse(line):
- """Parses a line from the results of parallel-find.pl.
- This will work out of the box with fileinput"""
- try:
- location, deploydir = line.rstrip().split(":")
- except ValueError:
- raise DeploymentParseError
- name = deploydir.split("/")[-1]
- if name.find("-") != -1:
- app, version = name.split("-")
- elif name == "deploy":
- # Assume that it's django, since those were botched
- app = "django"
- version = "0.1-scripts"
- else:
- raise DeploymentParseError
- try:
- return Deployment(location, applications[app].getVersion(version))
- except KeyError:
- raise NoSuchApplication
- def count(self):
- """Simple method which registers the deployment as a +1 on the
- appropriate version. No further inspection is done."""
- self.version.count(self)
- return True
- def count_exists(self, file):
- """Checks if the codebase has a certain file/directory in it."""
- if os.path.exists(self.location + "/" + file):
- self.version.count_exists(self, file)
- return True
- return False
+import wizard
+from wizard import command, prompt
-class Application(object):
- HISTOGRAM_WIDTH = 30
- def __init__(self, name):
- self.name = name
- self.versions = {}
- # Some cache variables for fast access of calculated data
- self._total = 0
- self._max = 0
- self._c_exists = {}
- def getVersion(self, version):
- if version not in self.versions:
- self.versions[version] = ApplicationVersion(Version(version), self)
- return self.versions[version]
- def _graph(self, v):
- return '+' * int(math.ceil(float(v)/self._max * self.HISTOGRAM_WIDTH))
- def __str__(self):
- if not self.versions: return "%-11s no installs" % self.name
- ret = \
- ["%-16s %3d installs" % (self.name, self._total)] + \
- [str(v) for v in sorted(self.versions.values())]
- for f,c in self._c_exists.items():
- ret.append("%d users have %s" % (c,f))
- return "\n".join(ret)
+def main():
+ usage = """usage: %prog COMMAND [ARGS]
-class ApplicationVersion(object):
- def __init__(self, version, application):
- self.version = version
- self.application = application
- self.c = 0
- self.c_exists = {}
- def __cmp__(x, y):
- return cmp(x.version, y.version)
- def count(self, deployment):
- self.c += 1
- self.application._total += 1
- if self.c > self.application._max:
- self.application._max = self.c
- def count_exists(self, deployment, n):
- if n in self.c_exists: self.c_exists[n] += 1
- else: self.c_exists[n] = 1
- if n in self.application._c_exists: self.application._c_exists[n] += 1
- else: self.application._c_exists[n] = 1
- def __str__(self):
- return " %-12s %3d %s" \
- % (self.version, self.c, self.application._graph(self.c))
+Wizard is a Git-based autoinstall management system for scripts.
-class Printer(object):
- def __init__(self, quiet, verbose):
- self.i = 0
- self.quiet = quiet
- self.verbose = verbose
- self.hanging = False
- def tick(self):
- self.i += 1
- if not self.quiet and self.i % 10 == 0:
- sys.stdout.write(".")
- sys.stdout.flush()
- self.hanging = True
- def _hang(self):
- if self.hanging:
- self.hanging = False
- print
- def write(self, str = ""):
- self._hang()
- print str
- def qwrite(self, str = ""):
- if not self.quiet:
- self._hang
- print str
- def tweet(self, str = ""):
- if not self.quiet:
- self._hang()
- print str, # note comma
- def chat(self, str = ""):
- if self.verbose:
- self._hang()
- print str
+User commands:
+ backup Backup data not on filesystem (database, etc)
+ install Installs an application
+ migrate Migrate autoinstalls from old format to Git-based format
+ remove Removes an autoinstall, databases and other files
+ restore Restores files and database to previous version
+ upgrade Upgrades an autoinstall to the latest version
-application_list = [
- "mediawiki", "wordpress", "joomla", "e107", "gallery2",
- "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
- # these are technically deprecated
- "advancedpoll", "gallery",
-]
+Administrative commands:
+ blacklist Marks an autoinstall to not try upgrades
+ errors Lists all broken autoinstall metadata
+ list Lists autoinstalls, with optional filtering
+ mass-migrate Performs mass migration of autoinstalls of an application
+ mass-upgrade Performs mass upgrade of autoinstalls of an application
+ research Print statistics about a possible upgrade
+ summary Generate statistics (see help for subcommands)
-"""Hash table for looking up string application name to instance"""
-applications = dict([(n,Application(n)) for n in application_list ])
+Utility commands:
+ prepare-pristine Downloads and extracts pristine upstream files
+ prepare-new Prepares a new repository
+ prepare-config Prepares configuration files for versioning
+ quota Prints the usage and available quota of a directory
-def main():
- usage = """usage: %prog [options] [application]
+See '%prog help COMMAND' for more information on a specific command."""
-Scans all of the collected data from parallel-find.pl, and
-determines version histograms for our applications. You may
-optionally pass application parameters to filter the installs.
-
-Examples:
- %prog
- Basic usage
- %prog mediawiki
- Displays only MediaWiki statistics
- %prog -v -q mediawiki-1.2.3
- Displays all deployments of this version"""
parser = optparse.OptionParser(usage)
- parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
- default=False, help="Print interesting directories")
- parser.add_option("-q", "--quiet", dest="quiet", action="store_true",
- default=False, help="Suppresses progress output")
- parser.add_option("-d", "--version-dir", dest="version_dir",
- default="/afs/athena.mit.edu/contrib/scripts/sec-tools/store/versions",
- help="Location of parallel-find output")
- parser.add_option("--count-exists", dest="count_exists",
- default=False, help="Count deployments that contain a file")
- # There should be machine friendly output
- options, show = parser.parse_args()
- if not show: show = applications.keys()
- show = frozenset(show)
- vd = options.version_dir
+ parser.disable_interspersed_args()
+ _, args = parser.parse_args() # no global options
+ rest_argv = args[1:]
+ baton = command.OptionBaton()
+ baton.add("--versions-path", dest="versions_path", metavar="PATH",
+ default=getenvpath("WIZARD_VERSIONS_PATH") or "/afs/athena.mit.edu/contrib/scripts/sec-tools/store/versions",
+ help="Location of parallel-find output directory, or a file containing a newline separated list of 'all autoinstalls' (for development work). Environment variable is WIZARD_VERSIONS_PATH.")
+ baton.add("--srv-path", dest="srv_path", metavar="PATH",
+ default=getenvpath("WIZARD_SRV_PATH") or "/afs/athena.mit.edu/contrib/scripts/git/autoinstalls",
+ help="Location of autoinstall Git repositories, such that $REPO_PATH/$APP.git is a repository (for development work). Environment variable is WIZARD_SRV_PATH.")
+ baton.add("--dry-run", dest="dry_run", action="store_true",
+ default=False, help="Performs the operation without actually modifying any files. Use in combination with --verbose to see commands that will be run.")
+ # common variables for mass commands
+ baton.add("--seen", dest="seen",
+ default=None, help="File to read/write paths of successfully modified installs;"
+ "these will be skipped on re-runs. If --log-dir is specified, this is automatically enabled.")
+ baton.add("--no-parallelize", dest="no_parallelize", action="store_true",
+ default=False, help="Turn off parallelization")
+ baton.add("--max-processes", dest="max_processes", type="int", metavar="N",
+ default=5, help="Maximum subprocesses to run concurrently")
+ baton.add("--limit", dest="limit", type="int",
+ default=None, help="Limit the number of autoinstalls to look at.")
+ baton.add("--user", "-u", dest="user",
+ default=None, help="Only mass migrate a certain user's installs. No effect if versions_path is a file.")
try:
- fi = fileinput.input([vd + "/" + f for f in os.listdir(vd)])
- except OSError:
- print "No permissions; check if AFS is mounted"
- raise SystemExit(-1)
- errors = 0
- unrecognized = 0
- processed = 0
- printer = Printer(options.quiet, options.verbose)
- # I really don't like this boolean
- hanging = False # whether or not we last outputted a newline
- printer.tweet("Processing")
- for line in fi:
- printer.tick()
+ command_name = args[0]
+ except IndexError:
+ parser.print_help()
+ sys.exit(1)
+ baton.add("--log-dir", dest="log_dir",
+ default=getenvpath("WIZARD_LOG_DIR") or "/tmp/wizard-%s" % command_name,
+ help="Log files for Wizard children processes are placed here.")
+ if command_name == "help":
try:
- deploy = Deployment.parse(line)
- except DeploymentParseError:
- errors += 1
- continue
- except NoSuchApplication:
- unrecognized += 1
- continue
- if deploy.application.name + "-" + str(deploy.version.version) in show:
- printer.write("%s-%s deployment at %s" \
- % (deploy.application.name, deploy.version.version, deploy.location))
- elif deploy.application.name in show:
- pass
+ help_module = get_command(rest_argv[0])
+ except ImportError:
+ parser.error("invalid action")
+ except IndexError:
+ parser.print_help()
+ sys.exit(1)
+ help_module.main(['--help'], baton)
+ # Dispatch commands
+ command_module = get_command(command_name)
+ try:
+ command_module.main(rest_argv, baton)
+ except prompt.UserCancel as e:
+ print str(e)
+ sys.exit(1)
+ except Exception as e:
+ # log the exception
+ msg = traceback.format_exc()
+ if command.logging_setup:
+ outfun = logging.error
+ else:
+ outfun = sys.stderr.write
+ if isinstance(e, wizard.Error):
+ if e.quiet and not command.debug:
+ msg = str(e)
+ if command.logging_setup:
+ msg = msg.replace("ERROR: ", "")
+ outfun(msg)
+ sys.exit(e.exitcode)
else:
- continue
- deploy.count()
- if options.count_exists:
- r = deploy.count_exists(options.count_exists)
- if r:
- printer.chat("Found " + options.count_exists + " in " + deploy.location)
- printer.write()
- for app in applications.values():
- if app.name not in show: continue
- printer.write(app)
- printer.write()
- printer.write("With %d errors and %d unrecognized applications" % (errors, unrecognized))
+ outfun(msg)
+ sys.exit(1)
+
+def get_command(name):
+ name = name.replace("-", "_")
+ __import__("wizard.command." + name)
+ return getattr(wizard.command, name)
+
+def getenvpath(name):
+ val = os.getenv(name)
+ if val:
+ val = os.path.abspath(val)
+ return val
if __name__ == "__main__":
main()