X-Git-Url: https://scripts.mit.edu/gitweb/wizard.git/blobdiff_plain/5dc9b5012125951d56f2657c191f17d1e6ac4aae..9f2c4a526259a93972135cf1e8ddc22aa2631da3:/wizard/command/__init__.py diff --git a/wizard/command/__init__.py b/wizard/command/__init__.py index 3e0f43f..fb8b45b 100644 --- a/wizard/command/__init__.py +++ b/wizard/command/__init__.py @@ -4,37 +4,15 @@ import os import sys import optparse import errno +import pwd +import shutil +import cStringIO import wizard +from wizard import util logging_setup = False - -class Error(wizard.Error): - """Base error class for all command errors""" - pass - -class PermissionsError(Error): - def __init__(self, dir): - self.dir = dir - def __str__(self): - return """ - -ERROR: You don't have permissions to access this directory. -Do you have tokens for AFS with your root instance, and -is your root instance on scripts-security-upd? - -You can check by running the commands 'klist' and -'blanche scripts-security-upd'. -""" - -class NoSuchDirectoryError(Error): - def __init__(self, dir): - self.dir = dir - def __str__(self): - return """ - -ERROR: No such directory... check your typing -""" +debug = True # This will get overwritten with the real value early on def boolish(val): """ @@ -55,46 +33,40 @@ def boolish(val): return False return bool(val) -def chdir(dir): - try: - os.chdir(dir) - except OSError as e: - if e.errno == errno.EACCES: - raise PermissionsError(dir) - elif e.errno == errno.ENOENT: - raise NoSuchDirectoryError(dir) - else: raise e - -def makeLogger(options, numeric_args): +def setup_logger(options, numeric_args): global logging_setup if logging_setup: return logging.getLogger() logger = logging.getLogger() + logger.handlers = [] # under certain cases, a spurious stream handler is set. We don't know why logger.setLevel(logging.INFO) stderr = logging.StreamHandler(sys.stderr) stderr.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) - if not options.quiet: logger.addHandler(stderr) - else: logger.addHandler(NullLogHandler()) # prevent default + if not options.quiet: + logger.addHandler(stderr) + else: + logger.addHandler(NullLogHandler()) # prevent default if options.log_file: - file = logging.FileHandler(options.log_file) - logformatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s", "%H:%M:%S") - file.setFormatter(logformatter) - logger.addHandler(file) + setup_file_logger(options.log_file, options.debug) if options.debug: logger.setLevel(logging.DEBUG) else: stderr.setLevel(logging.WARNING) - if options.verbose or hasattr(options, "dry_run"): + if options.verbose: stderr.setLevel(logging.INFO) - if options.log_file: - file.setLevel(logging.INFO) - def our_excepthook(type, value, tb): - logging.error("".join(traceback.format_exception(type,value,tb))) - sys.exit(1) - sys.excepthook = our_excepthook logging_setup = True return logger -def makeBaseArgs(options, **grab): +def setup_file_logger(log_file, debug): + logger = logging.getLogger() + file = logging.FileHandler(log_file) + logformatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s", "%Y-%m-%d %H:%M") + file.setFormatter(logformatter) + logger.addHandler(file) + if not debug: + file.setLevel(logging.INFO) + return file + +def make_base_args(options, **grab): """Takes parsed options, and breaks them back into a command line string that we can pass into a subcommand""" args = [] @@ -110,6 +82,125 @@ def makeBaseArgs(options, **grab): args.append(str(value)) return args +def security_check_homedir(location): + """ + Performs a check against a directory to determine if current + directory's owner has a home directory that is a parent directory. + This protects against malicious mountpoints, and is roughly equivalent + to the suexec checks. + """ + try: + uid = util.get_dir_uid(location) + real = os.path.realpath(location) + if not real.startswith(pwd.getpwuid(uid).pw_dir + "/"): + logging.error("Security check failed, owner of deployment and " + "owner of home directory mismatch for %s" % location) + return False + except KeyError: + logging.error("Security check failed, could not look up " + "owner of %s (uid %d)" % (location, uid)) + return False + except OSError as e: + logging.error("OSError: %s" % str(e)) + return False + return True + +def calculate_log_name(log_dir, i): + """ + Calculates a log entry given a numeric identifier, and + directory under operation. + """ + return os.path.join(log_dir, "%04d.log" % i) + +def create_logdir(log_dir): + """ + Creates a log directory and chmods it 777 to enable de-priviledged + processes to create files. + """ + try: + os.mkdir(log_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + #if create_subdirs: + # log_dir = os.path.join(log_dir, str(int(time.time()))) + # os.mkdir(log_dir) # if fails, be fatal + # # XXX: update last symlink + os.chmod(log_dir, 0o777) + +class Report(object): + #: Set of indices that should be skipped + skip = None + #: Dict of append names to counts. You should manually increment these as necessary + fails = None + #: Number of successes + successes = 0 + #: Names of the files objects + names = None + def __init__(self, names, fobjs, skip, fails): + self.skip = skip + self.names = names + self.fails = fails + for name, fobj in zip(names, fobjs): + setattr(self, name, fobj) + def flush(self): + for n in self.names: + getattr(self, n).flush() + +def report_files(log_dir, names): + return [os.path.join(os.path.join(log_dir, "%s.txt" % x)) for x in names] + +def read_reports(log_dir, names): + """ + Reads a number of reports files. The return value is a :class:`Report` + object with attributes that are open file objects that correspond to ``names``. + """ + return Report(names, [(os.path.exists(f) and open(f, "r") or cStringIO.StringIO()) for f in report_files(log_dir, names)], set(), {}) + +def open_reports(log_dir, names=('warnings', 'errors'), redo=False, append_names=()): + """ + Returns a :class:`Report` object configured appropriately for the + parameters passed. This object has attributes names + append_names which + contain file objects opened as "w". ``names`` report files are cleared unconditionally + when they are opened (i.e. are not preserved from run to run.) ``append_names`` + report files are not cleared unless ``redo`` is True, and persist over + runs: assuming the convention that [0001] is the index of the deployment, + the ``skip`` attribute on the returned report object contains indexes that + should be skipped. + """ + skip = set() + fails = {} + if not redo: + rr = read_reports(log_dir, append_names) + def build_set(skip, fails, name, fobj): + lines = fobj.read().strip().splitlines() + skip |= set(int(l[1:5]) for l in lines) + fails[name] = len(lines) + fobj.close() + for name in append_names: + build_set(skip, fails, name, getattr(rr, name)) + else: + names += append_names + for name in append_names: + fails[name] = 0 + append_names = () + files = report_files(log_dir, names) + append_files = report_files(log_dir, append_names) + # backup old reports + old_reports = os.path.join(log_dir, "old-reports") + rundir = os.path.join(old_reports, "run") + if not os.path.exists(old_reports): + os.mkdir(old_reports) + else: + util.safe_unlink(rundir) + for f in files: + if os.path.exists(f): + os.rename(f, rundir) + for f in append_files: + if os.path.exists(f): + shutil.copy(f, rundir) + return Report(names + append_names, [open(f, "w") for f in files] + [open(f, "a") for f in append_files], skip, fails) + class NullLogHandler(logging.Handler): """Log handler that doesn't do anything""" def emit(self, record): @@ -117,23 +208,35 @@ class NullLogHandler(logging.Handler): class WizardOptionParser(optparse.OptionParser): """Configures some default user-level options""" + store_help = False def __init__(self, *args, **kwargs): kwargs["add_help_option"] = False + if "store_help" in kwargs: + self.store_help = kwargs["store_help"] + del kwargs["store_help"] optparse.OptionParser.__init__(self, *args, **kwargs) - def parse_all(self, argv): - self.add_option("-h", "--help", action="help", help=optparse.SUPPRESS_HELP) + def parse_all(self, *args, **kwargs): + if self.store_help: + self.add_option("-h", "--help", action="store_true", default=False, dest="help", help=optparse.SUPPRESS_HELP) + else: + self.add_option("-h", "--help", action="help", help=optparse.SUPPRESS_HELP) group = optparse.OptionGroup(self, "Common Options") group.add_option("-v", "--verbose", dest="verbose", action="store_true", - default=boolish(os.getenv("WIZARD_VERBOSE")), help="Turns on verbose output. Environment variable is WIZARD_VERBOSE") + default=boolish(os.getenv("WIZARD_VERBOSE")), help="Turns on verbose output. Envvar is WIZARD_VERBOSE") group.add_option("--debug", dest="debug", action="store_true", - default=boolish(os.getenv("WIZARD_DEBUG")), help="Turns on debugging output. Environment variable is WIZARD_DEBUG") + default=boolish(os.getenv("WIZARD_DEBUG")), help="Turns on debugging output. Envvar is WIZARD_DEBUG") group.add_option("-q", "--quiet", dest="quiet", action="store_true", - default=boolish(os.getenv("WIZARD_QUIET")), help="Turns off output to stdout. Environment variable is WIZARD_QUIET") + default=boolish(os.getenv("WIZARD_QUIET")), help="Turns off output to stdout. Envvar is WIZARD_QUIET") group.add_option("--log-file", dest="log_file", metavar="FILE", default=None, help="Logs verbose output to file") self.add_option_group(group) - options, numeric_args = self.parse_args(argv) - makeLogger(options, numeric_args) + options, numeric_args = self.parse_args(*args, **kwargs) + setup_logger(options, numeric_args) + debug = options.debug + # we're going to process the global --log-dir/--seen dependency here + if hasattr(options, "seen") and hasattr(options, "log_dir"): + if not options.seen and options.log_dir: + options.seen = os.path.join(options.log_dir, "seen.txt") return options, numeric_args class OptionBaton(object): @@ -151,3 +254,7 @@ class OptionBaton(object): """Hands off parameters to option parser""" for key in args: option_parser.add_option(self.store[key]) + +class Error(wizard.Error): + """Base error class for all command errors""" + pass