2 Miscellaneous utility functions and classes.
6 from wizard.util import *
26 from wizard import user
30 Parse the contents of an environment variable as a boolean.
31 This recognizes more values as ``False`` than :func:`bool` would.
42 except (ValueError, TypeError):
43 if val == "No" or val == "no" or val == "false" or val == "False":
47 class ChangeDirectory(object):
49 Context for temporarily changing the working directory.
51 >>> with ChangeDirectory("/tmp"):
55 def __init__(self, dir):
59 self.olddir = os.getcwd()
61 def __exit__(self, *args):
64 class Counter(object):
66 Object for counting different values when you don't know what
67 they are a priori. Supports index access and iteration.
69 >>> counter = Counter()
70 >>> counter.count("foo")
71 >>> print counter["foo"]
76 def count(self, value):
77 """Increments count for ``value``."""
78 self.dict.setdefault(value, 0)
80 def __getitem__(self, key):
83 return self.dict.__iter__()
85 """Returns the max counter value seen."""
86 return max(self.dict.values())
88 """Returns the sum of all counter values."""
89 return sum(self.dict.values())
91 """Returns the keys of counters."""
92 return self.dict.keys()
94 class PipeToLess(object):
96 Context for printing output to a pager. Use this if output
97 is expected to be long.
100 self.proc = subprocess.Popen("less", stdin=subprocess.PIPE)
101 self.old_stdout = sys.stdout
102 sys.stdout = self.proc.stdin
103 def __exit__(self, *args):
105 self.proc.stdin.close()
107 sys.stdout = self.old_stdout
109 class IgnoreKeyboardInterrupts(object):
111 Context for temporarily ignoring keyboard interrupts. Use this
112 if aborting would cause more harm than finishing the job.
115 signal.signal(signal.SIGINT,signal.SIG_IGN)
116 def __exit__(self, *args):
117 signal.signal(signal.SIGINT, signal.default_int_handler)
119 class LockDirectory(object):
121 Context for locking a directory.
123 def __init__(self, lockfile, expiry = 3600):
124 self.lockfile = lockfile
125 self.expiry = expiry # by default an hour
128 for i in range(0, 3):
130 os.open(self.lockfile, os.O_CREAT | os.O_EXCL)
131 open(self.lockfile, "w").write("%d" % os.getpid())
133 if e.errno == errno.EEXIST:
134 # There is a possibility of infinite recursion, but we
135 # expect it to be unlikely, and not harmful if it does happen
136 with LockDirectory(self.lockfile + "_"):
137 # See if we can break the lock
139 pid = open(self.lockfile, "r").read().strip()
140 if not os.path.exists("/proc/%s" % pid):
141 # break the lock, try again
142 logging.warning("Breaking orphaned lock at %s", self.lockfile)
143 os.unlink(self.lockfile)
146 # check if the file is expiry old, if so, break the lock, try again
147 if time.time() - os.stat(self.lockfile).st_mtime > self.expiry:
148 logging.warning("Breaking stale lock at %s", self.lockfile)
149 os.unlink(self.lockfile)
152 if e.errno == errno.ENOENT:
156 # oh hey, it went away; try again
158 raise DirectoryLockedError(os.getcwd())
159 elif e.errno == errno.EACCES:
160 raise PermissionsError(os.getcwd())
163 raise DirectoryLockedError(os.getcwd())
164 def __exit__(self, *args):
166 os.unlink(self.lockfile)
172 Changes a directory, but has special exceptions for certain
178 if e.errno == errno.EACCES:
179 raise PermissionsError()
180 elif e.errno == errno.ENOENT:
181 raise NoSuchDirectoryError()
186 A map function for dictionaries. Only changes values.
188 >>> dictmap(lambda x: x + 2, {'a': 1, 'b': 2})
191 return dict((k,f(v)) for k,v in d.items())
195 A map function for dictionaries that passes key and value.
197 >>> dictkmap(lambda x, y: x + y, {1: 4, 3: 4})
200 return dict((k,f(k,v)) for k,v in d.items())
202 def get_exception_name(output):
204 Reads the traceback from a Python program and grabs the
205 fully qualified exception name.
207 lines = output.split("\n")
210 for line in lines[1:]:
212 if not line: continue
218 return line.partition(':')[0]
221 def get_dir_uid(dir):
222 """Finds the uid of the person who owns this directory."""
223 return os.stat(dir).st_uid
226 """Returns the commit ID of the current Wizard install."""
227 # If you decide to convert this to use wizard.shell, be warned
228 # that there is a circular dependency, so this function would
229 # probably have to live somewhere else, probably wizard.git
230 wizard_git = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".git")
231 return subprocess.Popen(["git", "--git-dir=" + wizard_git, "rev-parse", "HEAD"], stdout=subprocess.PIPE).communicate()[0].rstrip()
233 def get_operator_git():
235 Returns ``Real Name <username@mit.edu>`` suitable for use in
236 Git ``Something-by:`` string. Throws :exc:`NoOperatorInfo` if
237 no operator information is available.
242 info = user.pwnam(op)
243 return "%s <%s>" % (info.realname, info.email)
245 def set_operator_env():
247 Sets :envvar:`GIT_COMMITTER_NAME` and :envvar:`GIT_COMMITTER_EMAIL`
248 environment variables if applicable. Does nothing if no information
254 info = user.pwnam(op)
255 os.putenv("GIT_COMMITTER_NAME", info.realname)
256 os.putenv("GIT_COMMITTER_EMAIL", info.email)
258 def set_author_env():
260 Sets :envvar:`GIT_AUTHOR_NAME` and :envvar:`GIT_AUTHOR_EMAIL`
261 environment variables if applicable. Does nothing if
262 :func:`wizard.user.passwd` fails.
267 os.putenv("GIT_AUTHOR_NAME", "%s" % info.realname)
268 os.putenv("GIT_AUTHOR_EMAIL", "%s" % info.email)
271 """Sets all appropriate environment variables for Git commits."""
275 def get_git_footer():
276 """Returns strings for placing in Git log info about Wizard."""
277 return "\n".join(["Wizard-revision: %s" % get_revision()
278 ,"Wizard-args: %s" % " ".join(sys.argv)
281 def safe_unlink(file):
282 """Moves a file/dir to a backup location."""
283 if not os.path.lexists(file):
285 prefix = "%s.bak" % file
287 for i in itertools.count():
288 name = "%s.%d" % (prefix, i)
289 if not os.path.lexists(name):
291 os.rename(file, name)
294 def soft_unlink(file):
295 """Unlink a file, but don't complain if it doesn't exist."""
303 Create a directory path (a la ``mkdir -p`` or ``os.makedirs``),
304 but don't complain if it already exists.
308 except OSError as exc:
309 if exc.errno == errno.EEXIST:
314 def fetch(host, path, subpath, post=None):
316 # XXX: Should use urllib instead
317 h = httplib.HTTPConnection(host)
318 fullpath = path.rstrip("/") + "/" + subpath.lstrip("/") # to be lenient about input we accept
320 headers = {"Content-type": "application/x-www-form-urlencoded"}
321 logging.info("POST request to http://%s%s", host, fullpath)
322 logging.debug("POST contents:\n" + urllib.urlencode(post))
323 h.request("POST", fullpath, urllib.urlencode(post), headers)
325 logging.info("GET request to http://%s%s", host, fullpath)
326 h.request("GET", fullpath)
331 except socket.gaierror as e:
332 if e.errno == socket.EAI_NONAME:
337 def mixed_newlines(filename):
338 """Returns ``True`` if ``filename`` has mixed newlines."""
339 f = open(filename, "U") # requires universal newline support
341 ret = isinstance(f.newlines, tuple)
342 f.close() # just to be safe
345 def disk_usage(dir=None, excluded_dir=".git"):
347 Recursively determines the disk usage of a directory, excluding
348 .git directories. Value is in bytes. If ``dir`` is omitted, the
349 current working directory is assumed.
351 if dir is None: dir = os.getcwd()
353 for root, _, files in os.walk(dir):
355 if not os.path.join(root, name).startswith(os.path.join(dir, excluded_dir)):
356 file = os.path.join(root, name)
358 if os.path.islink(file): continue
359 sum_sizes += os.path.getsize(file)
361 if e.errno == errno.ENOENT:
362 logging.warning("%s disappeared before we could stat", file)
367 def random_key(length=30):
368 """Generates a random alphanumeric key of ``length`` size."""
369 return ''.join(random.choice(string.letters + string.digits) for i in xrange(length))
371 def truncate(version):
372 """Truncates the Scripts specific version number."""
373 return str(version).partition('-scripts')[0]
375 def init_wizard_dir():
377 Generates a .wizard directory and initializes it with some common
378 files. This operation is idempotent.
380 # no harm in doing this repeatedly
381 wizard_dir = ".wizard"
382 if not os.path.isdir(wizard_dir):
384 open(os.path.join(wizard_dir, ".htaccess"), "w").write("Deny from all\n")
385 open(os.path.join(wizard_dir, ".gitignore"), "w").write("*\n")
387 class NoOperatorInfo(wizard.Error):
388 """No information could be found about the operator from Kerberos."""
391 class PermissionsError(IOError):
394 class NoSuchDirectoryError(IOError):
397 class DirectoryLockedError(wizard.Error):
398 def __init__(self, dir):
403 ERROR: Could not acquire lock on directory. Maybe there is
404 another migration process running?
407 class DNSError(socket.gaierror):
408 errno = socket.EAI_NONAME
409 #: Hostname that could not resolve name
411 def __init__(self, host):
416 ERROR: Could not resolve hostname %s.