2 Miscellaneous utility functions and classes.
6 from wizard.util import *
26 from wizard import user
30 Parse the contents of an environment variable as a boolean.
31 This recognizes more values as ``False`` than :func:`bool` would.
42 except (ValueError, TypeError):
43 if val == "No" or val == "no" or val == "false" or val == "False":
47 class ChangeDirectory(object):
49 Context for temporarily changing the working directory.
51 >>> with ChangeDirectory("/tmp"):
55 def __init__(self, dir):
59 self.olddir = os.getcwd()
61 def __exit__(self, *args):
64 class Counter(object):
66 Object for counting different values when you don't know what
67 they are a priori. Supports index access and iteration.
69 >>> counter = Counter()
70 >>> counter.count("foo")
71 >>> print counter["foo"]
76 def count(self, value):
77 """Increments count for ``value``."""
78 self.dict.setdefault(value, 0)
80 def __getitem__(self, key):
83 return self.dict.__iter__()
85 """Returns the max counter value seen."""
86 return max(self.dict.values())
88 """Returns the sum of all counter values."""
89 return sum(self.dict.values())
91 """Returns the keys of counters."""
92 return self.dict.keys()
94 class PipeToLess(object):
96 Context for printing output to a pager. Use this if output
97 is expected to be long.
100 self.proc = subprocess.Popen("less", stdin=subprocess.PIPE)
101 self.old_stdout = sys.stdout
102 sys.stdout = self.proc.stdin
103 def __exit__(self, *args):
105 self.proc.stdin.close()
107 sys.stdout = self.old_stdout
109 class IgnoreKeyboardInterrupts(object):
111 Context for temporarily ignoring keyboard interrupts. Use this
112 if aborting would cause more harm than finishing the job.
115 signal.signal(signal.SIGINT,signal.SIG_IGN)
116 def __exit__(self, *args):
117 signal.signal(signal.SIGINT, signal.default_int_handler)
119 class LockDirectory(object):
121 Context for locking a directory.
123 def __init__(self, lockfile, expiry = 3600):
124 self.lockfile = lockfile
125 self.expiry = expiry # by default an hour
128 for i in range(0, 3):
130 os.open(self.lockfile, os.O_CREAT | os.O_EXCL)
131 open(self.lockfile, "w").write("%d" % os.getpid())
133 if e.errno == errno.EEXIST:
134 # There is a possibility of infinite recursion, but we
135 # expect it to be unlikely, and not harmful if it does happen
136 with LockDirectory(self.lockfile + "_"):
137 # See if we can break the lock
139 pid = open(self.lockfile, "r").read().strip()
140 if not os.path.exists("/proc/%s" % pid):
141 # break the lock, try again
142 logging.warning("Breaking orphaned lock at %s", self.lockfile)
143 os.unlink(self.lockfile)
146 # check if the file is expiry old, if so, break the lock, try again
147 if time.time() - os.stat(self.lockfile).st_mtime > self.expiry:
148 logging.warning("Breaking stale lock at %s", self.lockfile)
149 os.unlink(self.lockfile)
152 if e.errno == errno.ENOENT:
156 # oh hey, it went away; try again
158 raise DirectoryLockedError(os.getcwd())
159 elif e.errno == errno.EACCES:
160 raise PermissionsError(os.getcwd())
163 raise DirectoryLockedError(os.getcwd())
164 def __exit__(self, *args):
166 os.unlink(self.lockfile)
172 Changes a directory, but has special exceptions for certain
178 if e.errno == errno.EACCES:
179 raise PermissionsError()
180 elif e.errno == errno.ENOENT:
181 raise NoSuchDirectoryError()
186 A map function for dictionaries. Only changes values.
188 >>> dictmap(lambda x: x + 2, {'a': 1, 'b': 2})
191 return dict((k,f(v)) for k,v in d.items())
195 A map function for dictionaries that passes key and value.
197 >>> dictkmap(lambda x, y: x + y, {1: 4, 3: 4})
200 return dict((k,f(k,v)) for k,v in d.items())
202 def get_exception_name(output):
204 Reads the traceback from a Python program and grabs the
205 fully qualified exception name.
207 lines = output.split("\n")
210 for line in lines[1:]:
212 if not line: continue
218 return line.partition(':')[0]
221 def get_dir_uid(dir):
222 """Finds the uid of the person who owns this directory."""
223 return os.stat(dir).st_uid
226 """Returns the commit ID of the current Wizard install."""
227 # If you decide to convert this to use wizard.shell, be warned
228 # that there is a circular dependency, so this function would
229 # probably have to live somewhere else, probably wizard.git
230 wizard_git = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".git")
231 return subprocess.Popen(["git", "--git-dir=" + wizard_git, "rev-parse", "HEAD"], stdout=subprocess.PIPE).communicate()[0].rstrip()
233 def get_operator_git():
235 Returns ``Real Name <username@mit.edu>`` suitable for use in
236 Git ``Something-by:`` string. Throws :exc:`NoOperatorInfo` if
237 no operator information is available.
242 info = user.pwnam(op)
243 return "%s <%s>" % (info.realname, info.email)
245 def set_operator_env():
247 Sets :envvar:`GIT_COMMITTER_NAME` and :envvar:`GIT_COMMITTER_EMAIL`
248 environment variables if applicable. Does nothing if no information
254 info = user.pwnam(op)
255 if not info.realname:
257 os.putenv("GIT_COMMITTER_NAME", info.realname)
258 os.putenv("GIT_COMMITTER_EMAIL", info.email)
260 def set_author_env():
262 Sets :envvar:`GIT_AUTHOR_NAME` and :envvar:`GIT_AUTHOR_EMAIL`
263 environment variables if applicable. Does nothing if
264 :func:`wizard.user.passwd` fails.
269 if not info.realname:
271 os.putenv("GIT_AUTHOR_NAME", "%s" % info.realname)
272 os.putenv("GIT_AUTHOR_EMAIL", "%s" % info.email)
275 """Sets all appropriate environment variables for Git commits."""
279 def get_git_footer():
280 """Returns strings for placing in Git log info about Wizard."""
281 return "\n".join(["Wizard-revision: %s" % get_revision()
282 ,"Wizard-args: %s" % " ".join(sys.argv)
285 def safe_unlink(file):
286 """Moves a file/dir to a backup location."""
287 if not os.path.lexists(file):
289 prefix = "%s.bak" % file
291 for i in itertools.count():
292 name = "%s.%d" % (prefix, i)
293 if not os.path.lexists(name):
295 os.rename(file, name)
298 def soft_unlink(file):
299 """Unlink a file, but don't complain if it doesn't exist."""
307 Create a directory path (a la ``mkdir -p`` or ``os.makedirs``),
308 but don't complain if it already exists.
312 except OSError as exc:
313 if exc.errno == errno.EEXIST:
318 def fetch(host, path, subpath, post=None):
320 # XXX: Should use urllib instead
321 h = httplib.HTTPConnection(host)
322 fullpath = path.rstrip("/") + "/" + subpath.lstrip("/") # to be lenient about input we accept
324 headers = {"Content-type": "application/x-www-form-urlencoded"}
325 logging.info("POST request to http://%s%s", host, fullpath)
326 logging.debug("POST contents:\n" + urllib.urlencode(post))
327 h.request("POST", fullpath, urllib.urlencode(post), headers)
329 logging.info("GET request to http://%s%s", host, fullpath)
330 h.request("GET", fullpath)
332 logging.debug("Response code: %d", r.status)
333 logging.debug("Response headers: %s", r.msg)
337 except socket.gaierror as e:
338 if e.errno == socket.EAI_NONAME:
343 def mixed_newlines(filename):
344 """Returns ``True`` if ``filename`` has mixed newlines."""
345 f = open(filename, "U") # requires universal newline support
347 ret = isinstance(f.newlines, tuple)
348 f.close() # just to be safe
351 def disk_usage(dir=None, excluded_dir=".git"):
353 Recursively determines the disk usage of a directory, excluding
354 .git directories. Value is in bytes. If ``dir`` is omitted, the
355 current working directory is assumed.
357 if dir is None: dir = os.getcwd()
359 for root, _, files in os.walk(dir):
361 if not os.path.join(root, name).startswith(os.path.join(dir, excluded_dir)):
362 file = os.path.join(root, name)
364 if os.path.islink(file): continue
365 sum_sizes += os.path.getsize(file)
367 if e.errno == errno.ENOENT:
368 logging.warning("%s disappeared before we could stat", file)
373 def random_key(length=30):
374 """Generates a random alphanumeric key of ``length`` size."""
375 return ''.join(random.choice(string.letters + string.digits) for i in xrange(length))
377 def truncate(version):
378 """Truncates the Scripts specific version number."""
379 return str(version).partition('-scripts')[0]
381 def init_wizard_dir():
383 Generates a .wizard directory and initializes it with some common
384 files. This operation is idempotent.
386 # no harm in doing this repeatedly
387 wizard_dir = ".wizard"
388 if not os.path.isdir(wizard_dir):
390 open(os.path.join(wizard_dir, ".htaccess"), "w").write("Deny from all\n")
391 open(os.path.join(wizard_dir, ".gitignore"), "w").write("*\n")
393 class NoOperatorInfo(wizard.Error):
394 """No information could be found about the operator from Kerberos."""
397 class PermissionsError(IOError):
400 class NoSuchDirectoryError(IOError):
403 class DirectoryLockedError(wizard.Error):
404 def __init__(self, dir):
409 ERROR: Could not acquire lock on directory. Maybe there is
410 another migration process running?
413 class DNSError(socket.gaierror):
414 errno = socket.EAI_NONAME
415 #: Hostname that could not resolve name
417 def __init__(self, host):
422 ERROR: Could not resolve hostname %s.