bin/install-statistics

   1 #!/usr/bin/env python
   2
   3 """
   4 This script generates basic statistics about our autoinstalls.
   5 """
   6
   7 import os
   8 import optparse
   9 import fileinput
  10 import math
  11 import sys
  12 from distutils.version import LooseVersion as Version
  13
  14 class NoSuchApplication(Exception):
  15     pass
  16
  17 class DeploymentParseError(Exception):
  18     pass
  19
  20 class Deployment(object):
  21     def __init__(self, location, version):
  22         self.location = location
  23         self.version = version
  24         self.application = version.application
  25     @staticmethod
  26     def parse(line):
  27         """Parses a line from the results of parallel-find.pl.
  28         This will work out of the box with fileinput"""
  29         try:
  30             location, deploydir = line.rstrip().split(":")
  31         except ValueError:
  32             raise DeploymentParseError
  33         name = deploydir.split("/")[-1]
  34         if name.find("-") != -1:
  35             app, version = name.split("-")
  36         elif name == "deploy":
  37             # Assume that it's django, since those were botched
  38             app = "django"
  39             version = "0.1-scripts"
  40         else:
  41             raise DeploymentParseError
  42         try:
  43             return Deployment(location, applications[app].getVersion(version))
  44         except KeyError:
  45             raise NoSuchApplication
  46     def count(self):
  47         """Simple method which registers the deployment as a +1 on the
  48         appropriate version. No further inspection is done."""
  49         self.version.count(self)
  50         return True
  51     def count_exists(self, file):
  52         """Checks if the codebase has a certain file/directory in it."""
  53         if os.path.exists(self.location + "/" + file):
  54             self.version.count_exists(self, file)
  55             return True
  56         return False
  57
  58 class Application(object):
  59     HISTOGRAM_WIDTH = 30
  60     def __init__(self, name):
  61         self.name = name
  62         self.versions = {}
  63         # Some cache variables for fast access of calculated data
  64         self._total = 0
  65         self._max   = 0
  66         self._c_exists = {}
  67     def getVersion(self, version):
  68         if version not in self.versions:
  69             self.versions[version] = ApplicationVersion(Version(version), self)
  70         return self.versions[version]
  71     def _graph(self, v):
  72         return '+' * int(math.ceil(float(v)/self._max * self.HISTOGRAM_WIDTH))
  73     def __str__(self):
  74         if not self.versions: return "%-11s   no installs" % self.name
  75         ret = \
  76             ["%-16s %3d installs" % (self.name, self._total)] + \
  77             [str(v) for v in sorted(self.versions.values())]
  78         for f,c in self._c_exists.items():
  79             ret.append("%d users have %s" % (c,f))
  80         return "\n".join(ret)
  81
  82 class ApplicationVersion(object):
  83     def __init__(self, version, application):
  84         self.version = version
  85         self.application = application
  86         self.c = 0
  87         self.c_exists = {}
  88     def __cmp__(x, y):
  89         return cmp(x.version, y.version)
  90     def count(self, deployment):
  91         self.c += 1
  92         self.application._total += 1
  93         if self.c > self.application._max:
  94             self.application._max = self.c
  95     def count_exists(self, deployment, n):
  96         if n in self.c_exists: self.c_exists[n] += 1
  97         else: self.c_exists[n] = 1
  98         if n in self.application._c_exists: self.application._c_exists[n] += 1
  99         else: self.application._c_exists[n] = 1
 100     def __str__(self):
 101         return "    %-12s %3d  %s" \
 102             % (self.version, self.c, self.application._graph(self.c))
 103
 104 application_list = [
 105     "mediawiki", "wordpress", "joomla", "e107", "gallery2",
 106     "phpBB", "advancedbook", "phpical", "trac", "turbogears", "django",
 107     # these are technically deprecated
 108     "advancedpoll", "gallery",
 109 ]
 110
 111 """Hash table for looking up string application name to instance"""
 112 applications = dict([(n,Application(n)) for n in application_list ])
 113
 114 def main():
 115     usage = """usage: %prog [options] [application]
 116
 117 Scans all of the collected data from parallel-find.pl, and
 118 determines version histograms for our applications.  You may
 119 optionally pass application parameters to filter the installs."""
 120     parser = optparse.OptionParser(usage)
 121     parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
 122             default=False, help="Print interesting directories")
 123     parser.add_option("-q", "--quiet", dest="quiet", action="store_true",
 124             default=False, help="Suppresses progress output")
 125     parser.add_option("-d", "--version-dir", dest="version_dir",
 126             default="/afs/athena.mit.edu/contrib/scripts/sec-tools/store/versions",
 127             help="Location of parallel-find output")
 128     parser.add_option("--count-exists", dest="count_exists",
 129             default=False, help="Count deployments that contain a file")
 130     # There should be machine friendly output
 131     options, show = parser.parse_args()
 132     if not show: show = applications.keys()
 133     show = frozenset(show)
 134     vd = options.version_dir
 135     try:
 136         fi = fileinput.input([vd + "/" + f for f in os.listdir(vd)])
 137     except OSError:
 138         print "No permissions; check if AFS is mounted"
 139         raise SystemExit(-1)
 140     errors = 0
 141     unrecognized = 0
 142     processed = 0
 143     # I really don't like this boolean
 144     hanging = False # whether or not we last outputted a newline
 145     if not options.quiet: print "Processing",
 146     for line in fi:
 147         processed += 1
 148         if not options.quiet and processed % 10 == 0:
 149             sys.stdout.write(".")
 150             sys.stdout.flush()
 151             hanging = True
 152         try:
 153             deploy = Deployment.parse(line)
 154         except DeploymentParseError:
 155             errors += 1
 156             continue
 157         except NoSuchApplication:
 158             unrecognized += 1
 159             continue
 160         if deploy.application.name + "-" + str(deploy.version.version) in show:
 161             if hanging:
 162                 hanging = False
 163                 print
 164             print "%s-%s deployment at %s" \
 165                 % (deploy.application.name, deploy.version.version, deploy.location)
 166         elif deploy.application.name in show:
 167             pass
 168         else:
 169             continue
 170         deploy.count()
 171         if options.count_exists:
 172             r = deploy.count_exists(options.count_exists)
 173             if r and options.verbose:
 174                 if hanging:
 175                     hanging = False
 176                     print
 177                 print "Found " + options.count_exists + " in " + deploy.location
 178     if hanging: print
 179     print
 180     for app in applications.values():
 181         if app.name not in show: continue
 182         print app
 183         print
 184     print "With %d errors and %d unrecognized applications" % (errors, unrecognized)
 185
 186 if __name__ == "__main__":
 187     main()
 188