From a0587472768ec8c72baa48708a4c3519814aa0f0 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Thu, 5 Nov 2009 13:49:29 -0500 Subject: [PATCH] Revamp 'wizard summary', and start writing tutorial docs. Signed-off-by: Edward Z. Yang --- TODO | 2 + doc/index.rst | 12 ++-- doc/repository-conversion.rst | 96 ++++++++++++++++++++++++++++++- wizard/app/__init__.py | 6 ++ wizard/command/summary/version.py | 43 +++++++++----- wizard/util.py | 9 +++ 6 files changed, 145 insertions(+), 23 deletions(-) diff --git a/TODO b/TODO index a99f048..dda76d0 100644 --- a/TODO +++ b/TODO @@ -126,6 +126,8 @@ OVERALL PLAN: X. Commit, with name "Appname x.y.z" + X. Tag as appname-x.y.z + 4. Checkout the master branch 5. Merge the pristine branch in. Resolve any conflicts that our diff --git a/doc/index.rst b/doc/index.rst index 3253239..5da42e8 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,10 +1,9 @@ Wizard ====== -Wizard is the next-generation autoinstall management system for -scripts.mit.edu. It's current focus is on automating the upgrading -process using a Git backend, and in the future will administrate all -aspects of autoinstalls (installation and upgrades). +Wizard is the next-generation autoinstall management system. It +is currently being developed as an in-house tool for scripts.mit.edu, +with a focus on automating the upgrading process using a Git backend. Ultimately, we would like to see Wizard become a general purpose web application package manager, with first class support for a variety @@ -25,8 +24,7 @@ into a scripts server and add Wizard to your path:: Individual tools that are not scripts-specific, such as ``wizard upgrade`` or ``wizard install``, can be run locally with Python 2.6 and a reasonably -new version of Git. You will, however, have to explicitly specify configuration -appropriate for your local machine. +new version of Git. .. highlight:: python @@ -37,7 +35,7 @@ The canonical source of the Wizard source code is the directory :file:`/mit/scripts/git/wizard.git` on AFS. The live version of the source code lives at :file:`/mit/scripts/wizard` -and should be periodically updated as necessary (you will +and should be periodically updated as necessary (use the ``pull.sh`` script; you will need scripts-root bits to do so). Documentation lives in :file:`/mit/scripts/web_scripts/home/wizard`; the post-merge hook on this Git repository should perform the appropriate rebuild. diff --git a/doc/repository-conversion.rst b/doc/repository-conversion.rst index 5e60162..5c529de 100644 --- a/doc/repository-conversion.rst +++ b/doc/repository-conversion.rst @@ -12,5 +12,99 @@ of a php.ini file and appropriate symlinks). Conversion to Wizard involves placing pristine versions of the source code (from the upstream tarballs) and appropriately patched scripts -versions into a Git repository. +versions into a Git repository, as well as writing a :mod:`wizard.app` +module for the application that defines common operations. +Here is a tutorial for performing a conversion, using Wordpress as +an example. + +Setup +----- + +.. highlight:: sh + +Probably the easiest way to do development is entirely on AFS: all +of your source code should live in publically readable (i.e. +system:anyuser as read permissions) directories, so that if you +SSH into a scripts server to perform testing, you will be able +to invoke your tools and read your development repository. In that +case, setup is as simple as:: + + git clone /mit/scripts/git/wizard.git /mit/$USER/wizard + athrun consult fsr /mit/$USER/wizard system:anyuser read + # for any application you're going to do development on, also: + git clone /mit/scripts/git/autoinstalls/$APP.git ~/wizard/srv/$APP + +If you'd like to be able to develop offline, just note that you will +have to push your changes to AFS once you start doing testing on +scripts servers, but before your changes get incorporated into +canonical upstream. Git doesn't exactly make this easy, but you +can follow this recipe:: + + git clone /mit/scripts/git/wizard.git ~/wizard + cd /mit/$USER + mkdir wizard.git + cd wizard.git + git init --bare + cd ~/wizard + git remote add afs /mit/$USER/wizard.git + git push -f afs master + git clone /mit/$USER/wizard.git /mit/$USER/wizard + +And then you can perform updates from your local copy with:: + + git push afs master + cd /mit/$USER/wizard + git pull + +If ``/mit/$USER/wizard.git`` has write permissions for +``system:scripts-security-upd``, this is especially useful if you were hacking +on a copy living on ``not-backward.mit.edu``, and now need to transfer the +changes back to the canonical repository (please don't give ``not-backward.mit.edu`` +your root tickets!) + +From this point on, we will assume you are doing development from an AFS directory +named ``$WIZARD``; note that application repositories live in ``$WIZARD/srv``. + +Pristine +-------- + +.. highlight:: sh + +This is a tutorial for migrating Wordpress into our Git repository. As such +a repository doesn't exist, we should create it:: + + cd "$WIZARD/srv" + mkdir wordpress + cd wordpress + git init + +.. highlight:: python + +We also have to create a module for the application, so we +create ``$WIZARD/wizard/app/wordpress.py`` and fill it in with a bare bones template:: + + from wizard import app + class Application(app.Application): + pass + +.. highlight:: sh + +Now we are ready to put some code in our repository: the first thing we will +add is the "pristine branch", which contains verbatim the code from upstream. +If we were starting a new autoinstaller, we'd pop off and use the latest version, +but since we're dealing with legacy we want to start our repository history +with the **oldest** version still extant on our servers. To find this out run:: + + wizard summary version APP + +You'll need to be in the ``scripts-team`` list in order to access the data to +run this command. + +Try running the following command in "$WIZARD/srv/wordpress":: + + wizard prepare-pristine wordpress-2.0.2 + +You should get an error complaining about ``download()`` not being implemented yet. + +(to be continued) diff --git a/wizard/app/__init__.py b/wizard/app/__init__.py index 8395439..be8ba55 100644 --- a/wizard/app/__init__.py +++ b/wizard/app/__init__.py @@ -222,6 +222,12 @@ class Application(object): should provide an implementation. """ raise NotImplemented + def download(self, version): + """ + Returns a URL that can be used to download a tarball of ``version`` of + this application. + """ + raise NotImplemented def checkWeb(self, deployment, output=None): """ Checks if the autoinstall is viewable from the web. To get diff --git a/wizard/command/summary/version.py b/wizard/command/summary/version.py index b14b184..c0fd1b7 100644 --- a/wizard/command/summary/version.py +++ b/wizard/command/summary/version.py @@ -1,29 +1,42 @@ import math +import distutils.version -from wizard import command, deploy, util +from wizard import app, command, deploy, util def main(argv, baton): - options, show = parse_args(argv, baton) + options, str_show = parse_args(argv, baton) HISTOGRAM_WIDTH = 30 - show = set() - c_version = util.Counter() - c_application = util.Counter() - for d in deploy.parse_install_lines(show, options.versions_path): - version = d.app_version - c_version.count(version) - c_application.count(version.application) - show.add(version.application) + if str_show: + apps = app.applications() + show = set(apps[x] for x in str_show) + accumulate = False + else: + str_show = [] + show = set() + accumulate = True + c_application = {} + for d in deploy.parse_install_lines(str_show, options.versions_path): + c_application.setdefault(d.application, util.Counter()) + version = truncate(d.app_version.version) + c_application[d.application].count(version) + if accumulate: + show.add(d.application) if not show: print "No applications found" for application in show: - print "%-20s %3d installs" % (application.name, c_application[application]) - vmax = max(c_version[x] for x in application.versions.values()) - for version in sorted(application.versions.values()): - v = c_version[version] + counter = c_application[application] + total = counter.sum() + print "%-20s %3d installs" % (application.name, total) + vmax = counter.max() + for version in sorted(counter.keys(), key=distutils.version.LooseVersion): + v = counter[version] graph = '+' * int(math.ceil(float(v)/vmax * HISTOGRAM_WIDTH)) - print " %-16s %3d %s" % (version.version, v, graph) + print " %-16s %3d %s" % (version, v, graph) print +def truncate(version): + return str(version).partition('-scripts')[0] + def parse_args(argv, baton): usage = """usage: %prog summary version [ARGS] [APP] diff --git a/wizard/util.py b/wizard/util.py index 3aa0678..0cd8fc8 100644 --- a/wizard/util.py +++ b/wizard/util.py @@ -57,6 +57,15 @@ class Counter(object): return self.dict[key] def __iter__(self): return self.dict.__iter__() + def max(self): + """Returns the max counter value seen.""" + return max(self.dict.values()) + def sum(self): + """Returns the sum of all counter values.""" + return sum(self.dict.values()) + def keys(self): + """Returns the keys of counters.""" + return self.dict.keys() class PipeToLess(object): """ -- 2.45.0