From 3f3c8a7ec822d73f7af28b6b7ef94f3476863527 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Mon, 15 Jun 2009 01:51:00 -0400 Subject: [PATCH] Implement migration script, update TODO with new things to do. Signed-off-by: Edward Z. Yang --- TODO | 129 +++++++++++++++++----------------- lib/wizard/command/migrate.py | 92 +++++++++++++++++++++++- lib/wizard/shell.py | 24 +++++++ 3 files changed, 177 insertions(+), 68 deletions(-) create mode 100644 lib/wizard/shell.py diff --git a/TODO b/TODO index 36bcead..fbcb2cb 100644 --- a/TODO +++ b/TODO @@ -2,8 +2,18 @@ The Git Autoinstaller TODO NOW: -* Migrate and upgrade the lone mediawiki-1.5.6 install -* Create the migration script +- Fix mediawiki repository (has lots of stale files and a weird + history. This also means that the ichuang wiki will need to be + remigrated; frob .scripts-version to make it acceptable, but + otherwise should be fairly trivial). When I say "fix", I mean + "redo". Thorough documentation would be good too. Some bits + can be automated and/or have tools to assist it +- Whiteboard the flow for performing an upgrade on a single + install. How assisted does it need to be? +- Conduct migration tool testing +- Create mass-migration tool (should be able to limit on mediawiki) +- Run parallel-find.pl +- Migrate all mediawikis OVERALL PLAN: @@ -19,64 +29,69 @@ OVERALL PLAN: nice, but is priority. For the long term, seeing this scripts be packaged with rest of our code would be optimal. -* The new procedure for generating an update is as follows: +* The new procedure for generating an update is as follows (this is + also similar to procedure for creating these repositories): 1. Have the Git repository and working copy for the project on hand. - 2. Download the new tarball + 2. Checkout the pristine branch - 3. Extract the tarball over the working copy (`cp -R a/. b` works well) + 3. Remove all files from the working copy (rm -Rf *, and then delete + any dot stragglers. A script to do this would be handy) - 4. Check if there are any special update procedures, and update the - .scripts/update shell script as necessary (this means that any - application specific update logic will be kept with the actual - source code. The language of this update script will vary - depending on context.) + 4. Download the new tarball - X. Check for empty directories and add stub files as necessary - (use preserve-empty-dir) - - 5. Commit your changes, and tag as v1.2.3-scripts + 5. Extract the tarball over the working copy (`cp -R a/. b` works well, + remember that the working copy is empty) - 6. Run the "dry-run script", which uses Git commands to check how many - working copies apply the change cleanly, and writes out a logfile - with the working copies that don't apply cleanly. + 6. Check for empty directories and add stub files as necessary + (use preserve-empty-dir) - 7. Run the "limited run" script, which applies the update to our - test-bed, and lets us check the basic functionality of the update. + 7. Git add it all, and then commit as a new pristine version (v1.2.3) - 8. Run the "deploy" script, which applies the update to all working - copies possible, and sends mail to users to whom the working copy - did not apply cleanly. (It also frobs .scripts/version) + 8. Checkout the master branch - Note: The last three scripts will need to be implemented, with an - eye towards speed. + 9. [FOR EXISTING REPOSITORIES] + Merge the pristine branch in. Resolve any conflicts that our + patches have with new changes. Do NOT let Git auto-commit it + with --no-commit (otherwise, you want to git commit --amend + to keep our history clean -* How to migrate an old autoinstaller to the new autoinstaller + [FOR THE FIRST TIME] + Apply the scripts patch that was used for that version here + (usually patch -p1 < patch) - - Find the oldest tarball/patch set for the application that still - is in use and upgradable. + 10. Check if there are any special update procedures, and update the + .scripts/update shell script as necessary (this means that any + application specific update logic will be kept with the actual + source code. The language of this update script will vary + depending on context.) - - Untar, apply patch, place in a directory (unfurl) and git init + 11. Commit your changes, and tag as v1.2.3-scripts - - Commit this as the "pristine" version (branch "pristine") + If you're setting up a repository from scratch, stop here, and + repeat as necessary - - Apply scripts patches + XXX: Should we force people to push to the real repository at + this point, or just make the repository that the script pulls + stuff out of configurable? (Twiddling origin can get you a + devel setup with no code changes) - - Create the .scripts directory and populate it with the interesting - information (see below) + 12. Run the "dry-run script", which uses Git commands to check how many + working copies apply the change cleanly, and writes out a logfile + with the working copies that don't apply cleanly. - - Commit this as the "scripts" version (branch "master") + 13. Run the "limited run" script, which applies the update to our + test-bed, and lets us check the basic functionality of the update. + This can include a script that lets us update a single directory + with verbose output. -* How to update the autoinstaller repository + 14. Run the "deploy" script, which applies the update to all working + copies possible, and sends mail to users to whom the working copy + did not apply cleanly. It also frobs .scripts/version for successful + upgrades. - - Checkout pristine branch - - Delete contents of pristine branch (excluding .git, try rm -Rf * and remove stragglers) - - Unfurl tarball - - Commit as new pristine branch - - Checkout scripts branch - - Merge pristine branch - - Fix as necessary + 15. Run parallel-find.pl * The repository for a given application will contain the following files: @@ -99,32 +114,14 @@ OVERALL PLAN: the script is) (This is the same as .scripts-version right now; probably want to keep that for now) - - Because there will be no .gitignore file, you *must not* run - `git add .` on an actual running copy of the application. - `git add -u .` will generally be safe, but preferred mode - of operation is to operate on a clean install. - -* The migration process shall be as such: - - 1. git init - - 2. git remote add origin /foo - - 3. git config branch.master.merge refs/heads/master - - 4. git fetch origin - - 5. git reset v1.2.3-scripts - - 5. git checkout .scripts + XXX: It's unclear if we want to move to this wholesale, or + delay this indefinitely. - 6. Setup .scripts/version (probably pipe the output of real-version) - UNCLEAR if this is a good thing; if it is, make sure we add - a .gitignore to the .scripts directory +* The migration process has been implemented, see 'wizard migrate'. -* We will not add special code to handle .htaccess; thus the kernel patch - for allowing Apache access to .htaccess sent to scripts-team@mit.edu - must be handled first. + XXX: We have not decided what migration should do to .scripts-version; + if it does move it to .scripts, repositories should have a .gitignore + in those directories * The autoupgrade shall be the process of: @@ -138,5 +135,5 @@ OVERALL PLAN: (with some more robust error checking) -* Make install-statistics generate nice pretty graphs of installs by date +* Make 'wizard summary' generate nice pretty graphs of installs by date (more histograms, will need to check actual .scripts-version files.) diff --git a/lib/wizard/command/migrate.py b/lib/wizard/command/migrate.py index f626322..fcf1247 100644 --- a/lib/wizard/command/migrate.py +++ b/lib/wizard/command/migrate.py @@ -1,7 +1,10 @@ import optparse import sys +import os +import shutil import wizard.deploy as wd +import wizard.shell as sh def main(argv, global_options): usage = """usage: %prog migrate [ARGS] DIR @@ -22,5 +25,90 @@ NOTICE: Currently doesn't do anything.""" elif not args: parser.error("must specify directory") dir = args[0] - print dir - print options + print "Changing working directory to autoinstall directory" + try: + os.chdir(dir) + except OSError as e: + if e.errno == 13: + print + print "ERROR: You don't have permissions to access this directory." + print "Do you have tickets for AFS with your root instance, and" + print "is your root instance on scripts-security-upd?" + print + print "You can check by running the commands 'klist' and" + print "'blanche scripts-security-upd'. We recommend getting" + print "root tickets using Nelson Elhage's krbroot script" + print "at /mit/nelhage/Public/krbroot (for which you run" + print "'krbroot shell' and then 'aklog')." + raise SystemExit(-1) + elif e.errno == 2: + print + print "ERROR: No such directory... check your typing" + raise SystemExit(-1) + else: raise e + try: + deploy = wd.Deployment.fromDir(".") + version = deploy.getAppVersion() + except IOError as e: + if e.errno == 2: + print + print "ERROR: Could not find .scripts-version file. Are you sure" + print "this is an autoinstalled application?" + raise SystemExit(-1) + else: raise e + # calculate the repository we'll be pulling out of + application = version.application + app = application.name + repo = os.path.join("/afs/athena.mit.edu/contrib/scripts/wizard/srv", app + ".git") + if not os.path.isdir(repo): + print + print "ERROR: Could not find repository for this application. Have" + print "you converted the repository over? Is the name %s" % app + print "the same as the the name of the foo.git folder?" + raise SystemExit(-1) + # begin the command line process + shell = sh.Shell(options.verbose, options.dry_run) + # check if the version we're trying to convert exists. We assume + # a convention here, namely, v1.2.3-scripts is what we want. If + # you broke the convention... shame on you. + try: + tag = "v%s-scripts" % version.version + shell.call("git", "--git-dir", repo, "rev-parse", tag) + except sh.CalledProcessError: + print + print "ERROR: Could not find tag v%s-scripts for" % version.version + print "this application's version. Double check and make sure" + print "the repository was prepared with all necessary tags!" + raise SystemExit(-1) + did_git_init = False + did_git_checkout_scripts = False + try: + # create repository + shell.call("git", "init") + did_git_init = True + # configure our remote + shell.call("git", "remote", "add", "origin", repo) + # configure what would normally be set up on a 'git clone' for consistency + shell.call("git", "config", "branch.master.remote", "origin") + shell.call("git", "config", "branch.master.merge", "refs/heads/master") + # perform the initial fetch + shell.call("git", "fetch", "origin") + # soft reset to our tag + shell.call("git", "reset", tag) + # checkout the .scripts directory + shell.call("git", "checkout", ".scripts") + did_git_checkout_scripts = True + # XXX: setup .scripts/version??? + # for verbose purposes, give us a git status and git diff + shell.call("git", "status") + shell.call("git", "diff") + except: + print + print "ERROR: Exception detected! Rolling back..." + if did_git_init: + print "Deleting .git directory" + shell.call("rm", "-Rf", ".git") + if did_git_checkout_scripts: + print "Deleting .scripts directory" + shell.call("rm", "-Rf", ".scripts") + raise diff --git a/lib/wizard/shell.py b/lib/wizard/shell.py new file mode 100644 index 0000000..a7dee83 --- /dev/null +++ b/lib/wizard/shell.py @@ -0,0 +1,24 @@ +import subprocess +from subprocess import CalledProcessError +import sys + +class Shell(object): + """An advanced shell, with the ability to do dry-run and log commands""" + def __init__(self, verbose = False, dry = False): + """ `verbose` Whether or not to print the command and outputs + `dry` Whether or not to not run any commands, and just print""" + self.verbose = verbose + self.dry = dry + def call(self, *args): + if self.dry or self.verbose: + print "$ " + ' '.join(args) + if self.dry: return + proc = None + if self.verbose: + proc = subprocess.Popen(args, stdout=sys.stdout, stderr=sys.stderr) + else: + proc = subprocess.Popen(args) + proc.communicate() + if proc.returncode: + raise CalledProcessError(proc.returncode, args) + -- 2.45.2