From 30380c4b5b28df9670ea5952e14bc485d1d34133 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Fri, 25 Dec 2009 21:43:05 -0500 Subject: [PATCH] Convert ad hoc shell calls to singleton instance; fix upgrade bug. Signed-off-by: Edward Z. Yang --- tests/mediawiki-crlf-upgrade-test.sh | 2 +- wizard/app/__init__.py | 26 +++++++++++--------------- wizard/app/mediawiki.py | 5 ++--- wizard/command/blacklist.py | 3 +-- wizard/command/install.py | 5 ++--- wizard/command/prepare_config.py | 7 +++---- wizard/command/prepare_pristine.py | 16 +++++++--------- wizard/command/research.py | 3 +-- wizard/command/restore.py | 5 ++--- wizard/command/upgrade.py | 1 + wizard/deploy.py | 11 +++++------ wizard/git.py | 4 ++-- wizard/install/__init__.py | 7 +++---- 13 files changed, 41 insertions(+), 54 deletions(-) diff --git a/tests/mediawiki-crlf-upgrade-test.sh b/tests/mediawiki-crlf-upgrade-test.sh index bd6d7af..b12c866 100755 --- a/tests/mediawiki-crlf-upgrade-test.sh +++ b/tests/mediawiki-crlf-upgrade-test.sh @@ -1,7 +1,7 @@ #!/bin/bash -e cd `dirname $0` -TESTNAME="mediawiki_continue_upgrade" +TESTNAME="mediawiki_crlf_upgrade" source ./setup source ./mediawiki-install diff --git a/wizard/app/__init__.py b/wizard/app/__init__.py index 4b64f80..1d01966 100644 --- a/wizard/app/__init__.py +++ b/wizard/app/__init__.py @@ -237,14 +237,13 @@ class Application(object): default implementation uses :attr:`resolutions`. """ resolved = True - sh = shell.Shell() files = set() - for status in sh.eval("git", "ls-files", "--unmerged").splitlines(): + for status in shell.eval("git", "ls-files", "--unmerged").splitlines(): files.add(status.split()[-1]) for file in files: # check for newline mismatch # HACK: using git diff to tell if files are binary or not - if not len(sh.eval("git", "diff", file).splitlines()) == 1 and util.mixed_newlines(file): + if not len(shell.eval("git", "diff", file).splitlines()) == 1 and util.mixed_newlines(file): # this code only works on Unix def get_newline(filename): f = open(filename, "U") @@ -257,7 +256,7 @@ class Application(object): return f.newlines def create_reference(id): f = tempfile.NamedTemporaryFile(prefix="wizardResolve", delete=False) - sh.call("git", "cat-file", "blob", ":%d:%s" % (id, file), stdout=f) + shell.call("git", "cat-file", "blob", ":%d:%s" % (id, file), stdout=f) f.close() return get_newline(f.name), f.name def convert(filename, dest_nl): @@ -282,9 +281,9 @@ class Application(object): logging.info("Remerging %s", file) with open(file, "wb") as f: try: - sh.call("git", "merge-file", "--stdout", our_file, common_file, their_file, stdout=f) + shell.call("git", "merge-file", "--stdout", our_file, common_file, their_file, stdout=f) logging.info("New merge was clean") - sh.call("git", "add", file) + shell.call("git", "add", file) continue except shell.CallError: pass @@ -301,7 +300,7 @@ class Application(object): logging.info("Did resolution with spec:\n" + spec) open(file, "w").write(contents) if not resolve.is_conflict(contents): - sh.call("git", "add", file) + shell.call("git", "add", file) else: resolved = False else: @@ -682,11 +681,10 @@ def backup_mysql_database(outdir, deployment): """ Database backups for MySQL using the :command:`mysqldump` utility. """ - sh = shell.Shell() outfile = os.path.join(outdir, "db.sql") try: - sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment.dsn)) - sh.call("gzip", "--best", outfile) + shell.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment.dsn)) + shell.call("gzip", "--best", outfile) except shell.CallError as e: raise BackupFailure(e.stderr) @@ -704,13 +702,12 @@ def restore_mysql_database(backup_dir, deployment): """ Database restoration for MySQL by piping SQL commands into :command:`mysql`. """ - sh = shell.Shell() if not os.path.exists(backup_dir): raise RestoreFailure("Backup %s doesn't exist", backup_dir.rpartition("/")[2]) sql = open(os.path.join(backup_dir, "db.sql"), 'w+') - sh.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql) + shell.call("gunzip", "-c", os.path.join(backup_dir, "db.sql.gz"), stdout=sql) sql.seek(0) - sh.call("mysql", *get_mysql_args(deployment.dsn), stdin=sql) + shell.call("mysql", *get_mysql_args(deployment.dsn), stdin=sql) sql.close() def remove_database(deployment): @@ -718,10 +715,9 @@ def remove_database(deployment): Generic database removal function. Actually, not so generic because we go and check if we're on scripts and if we are run a different command. """ - sh = shell.Shell() if deployment.dsn.host == "sql.mit.edu": try: - sh.call("/mit/scripts/sql/bin/drop-database", deployment.dsn.database) + shell.call("/mit/scripts/sql/bin/drop-database", deployment.dsn.database) return except shell.CallError: pass diff --git a/wizard/app/mediawiki.py b/wizard/app/mediawiki.py index 9d04684..c6ac8ae 100644 --- a/wizard/app/mediawiki.py +++ b/wizard/app/mediawiki.py @@ -71,11 +71,10 @@ class Application(app.Application): raise app.RecoverableInstallFailure(error_messages) os.rename('config/LocalSettings.php', 'LocalSettings.php') def upgrade(self, d, version, options): - sh = shell.Shell() if not os.path.isfile("AdminSettings.php"): - sh.call("git", "checkout", "-q", "mediawiki-" + str(version), "--", "AdminSettings.php") + shell.call("git", "checkout", "-q", "mediawiki-" + str(version), "--", "AdminSettings.php") try: - result = sh.eval("php", "maintenance/update.php", "--quick", log=True) + result = shell.eval("php", "maintenance/update.php", "--quick", log=True) except shell.CallError as e: raise app.UpgradeFailure("Update script returned non-zero exit code\nSTDOUT: %s\nSTDERR: %s" % (e.stdout, e.stderr)) results = result.rstrip().split() diff --git a/wizard/command/blacklist.py b/wizard/command/blacklist.py index 8665524..535fdb3 100644 --- a/wizard/command/blacklist.py +++ b/wizard/command/blacklist.py @@ -5,10 +5,9 @@ from wizard import command, shell, util def main(argv, baton): options, args = parse_args(argv, baton) reason = args[0] - sh = shell.Shell() # XXX: this should be abstracted away! if os.path.exists(".git/WIZARD_REPO"): - util.chdir(sh.eval('git', 'config', 'remote.origin.url')) + util.chdir(shell.eval('git', 'config', 'remote.origin.url')) open('.scripts/blacklisted', 'w').write(reason + "\n") def parse_args(argv, baton): diff --git a/wizard/command/install.py b/wizard/command/install.py index a4faf83..4ae9058 100644 --- a/wizard/command/install.py +++ b/wizard/command/install.py @@ -40,13 +40,12 @@ Autoinstalls the application %s in the directory DIR.""" % (appname, appname)) else: ihandler.ask(options) - sh = shell.Shell() input.infobox("Copying files (this may take a while)...") if not os.path.exists(dir): - sh.call("git", "clone", "-q", "--shared", application.repository(old_options.srv_path), dir) + shell.call("git", "clone", "-q", "--shared", application.repository(old_options.srv_path), dir) with util.ChangeDirectory(dir): if not old_options.retry and version and version != "head-scripts": # for ease in testing - sh.call("git", "reset", "-q", "--hard", appstr) + shell.call("git", "reset", "-q", "--hard", appstr) input.infobox("Installing...") application.install(distutils.version.LooseVersion(version), options) if not old_options.no_commit: diff --git a/wizard/command/prepare_config.py b/wizard/command/prepare_config.py index 7d49339..13a4bc9 100644 --- a/wizard/command/prepare_config.py +++ b/wizard/command/prepare_config.py @@ -5,15 +5,14 @@ from wizard import command, deploy, shell def main(argv, baton): options, args = parse_args(argv, baton) - sh = shell.Shell() wc = deploy.WorkingCopy(".") wc.verify() wc.verifyConfigured() # worst case scenario protection for file in wc.application.parametrized_files: - sh.call("git", "add", file) - sh.call("git", "commit", "--allow-empty", "-am", "Protection commit") - sh.call("git", "reset", "HEAD~") + shell.call("git", "add", file) + shell.call("git", "commit", "--allow-empty", "-am", "Protection commit") + shell.call("git", "reset", "HEAD~") wc.prepareConfig() def parse_args(argv, baton): diff --git a/wizard/command/prepare_pristine.py b/wizard/command/prepare_pristine.py index 3f16e2f..80517d1 100644 --- a/wizard/command/prepare_pristine.py +++ b/wizard/command/prepare_pristine.py @@ -6,7 +6,6 @@ from wizard import app, command, shell def main(argv, baton): options, args = parse_args(argv, baton) - sh = shell.Shell() check_directory(options) if not os.path.exists(args[0]): appname, _, version = args[0].partition("-") @@ -16,24 +15,24 @@ def main(argv, baton): with open(base, "w") as outfile: infile = urllib.urlopen(url) shutil.copyfileobj(infile, outfile) - sh.call("tar", "xf", base) + shell.call("tar", "xf", base) os.unlink(base) else: base = args[0] - sh.call("tar", "xf", base) + shell.call("tar", "xf", base) # extract the files, but be smart: if only one directory is output, # move the contents of that directory here items = [f for f in os.listdir(os.getcwd()) if f[0] != "."] if len(items) == 1 and os.path.isdir(items[0]): os.rename(items[0], "_wizard_source") - sh.call("cp", "-R", "_wizard_source/.", ".") + shell.call("cp", "-R", "_wizard_source/.", ".") shutil.rmtree("_wizard_source") # populate empty directories with blank files for dirpath, dirnames, filenames in os.walk(os.getcwd()): if "/.git" in dirpath: continue if not filenames and not dirnames: open(os.path.join(dirpath, ".preserve-dir"), "w").write("") - sh.call("git", "add", ".") + shell.call("git", "add", ".") def parse_args(argv, baton): usage = """usage: %prog prepare-pristine APP-VERSION @@ -57,18 +56,17 @@ local diffs: you can override this safety mechanism with --force. return options, args def check_directory(options): - sh = shell.Shell() - files = sh.eval("git", "ls-files", "-o") + files = shell.eval("git", "ls-files", "-o") if files: raise Exception("Unversioned files exist, refusing to remove (override with --force)") try: - sh.call("git", "rev-parse", "HEAD") + shell.call("git", "rev-parse", "HEAD") _, _, ref = open(".git/HEAD").read().rstrip().partition(' ') if not options.force: if ref != "refs/heads/pristine" and os.path.exists(os.path.join(".git", ref)): raise Exception("Not on pristine branch (override with --force)") try: - sh.call("git", "status") + shell.call("git", "status") raise Exception("Working copy is dirty (override with --force)") except shell.CallError: pass diff --git a/wizard/command/research.py b/wizard/command/research.py index a95c32f..d36a850 100644 --- a/wizard/command/research.py +++ b/wizard/command/research.py @@ -9,7 +9,6 @@ def main(argv, baton): options, show = parse_args(argv, baton) appname = show[0] application = app.applications()[appname] - sh = shell.Shell() deploys = deploy.parse_install_lines(show, options.versions_path) stats = {} iffy = 0 @@ -28,7 +27,7 @@ def main(argv, baton): d.verifyConfigured() with util.ChangeDirectory(d.location): results = [] - out = sh.safeCall('git', 'diff', '--numstat', d.app_version.scripts_tag, strip=True) + out = shell.safeCall('git', 'diff', '--numstat', d.app_version.scripts_tag, strip=True) total += 1 for line in out.split("\n"): added, deleted, filename = line.split(None, 3) diff --git a/wizard/command/restore.py b/wizard/command/restore.py index 060ab78..eae157f 100644 --- a/wizard/command/restore.py +++ b/wizard/command/restore.py @@ -38,12 +38,11 @@ def main(argv, baton): d.verify() d.verifyConfigured() tag = "%s-%s" % (d.application.name, version) - sh = shell.Shell() try: - sh.call("git", "rev-parse", tag) + shell.call("git", "rev-parse", tag) except shell.CallError: raise Exception("Tag %s doesn't exist in repository" % tag) - sh.call("git", "reset", "-q", "--hard", tag) + shell.call("git", "reset", "-q", "--hard", tag) d.restore(backup, options) def parse_args(argv, baton): diff --git a/wizard/command/upgrade.py b/wizard/command/upgrade.py index 3175744..e35daa8 100644 --- a/wizard/command/upgrade.py +++ b/wizard/command/upgrade.py @@ -260,6 +260,7 @@ class Upgrade(object): if self.wc.resolveConflicts(): logging.info("Resolved conflicts with application specific knowledge") shell.call("git", "commit", "-a", "-m", "merge") + return files = set() for line in shell.eval("git", "ls-files", "--unmerged").splitlines(): files.add(line.split(None, 3)[-1]) diff --git a/wizard/deploy.py b/wizard/deploy.py index 16768fa..67c4118 100644 --- a/wizard/deploy.py +++ b/wizard/deploy.py @@ -151,7 +151,7 @@ class Deployment(object): """ repo = self.application.repository(srv_path) try: - shell.Shell().eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--') + shell.eval("git", "--git-dir", repo, "rev-parse", self.app_version.scripts_tag, '--') except shell.CallError: raise NoTagError(self.app_version.scripts_tag) @@ -163,13 +163,12 @@ class Deployment(object): corresponds to the one in the remote repository. """ with util.ChangeDirectory(self.location): - sh = shell.Shell() repo = self.application.repository(srv_path) def repo_rev_parse(tag): - return sh.eval("git", "--git-dir", repo, "rev-parse", tag) + return shell.eval("git", "--git-dir", repo, "rev-parse", tag) def self_rev_parse(tag): try: - return sh.safeCall("git", "rev-parse", tag, strip=True) + return shell.safeCall("git", "rev-parse", tag, strip=True) except shell.CallError: raise NoLocalTagError(tag) def compare_tags(tag): @@ -179,7 +178,7 @@ class Deployment(object): if not compare_tags(self.app_version.scripts_tag): raise InconsistentScriptsTagError(self.app_version.scripts_tag) parent = repo_rev_parse(self.app_version.scripts_tag) - merge_base = sh.safeCall("git", "merge-base", parent, "HEAD", strip=True) + merge_base = shell.safeCall("git", "merge-base", parent, "HEAD", strip=True) if merge_base != parent: raise HeadNotDescendantError(self.app_version.scripts_tag) @@ -275,7 +274,7 @@ class Deployment(object): except old_log.ScriptsVersionNoSuchFile: pass if not self._app_version: - appname = shell.Shell().eval("git", "config", "remote.origin.url").rpartition("/")[2].partition(".")[0] + appname = shell.eval("git", "config", "remote.origin.url").rpartition("/")[2].partition(".")[0] self._app_version = app.ApplicationVersion.make(appname, "unknown") return self._app_version @property diff --git a/wizard/git.py b/wizard/git.py index d98f757..6340766 100644 --- a/wizard/git.py +++ b/wizard/git.py @@ -6,7 +6,7 @@ from wizard import shell, util def describe(): """Finds the output of git describe --tags of the current directory.""" - return shell.Shell().safeCall("git", "describe", "--tags", strip=True) + return shell.safeCall("git", "describe", "--tags", strip=True) def commit_configure(): message = "Autoinstall configuration of %s locker.\n\n%s" % (util.get_dir_owner(), util.get_git_footer()) @@ -15,4 +15,4 @@ def commit_configure(): message += "\nConfigured-by: " + util.get_operator_git() except util.NoOperatorInfo: pass - shell.Shell().call("git", "commit", "--allow-empty", "-a", "-m", message) + shell.call("git", "commit", "--allow-empty", "-a", "-m", message) diff --git a/wizard/install/__init__.py b/wizard/install/__init__.py index c40ee62..8ca40a3 100644 --- a/wizard/install/__init__.py +++ b/wizard/install/__init__.py @@ -153,7 +153,7 @@ class ScriptsMysqlStrategy(Strategy): if self.application.database != "mysql": raise StrategyFailed try: - self._triplet = shell.Shell().eval("/mit/scripts/sql/bin/get-password").split() + self._triplet = shell.eval("/mit/scripts/sql/bin/get-password").split() except shell.CallError: raise StrategyFailed self._username = os.getenv('USER') @@ -161,11 +161,10 @@ class ScriptsMysqlStrategy(Strategy): raise StrategyFailed def execute(self, options): """Creates a new database for the user using ``get-next-database`` and ``create-database``.""" - sh = shell.Shell() host, username, password = self._triplet # race condition - database = self._username + '+' + sh.eval("/mit/scripts/sql/bin/get-next-database", os.path.basename(self.dir)) - sh.call("/mit/scripts/sql/bin/create-database", database) + database = self._username + '+' + shell.eval("/mit/scripts/sql/bin/get-next-database", os.path.basename(self.dir)) + shell.call("/mit/scripts/sql/bin/create-database", database) options.dsn = sqlalchemy.engine.url.URL("mysql", username=username, password=password, host=host, database=database) class ScriptsEmailStrategy(Strategy): -- 2.44.0