From: Edward Z. Yang <ezyang@mit.edu>
Date: Mon, 12 Oct 2009 02:52:52 +0000 (-0400)
Subject: Major updates to resolution code from runs.
X-Git-Url: https://scripts.mit.edu/gitweb/wizard.git/commitdiff_plain/48af129368493c5458c942aef025ed746b97b856

Major updates to resolution code from runs.

* Augment MediaWiki conflict resolution with correct version for
  end of line
* Add prepareMerge to application for pre-merge resolution
  helpers (mostly for line-ending problems)
* Fix MediaWiki resolveConflicts parsing of ls-files output,
  and another unrelated bug
* Add space between logging and summary outputs
* Move backup back to correct location, and perform size calculation
  twice
* UpgradeVerificationFailure gets useful output
* /dev/shm/wizard gets chmod'ed 0777
* Fix resolution algorithm to handle each conflict block individually.
  Test cases augmented accordingly.

Signed-off-by: Edward Z. Yang <ezyang@mit.edu>
---

diff --git a/wizard/app/mediawiki.py b/wizard/app/mediawiki.py
index e356c8f..ceae53a 100644
--- a/wizard/app/mediawiki.py
+++ b/wizard/app/mediawiki.py
@@ -27,10 +27,79 @@ seed = {
         }
 
 resolutions = {
-#'LocalSettings.php': [
-#    ("""
-#""", [])
-#    ]
+'LocalSettings.php': [
+    ("""
+<<<<<<<
+***1***
+=======
+## The URL base path to the directory containing the wiki;
+## defaults for all runtime URL paths are based off of this.
+## For more information on customizing the URLs please see:
+## http://www.mediawiki.org/wiki/Manual:Short_URL
+***2***
+$wgScriptExtension  = ".php";
+
+## UPO means: this is also a user preference option
+>>>>>>>
+""", [-1]),
+    ("""
+<<<<<<<
+***1***
+=======
+
+# MySQL specific settings
+$wgDBprefix         = "";
+>>>>>>>
+""", ["\n# MySQL specific settings", 1]),
+    ("""
+<<<<<<<
+## is writable, then uncomment this:
+***1***
+=======
+## is writable, then set this to true:
+$wgEnableUploads       = false;
+>>>>>>>
+""", [-1]),
+    ("""
+<<<<<<<
+***1***
+$wgMathPath         = "{$wgUploadPath}/math";
+$wgMathDirectory    = "{$wgUploadDirectory}/math";
+$wgTmpDirectory     = "{$wgUploadDirectory}/tmp";
+=======
+$wgUseTeX           = false;
+>>>>>>>
+""", [1]),
+    # order of these rules is important
+    ("""
+<<<<<<<
+?>
+=======
+# When you make changes to this configuration file, this will make
+# sure that cached pages are cleared.
+$wgCacheEpoch = max( $wgCacheEpoch, gmdate( 'YmdHis', @filemtime( __FILE__ ) ) );
+>>>>>>>
+""", [0]),
+    ("""
+<<<<<<<
+***1***
+?>
+=======
+# When you make changes to this configuration file, this will make
+# sure that cached pages are cleared.
+$wgCacheEpoch = max( $wgCacheEpoch, gmdate( 'YmdHis', @filemtime( __FILE__ ) ) );
+>>>>>>>
+""", [1, 0]),
+    ("""
+<<<<<<<
+***1***
+=======
+# When you make changes to this configuration file, this will make
+# sure that cached pages are cleared.
+$wgCacheEpoch = max( $wgCacheEpoch, gmdate( 'YmdHis', @filemtime( __FILE__ ) ) );
+>>>>>>>
+""", [1, 0]),
+    ]
 }
 
 class Application(deploy.Application):
@@ -66,18 +135,29 @@ class Application(deploy.Application):
         if type(out) is list:
             out.append(page)
         return page.find("<!-- Served") != -1
+    def prepareMerge(self, dir):
+        with util.ChangeDirectory(dir):
+            # XXX: this should be factored out
+            contents = open("LocalSettings.php", "r").read()
+            new_contents = "\n".join(contents.splitlines())
+            if contents != new_contents:
+                open("LocalSettings.php", "w").write(contents)
     def resolveConflicts(self, dir):
         # XXX: this is pretty generic
         resolved = True
         with util.ChangeDirectory(dir):
             sh = shell.Shell()
-            for file in sh.eval("git", "ls-files", "--unmerged").splitlines():
+            for status in sh.eval("git", "ls-files", "--unmerged").splitlines():
+                file = status.split()[-1]
                 if file in resolutions:
                     contents = open(file, "r").read()
-                    for spec, result in resolutions:
+                    for spec, result in resolutions[file]:
+                        old_contents = contents
                         contents = resolve.resolve(contents, spec, result)
+                        if old_contents != contents:
+                            logging.info("Did resolution with spec:\n" + spec)
+                    open(file, "w").write(contents)
                     if not resolve.is_conflict(contents):
-                        open(file, "w").write(contents)
                         sh.call("git", "add", file)
                     else:
                         resolved = False
diff --git a/wizard/command/mass_upgrade.py b/wizard/command/mass_upgrade.py
index 0f5e0b8..80dd2a6 100644
--- a/wizard/command/mass_upgrade.py
+++ b/wizard/command/mass_upgrade.py
@@ -7,6 +7,7 @@ import hashlib
 import errno
 import time
 import itertools
+import sys
 
 import wizard
 from wizard import deploy, util, scripts, shell, sset, command
@@ -91,6 +92,7 @@ def main(argv, baton):
                           on_success=on_success, on_error=on_error)
         sh.join()
     finally:
+        sys.stderr.write("\n")
         for name, deploys in errors.items():
             logging.warning("%s from %d installs" % (name, len(deploys)))
         def printPercent(description, number, total):
diff --git a/wizard/command/upgrade.py b/wizard/command/upgrade.py
index d47bcb1..f56d00b 100644
--- a/wizard/command/upgrade.py
+++ b/wizard/command/upgrade.py
@@ -10,7 +10,7 @@ import itertools
 
 from wizard import app, command, deploy, scripts, shell, util
 
-kib_buffer = 1024 * 10 # 10 MiB we will always leave available
+kib_buffer = 1024 * 30 # 30 MiB we will always leave available
 
 def main(argv, baton):
     options, args = parse_args(argv, baton)
@@ -51,9 +51,6 @@ def main(argv, baton):
         d.verifyVersion()
         if not options.dry_run:
             d.verifyWeb()
-        # perform database backup, this should be done before checking quota
-        if not options.dry_run:
-            backup = d.backup(options)
         repo = d.application.repository(options.srv_path)
         version = calculate_newest_version(sh, repo)
         if version == d.app_version.scripts_tag and not options.force:
@@ -87,7 +84,9 @@ def main(argv, baton):
             open(".git/WIZARD_REPO", "w").write(repo)
             open(".git/WIZARD_UPGRADE_VERSION", "w").write(version)
             open(".git/WIZARD_PARENTS", "w").write("%s\n%s" % (user_commit, next_commit))
-            if options.log_file: open(".git/WIZARD_LOG_FILE", "w").write(options.log_file)
+            open(".git/WIZARD_SIZE", "w").write(str(scripts.get_disk_usage()))
+            if options.log_file:
+                open(".git/WIZARD_LOG_FILE", "w").write(options.log_file)
             perform_merge(sh, repo, d, version, use_shm, kib_limit and kib_limit - kib_usage or None)
     # variables: version, user_commit, next_commit, temp_wc_dir
     with util.ChangeDirectory(temp_wc_dir):
@@ -114,6 +113,15 @@ def main(argv, baton):
     if options.dry_run:
         logging.info("Dry run, bailing.  See results at %s" % temp_wc_dir)
         return
+    # Ok, now we have to do a crazy complicated dance to see if we're
+    # going to have enough quota to finish what we need
+    pre_size = int(open(os.path.join(temp_wc_dir, ".git/WIZARD_SIZE"), "r").read())
+    post_size = scripts.get_disk_usage(temp_wc_dir)
+    kib_usage, kib_limit = scripts.get_quota_usage_and_limit()
+    backup = d.backup(options)
+    if kib_limit is not None and (kib_limit - kib_usage) - (post_size - pre_size) / 1024 < kib_buffer:
+        shutil.rmtree(os.path.join(".scripts/backups", sh.eval("wizard", "restore").splitlines()[0]))
+        raise QuotaTooLow
     # XXX: frob .htaccess to make site inaccessible
     with util.IgnoreKeyboardInterrupts():
         with util.LockDirectory(".scripts-upgrade-lock"):
@@ -137,9 +145,8 @@ def main(argv, baton):
                 raise
             except deploy.WebVerificationError as e:
                 logging.warning("Web verification failed: rolling back")
-                logging.info("Web page that was output was:\n\n%s" % e.contents)
                 perform_restore(d, backup)
-                raise app.UpgradeVerificationFailure("Upgrade caused website to become inaccessible; site was rolled back")
+                raise app.UpgradeVerificationFailure(e.contents)
     # XXX: frob .htaccess to make site accessible
     #       to do this, check if .htaccess changed, first.  Upgrade
     #       process might have frobbed it.  Don't be
@@ -187,6 +194,7 @@ def perform_tmp_clone(sh, use_shm):
         dir = "/dev/shm/wizard"
         if not os.path.exists(dir):
             os.mkdir(dir)
+            os.chmod(dir, 0o777)
     else:
         dir = None
     temp_dir = tempfile.mkdtemp(prefix="wizard", dir=dir)
@@ -221,11 +229,12 @@ def perform_merge(sh, repo, d, version, use_shm, kib_avail):
     user_virtual_commit = sh.eval("git", "commit-tree", user_tree,
             "-p", base_virtual_commit, input="", log=True)
     sh.call("git", "checkout", user_virtual_commit, "--")
-    pre_usage = scripts.get_disk_usage()
+    d.application.prepareMerge(os.getcwd())
+    sh.call("git", "commit", "--amend", "-a", "-m", "amendment")
     try:
         sh.call("git", "merge", next_virtual_commit)
     except shell.CallError as e:
-        conflicts = e.stderr.count("CONFLICT") # not perfect, if there is a file named CONFLICT
+        conflicts = e.stdout.count("CONFLICT") # not perfect, if there is a file named CONFLICT
         logging.info("Merge failed with these messages:\n\n" + e.stderr)
         # Run the application's specific merge resolution algorithms
         # and see if we can salvage it
@@ -250,11 +259,6 @@ def perform_merge(sh, repo, d, version, use_shm, kib_avail):
             os.chdir(curdir)
         print "%d %s" % (conflicts, curdir)
         raise MergeFailed
-    finally:
-        post_usage = scripts.get_disk_usage()
-        kib_delta = (post_usage - pre_usage) / 1024
-        if kib_avail is not None and kib_delta - kib_avail < kib_buffer:
-            raise QuotaTooLow
 
 def parse_args(argv, baton):
     usage = """usage: %prog upgrade [ARGS] [DIR]
diff --git a/wizard/deploy.py b/wizard/deploy.py
index 6197a9a..b4d2152 100644
--- a/wizard/deploy.py
+++ b/wizard/deploy.py
@@ -381,6 +381,14 @@ class Application(object):
         added to the index, but no commit is made.
         """
         return False
+    def prepareMerge(self, dir):
+        """
+        Takes a directory and performs various edits to files in
+        order to make a merge go more smoothly.  This is usually
+        used to fix botched line-endings.  If you add new files,
+        you have to 'git add' them; this is not necessary for edits.
+        """
+        pass
     def prepareConfig(self, deployment):
         """
         Takes a deployment and replaces any explicit instances
diff --git a/wizard/resolve.py b/wizard/resolve.py
index ae178d5..9111203 100644
--- a/wizard/resolve.py
+++ b/wizard/resolve.py
@@ -99,7 +99,25 @@ def resolve(contents, spec, result):
     rstring, mappings = spec_to_regex(spec)
     regex = re.compile(rstring, re.DOTALL)
     repl = result_to_repl(result, mappings)
-    return regex.sub(repl, contents)
+    ret = ""
+    conflict = ""
+    status = 0
+    for line in contents.splitlines(True):
+        if status == 0 and line.startswith("<<<<<<<"):
+            status = 1
+        elif status == 1 and line.startswith("======="):
+            status = 2
+        # ok, now process
+        if status == 2 and line.startswith(">>>>>>>"):
+            status = 0
+            conflict += line
+            ret += regex.sub(repl, conflict)
+            conflict = ""
+        elif status:
+            conflict += line
+        else:
+            ret += line
+    return ret
 
 def is_conflict(contents):
     # Really really simple heuristic
diff --git a/wizard/tests/resolve_test.py b/wizard/tests/resolve_test.py
index 65e17d4..ba5c2ee 100644
--- a/wizard/tests/resolve_test.py
+++ b/wizard/tests/resolve_test.py
@@ -65,13 +65,23 @@ def test_resolve_user():
     contents = """
 top
 <<<<<<<
+bar
 the user is right
+baz
 =======
 blah blah
+>>>>>>>
+bottom
+<<<<<<<
+Unrelated conflict
+=======
+Unrelated conflicts
 >>>>>>>"""
     spec = """
 <<<<<<<
 ***1***
+the user is right
+***2***
 =======
 blah blah
 >>>>>>>
@@ -79,7 +89,92 @@ blah blah
     result = [-1]
     assert resolve.resolve(contents, spec, result) == """
 top
+bar
+the user is right
+baz
+bottom
+<<<<<<<
+Unrelated conflict
+=======
+Unrelated conflicts
+>>>>>>>"""
+
+def test_resolve_multi_var():
+    contents = """
+top
+<<<<<<<
+the user is right
+this is ours
+more user stuff
+this is ours
+=======
+this is kept, but variable
+this is not kept
+>>>>>>>
+bottom
+<<<<<<<
+unrelated conflict
+=======
+unrelated conflicts
+>>>>>>>
+"""
+    spec = """
+<<<<<<<
+***1***
+this is ours
+***2***
+this is ours
+=======
+***3***
+this is not kept
+>>>>>>>
+"""
+    result = [3, 1, 2]
+    assert resolve.resolve(contents, spec, result) == """
+top
+this is kept, but variable
 the user is right
+more user stuff
+bottom
+<<<<<<<
+unrelated conflict
+=======
+unrelated conflicts
+>>>>>>>
+"""
+
+def test_resolve_simple():
+    contents = """
+bar
+<<<<<<< HEAD
+baz
+=======
+boo
+>>>>>>> upstream
+bing
+<<<<<<< HEAD
+oh?
+=======
+bad match
+>>>>>>> upstream
+"""
+    spec = """
+<<<<<<<
+***1***
+=======
+bad match
+>>>>>>>
+"""
+    result = [-1]
+    assert resolve.resolve(contents, spec, result) == """
+bar
+<<<<<<< HEAD
+baz
+=======
+boo
+>>>>>>> upstream
+bing
+oh?
 """
 
 def test_is_conflict():