Rate-limit vos queries; add ridiculous race-safe code to backups.

Signed-off-by: Edward Z. Yang <ezyang@mit.edu>

diff --git a/wizard/app/__init__.py b/wizard/app/__init__.py
index de127369060dfa413deea9bccf1e68c211edbb22..4b64f8034eaf4aa99e4eb553d1b1c14aeaf7b2c8 100644 (file)
--- a/wizard/app/__init__.py
+++ b/wizard/app/__init__.py
@@ -688,7 +688,6 @@ def backup_mysql_database(outdir, deployment):
         sh.call("mysqldump", "--compress", "-r", outfile, *get_mysql_args(deployment.dsn))
         sh.call("gzip", "--best", outfile)
     except shell.CallError as e:
-        shutil.rmtree(outdir)
         raise BackupFailure(e.stderr)
 
 def restore_database(backup_dir, deployment):

diff --git a/wizard/deploy.py b/wizard/deploy.py
index 7bc2c7671f82a75ff90b1304918227ad6eeed305..73f0095ea27014c83d76fa2ca3d0fc758c396d41 100644 (file)
--- a/wizard/deploy.py
+++ b/wizard/deploy.py
@@ -9,6 +9,7 @@ import fileinput
 import logging
 import decorator
 import datetime
+import tempfile
 
 import wizard
 from wizard import app, git, old_log, scripts, shell, sql, util
@@ -326,15 +327,40 @@ class ProductionCopy(Deployment):
         """
         Performs a backup of database schemas and other non-versioned data.
         """
-        backupdir = os.path.join(self.location, ".scripts", "backups")
-        backup = str(self.version) + "-" + datetime.date.today().isoformat()
-        outdir = os.path.join(backupdir, backup)
+        # There are retarded amounts of race-safety in this function,
+        # because we do NOT want to claim to have made a backup, when
+        # actually something weird happened to it.
+        backupdir = os.path.join(self.scripts_dir, "backups")
         if not os.path.exists(backupdir):
-            os.mkdir(backupdir)
-        if os.path.exists(outdir):
-            util.safe_unlink(outdir)
-        os.mkdir(outdir)
-        self.application.backup(self, outdir, options)
+            try:
+                os.mkdir(backupdir)
+            except OSError as e:
+                if e.errno == errno.EEXIST:
+                    pass
+                else:
+                    raise
+        tmpdir = tempfile.mkdtemp() # actually will be kept around
+        try:
+            self.application.backup(self, tmpdir, options)
+        except app.BackupFailure:
+            # the backup is bogus, don't let it show up
+            shutil.rmtree(tmpdir)
+            raise
+        backup = None
+        while 1:
+            backup = str(self.version) + "-" + datetime.datetime.today().strftime("%Y-%m-%dT%H%M%S")
+            outdir = os.path.join(backupdir, backup)
+            if os.path.exists(outdir):
+                logging.warning("Backup: A backup occurred in the last second. Trying again in a second...")
+                time.sleep(1)
+                continue
+            try:
+                os.rename(tmpdir, outdir)
+            except OSError:
+                logging.warning("Backup: We lost the race, trying again in a second...")
+                time.sleep(1)
+                continue
+            break
         return backup
     @chdir_to_location
     def restore(self, backup, options):

diff --git a/wizard/scripts.py b/wizard/scripts.py
index ac2cfb69415dd3160a71d7d168d3c1bd64155f7c..d151537c014d94b17bc2b4f8995cc07a872bfa4f 100644 (file)
--- a/wizard/scripts.py
+++ b/wizard/scripts.py
@@ -3,6 +3,7 @@ import shlex
 import errno
 import logging
 import urlparse
+import time
 
 import wizard
 from wizard import shell, util
@@ -46,6 +47,7 @@ def get_quota_usage_and_limit(dir=None):
             return _get_quota_usage_and_limit(dir)
         except QuotaParseError as e:
             exc = e
+            time.sleep(3) # give it a chance
     raise exc
 
 def _get_quota_usage_and_limit(dir=None):