Add 'wizard prepare-config' command, which is upgrade automation.

Signed-off-by: Edward Z. Yang <ezyang@mit.edu>

diff --git a/bin/wizard b/bin/wizard
index 411c31d6a5c41238881cf867dcf0cbce04ba06ff..09381a311bee534015172762438edc5a3c5ebf9a 100755 (executable)
--- a/bin/wizard
+++ b/bin/wizard
@@ -22,6 +22,7 @@ Its commands are:
     list            Lists autoinstalls, with optional filtering
     massmigrate     Performs mass migration of autoinstalls of an application
     migrate         Migrate autoinstalls from old format to Git-based format
     list            Lists autoinstalls, with optional filtering
     massmigrate     Performs mass migration of autoinstalls of an application
     migrate         Migrate autoinstalls from old format to Git-based format

+    prepare-config  Prepares configuration files for versioning

     summary         Generate statistics (see help for subcommands)
     upgrade         Upgrades an autoinstall to the latest version
 
     summary         Generate statistics (see help for subcommands)
     upgrade         Upgrades an autoinstall to the latest version
 


@@ -60,6 +61,7 @@ See '%prog help COMMAND' for more information on a specific command."""
     command_module.main(rest_argv, baton)
 
 def get_command(name):
     command_module.main(rest_argv, baton)
 
 def get_command(name):

+    name = name.replace("-", "_")

     __import__("wizard.command." + name)
     return getattr(wizard.command, name)
 
     __import__("wizard.command." + name)
     return getattr(wizard.command, name)
 

diff --git a/wizard/app/__init__.py b/wizard/app/__init__.py
index 4f8448e9ae7e47c23fd5c82c75647eface50cdb7..90657630c2cb59fb2c9d946da2b514c3d86bc39d 100644 (file)
--- a/wizard/app/__init__.py
+++ b/wizard/app/__init__.py
@@ -1,7 +1,15 @@
 import os.path
 import os.path

+import re
+
+def expand_re(val):
+    if isinstance(val, str):
+        return re.escape(val)
+    else:
+        return '(?:' + '|'.join(map(expand_re, val)) + ')'

 
 def filename_regex_extractor(f):
 
 def filename_regex_extractor(f):

-    """This is a decorator to apply to functions that take a name and return
+    """
+    This is a decorator to apply to functions that take a name and return

     (filename, RegexObject) tuples.  It converts it into a function
     that takes a name and returns another function (the actual extractor)
     which takes a deployment and returns the value of the extracted variable.
     (filename, RegexObject) tuples.  It converts it into a function
     that takes a name and returns another function (the actual extractor)
     which takes a deployment and returns the value of the extracted variable.


@@ -10,12 +18,17 @@ def filename_regex_extractor(f):
     (with the second subgroup being the value we care about), so that we can
     reuse the regex for other things.
 
     (with the second subgroup being the value we care about), so that we can
     reuse the regex for other things.
 

-    Its Haskell-style type signature would be:
+    Its Haskell-style type signature would be::
+

         (String -> (Filename, Regex)) -> (String -> (Deployment -> String))
 
         (String -> (Filename, Regex)) -> (String -> (Deployment -> String))
 

-    It's a little bit like a transformer."""
+    For convenience purposes, we also accept [Filename], in which case
+    we use the first entry (index 0).  Passing an empty list is invalid.
+    """

     def g(var):
         file, regex = f(var)
     def g(var):
         file, regex = f(var)

+        if not isinstance(file, str):
+            file = file[0]

         def h(deployment):
             contents = deployment.read(file) # cached
             match = regex.search(contents)
         def h(deployment):
             contents = deployment.read(file) # cached
             match = regex.search(contents)


@@ -25,3 +38,39 @@ def filename_regex_extractor(f):
         return h
     return g
 
         return h
     return g
 

+def filename_regex_substitution(f):
+    """
+    This is a decorator to apply to functions that take a name and return
+    (filename, RegexObject) tuples.  It converts it into a function
+    that takes a name and returns another function (that does substitution)
+    which takes a deployment and modifies its files to replace explicit
+    values with their generic WIZARD_* equivalents.  The final function returns
+    the number of replacements made.
+
+    The regular expression requires a very specific form, essentially ()()()
+    (with the second subgroup being the value to be replaced).
+
+    Its Haskell-style type signature would be::
+
+        (String -> ([Filename], Regex)) -> (String -> (Deployment -> IO Int))
+
+    For convenience purposes, we also accept Filename, in which case it is treated
+    as a single item list.
+    """
+    def g(key, var):
+        files, regex = f(var)
+        if isinstance(files, str):
+            files = (files,)
+        def h(deployment):
+            base = deployment.location
+            subs = 0
+            for file in files:
+                file = os.path.join(base, file)
+                contents = open(file, "r").read()
+                contents, n = regex.subn("\\1" + key + "\\3", contents)
+                subs += n
+                open(file, "w").write(contents)
+            return subs
+        return h
+    return g
+

diff --git a/wizard/app/mediawiki.py b/wizard/app/mediawiki.py
index d0888e8f33ce3249abea15a4e3dc3457e1950e1a..5c18b159594eec4b8480f1437efe9b7ac1fb7023 100644 (file)
--- a/wizard/app/mediawiki.py
+++ b/wizard/app/mediawiki.py
@@ -5,29 +5,38 @@ from wizard import app, deploy, install, util
 from wizard.app import php
 
 def make_filename_regex(var):
 from wizard.app import php
 
 def make_filename_regex(var):

-    return 'LocalSettings.php', re.compile('^(\$' + re.escape(var) + r'''\s*=\s*)(.*)(;)$''', re.M)
+    return 'LocalSettings.php', re.compile('^(\$' + app.expand_re(var) + r'''\s*=\s*)(.*)(;)$''', re.M)

 
 make_extractor = app.filename_regex_extractor(make_filename_regex)
 
 make_extractor = app.filename_regex_extractor(make_filename_regex)

+make_substitution = app.filename_regex_substitution(make_filename_regex)
+seed = {
+        'WIZARD_IP': 'IP', # obsolete, remove after we're done
+        'WIZARD_SITENAME': 'wgSitename',
+        'WIZARD_SCRIPTPATH': 'wgScriptPath',
+        'WIZARD_EMERGENCYCONTACT': ('wgEmergencyContact', 'wgPasswordSender'),
+        'WIZARD_DBSERVER': 'wgDBserver',
+        'WIZARD_DBNAME': 'wgDBname',
+        'WIZARD_DBUSER': 'wgDBuser',
+        'WIZARD_DBPASSWORD': 'wgDBpassword',
+        'WIZARD_SECRETKEY': ('wgSecretKey', 'wgProxyKey'),
+        }

 
 class Application(deploy.Application):
     parametrized_files = ['LocalSettings.php', 'php.ini']
 
 class Application(deploy.Application):
     parametrized_files = ['LocalSettings.php', 'php.ini']

+    deprecated_keys = set(['WIZARD_IP']) | php.deprecated_keys

     @property
     def extractors(self):
         if not self._extractors:
     @property
     def extractors(self):
         if not self._extractors:

-            self._extractors = util.dictmap(make_extractor,
-                {'WIZARD_IP': 'IP' # obsolete, remove after we're done
-                ,'WIZARD_SITENAME': 'wgSitename'
-                ,'WIZARD_SCRIPTPATH': 'wgScriptPath'
-                ,'WIZARD_EMERGENCYCONTACT': 'wgEmergencyContact'
-                ,'WIZARD_DBSERVER': 'wgDBserver'
-                ,'WIZARD_DBNAME': 'wgDBname'
-                ,'WIZARD_DBUSER': 'wgDBuser'
-                ,'WIZARD_DBPASSWORD': 'wgDBpassword'
-                ,'WIZARD_PROXYKEY': 'wgProxyKey'
-                })
+            self._extractors = util.dictmap(make_extractor, seed)

             self._extractors.update(php.extractors)
         return self._extractors
     @property
             self._extractors.update(php.extractors)
         return self._extractors
     @property

+    def substitutions(self):
+        if not self._substitutions:
+            self._substitutions = util.dictkmap(make_substitution, seed)
+            self._substitutions.update(php.substitutions)
+        return self._substitutions
+    @property

     def install_handler(self):
         handler = install.ArgHandler("mysql", "admin", "email")
         handler.add(install.Arg("title", help="Title of your new MediaWiki install"))
     def install_handler(self):
         handler = install.ArgHandler("mysql", "admin", "email")
         handler.add(install.Arg("title", help="Title of your new MediaWiki install"))

diff --git a/wizard/app/php.py b/wizard/app/php.py
index 5eb5130adbb74cc4b230b29f719d79750442966d..fdf201248c2d4d3ee6742dfbb19c3af18fe6b658 100644 (file)
--- a/wizard/app/php.py
+++ b/wizard/app/php.py
@@ -3,12 +3,16 @@ import re
 from wizard import app, util
 
 def make_filename_regex(var):
 from wizard import app, util
 
 def make_filename_regex(var):

-    return 'php.ini', re.compile('^(' + re.escape(var) + r'\s*=\s*)(.*)()$', re.M)
+    return 'php.ini', re.compile('^(' + app.expand_re(var) + r'\s*=\s*)(.*)()$', re.M)

 
 make_extractor = app.filename_regex_extractor(make_filename_regex)
 
 make_extractor = app.filename_regex_extractor(make_filename_regex)

+make_substitution = app.filename_regex_substitution(make_filename_regex)
+seed = {
+        'WIZARD_SESSIONNAME': 'session.name',
+        'WIZARD_TMPDIR': ('upload_tmp_dir', 'session.save_path'),
+        }

 
 

-extractors = util.dictmap(make_extractor,
-        {'WIZARD_SESSIONNAME': 'session.name'
-        ,'WIZARD_TMPDIR': 'upload_tmp_dir'
-        })
+extractors = util.dictmap(make_extractor, seed)
+substitutions = util.dictkmap(make_substitution, seed)
+deprecated_keys = set([])

 
 

diff --git a/wizard/command/migrate.py b/wizard/command/migrate.py
index 8c531674f9f0375711137e49fb0d87ddab9741dd..87a24d4e229931a851dfc0cbf693977fa92fc80c 100644 (file)
--- a/wizard/command/migrate.py
+++ b/wizard/command/migrate.py
@@ -137,9 +137,7 @@ def check_variables(d, options):
     """Attempt to extract variables and complain if some are missing."""
     variables = d.extract()
     for k,v in variables.items():
     """Attempt to extract variables and complain if some are missing."""
     variables = d.extract()
     for k,v in variables.items():

-        if v is None:
-            # once we get everything on the same version, you should
-            # actually start paying attention to these warnings
+        if v is None and k not in d.application.deprecated_keys:

             logging.warning("Variable %s not found" % k)
         else:
             logging.debug("Variable %s is %s" % (k,v))
             logging.warning("Variable %s not found" % k)
         else:
             logging.debug("Variable %s is %s" % (k,v))

diff --git a/wizard/command/prepare_config.py b/wizard/command/prepare_config.py
new file mode 100644 (file)
index 0000000..8dc1200
--- /dev/null
+++ b/wizard/command/prepare_config.py
@@ -0,0 +1,44 @@
+import os
+import shutil
+import logging
+import errno
+import sys
+
+import wizard
+from wizard import command, deploy, shell, util
+
+def main(argv, baton):
+    options, args = parse_args(argv, baton)
+    sh = shell.Shell()
+    d = deploy.Deployment(".")
+    try:
+        configure_args = args + command.makeBaseArgs(options)
+        sh.call("wizard", "configure", *configure_args, interactive=True)
+    except shell.PythonCallError:
+        sys.exit(1)
+    d.prepareConfig()
+
+def parse_args(argv, baton):
+    usage = """usage: %prog prepare-config -- [SETUPARGS]
+
+During the preparation of an upgrade, changes to configuration files
+must be taken into account.  Unfortunately, configuration files
+are not commonly versioned, and are commonly autogenerated.  Running
+this command will update all configuration files.
+
+To be more specific, it takes a checked out repository, and performs a
+configuration.  Then, it replaces the specific values from the installation
+back with generic values, which can be committed to the repository.  The final
+working copy state has HEAD pointing to the -scripts version that the commit
+was based off of, but with local changes that can be incorporated using
+`git commit --amend -a`.  You should inspect the changes with `git diff`;
+it is possible that the regular expressions in wizard.app.APPNAME are
+now non-functional.
+
+Wizard should be run in the environment installations are planned
+to be deployed to, because installers can have subtle differences
+in configuration files based on detected server configuration.
+"""
+    parser = command.WizardOptionParser(usage)
+    return parser.parse_all(argv)
+

diff --git a/wizard/deploy.py b/wizard/deploy.py
index 3ae65e3b2c9d4206652b0278758a14e5937ff0ef..8c694efe2bc941d8da80f7dd072eae7fd95baa37 100644 (file)
--- a/wizard/deploy.py
+++ b/wizard/deploy.py
@@ -9,6 +9,7 @@ import fileinput
 import dateutil.parser
 import distutils.version
 import tempfile
 import dateutil.parser
 import distutils.version
 import tempfile

+import logging

 
 import wizard
 from wizard import git, log, util
 
 import wizard
 from wizard import git, log, util


@@ -99,6 +100,12 @@ class Deployment(object):
         as such dir will generally not equal :attr:`location`.
         """
         return self.application.parametrize(self, dir)
         as such dir will generally not equal :attr:`location`.
         """
         return self.application.parametrize(self, dir)

+    def prepareConfig(self):
+        """
+        Edits files in the deployment such that any user-specific configuration
+        is replaced with generic WIZARD_* variables.
+        """
+        return self.application.prepareConfig(self)

     def updateVersion(self, version):
         """
         Update the version of this deployment.
     def updateVersion(self, version):
         """
         Update the version of this deployment.


@@ -213,7 +220,7 @@ class Application(object):
         self.versions = {}
         # cache variables
         self._extractors = {}
         self.versions = {}
         # cache variables
         self._extractors = {}

-        self._parametrizers = {}
+        self._substitutions = {}

     def repository(self, srv_path):
         """
         Returns the Git repository that would contain this application.
     def repository(self, srv_path):
         """
         Returns the Git repository that would contain this application.


@@ -259,6 +266,18 @@ class Application(object):
             tmp.write(contents)
             tmp.close()
             os.rename(tmp.name, fullpath)
             tmp.write(contents)
             tmp.close()
             os.rename(tmp.name, fullpath)

+    def prepareConfig(self, deployment):
+        """
+        Takes a deployment and replaces any explicit instances
+        of a configuration variable with generic WIZARD_* constants.
+        There is a sane default implementation built on substitutions;
+        you can override this method to provide arbitrary extra
+        behavior.
+        """
+        for key, subst in self.substitutions.items():
+            subs = subst(deployment)
+            if not subs and key not in self.deprecated_keys:
+                logging.warning("No substitutions for %s" % key)

     @property
     def extractors(self):
         """
     @property
     def extractors(self):
         """


@@ -268,6 +287,15 @@ class Application(object):
         See also :func:`wizard.app.filename_regex_extractor`.
         """
         return {}
         See also :func:`wizard.app.filename_regex_extractor`.
         """
         return {}

+    @property
+    def substitutions(self):
+        """
+        Dictionary of variable names to substitution functions.  These functions
+        take a :class:`Deployment` as an argument and modify the deployment such
+        that an explicit instance of the variable is released with the generic
+        WIZARD_* constant.  See also :func:`wizard.app.filename_regex_substitution`.
+        """
+        return {}

     @staticmethod
     def make(name):
         """Makes an application, but uses the correct subtype if available."""
     @staticmethod
     def make(name):
         """Makes an application, but uses the correct subtype if available."""

diff --git a/wizard/util.py b/wizard/util.py
index 933be8ab19b17aee9536d5036c4ec5204a075f27..071737ad700d9f179464f74caf7fb23d8ca1c247 100644 (file)
--- a/wizard/util.py
+++ b/wizard/util.py
@@ -62,6 +62,15 @@ def dictmap(f, d):
     """
     return dict((k,f(v)) for k,v in d.items())
 
     """
     return dict((k,f(v)) for k,v in d.items())
 

+def dictkmap(f, d):
+    """
+    A map function for dictionaries that passes key and value.
+
+        >>> dictkmap(lambda x, y: x + y, {1: 4, 3: 4})
+        {1: 5, 3: 6}
+    """
+    return dict((k,f(k,v)) for k,v in d.items())
+

 def get_exception_name(output):
     """
     Reads the traceback from a Python program and grabs the
 def get_exception_name(output):
     """
     Reads the traceback from a Python program and grabs the