From: Edward Z. Yang <ezyang@mit.edu>
Date: Thu, 30 Jul 2009 20:21:17 +0000 (-0400)
Subject: Add mediawiki variable extraction support for deployments.
X-Git-Url: https://scripts.mit.edu/gitweb/wizard.git/commitdiff_plain/bf58b59267060bdb462476d824b086131b444cd4?hp=6eae78b3d8ed73a0266ef09a30e4b09e791dd9fb

Add mediawiki variable extraction support for deployments.

Signed-off-by: Edward Z. Yang <ezyang@mit.edu>
---

diff --git a/wizard/app/__init__.py b/wizard/app/__init__.py
index e69de29..a4a6ca2 100644
--- a/wizard/app/__init__.py
+++ b/wizard/app/__init__.py
@@ -0,0 +1,23 @@
+import os.path
+
+def filename_regex_extractor(f):
+    """This is a decorator to apply to functions that take a name and return
+    (filename, RegexObject) tuples.  It converts it into a function
+    that takes a name and returns another function (the actual extractor)
+    which takes a deployment and returns the value of the extracted variable.
+
+    Its Haskell-style type signature would be:
+        (String -> (Filename, Regex)) -> (String -> (Deployment -> String))
+
+    It's a little bit like a transformer."""
+    def g(var):
+        file, regex = f(var)
+        def h(deployment):
+            contents = deployment.read(file) # cached
+            match = regex.search(contents)
+            if not match: return None
+            # assumes that the first match is the one we want. Hopefully
+            # our regexes can be clever enough to make this work
+            return match.group(1)
+        return h
+    return g
diff --git a/wizard/app/mediawiki.py b/wizard/app/mediawiki.py
index 9931b1b..7ae7583 100644
--- a/wizard/app/mediawiki.py
+++ b/wizard/app/mediawiki.py
@@ -1,20 +1,26 @@
 import re
 
-from wizard import deploy
+from wizard import app, deploy, util
+from wizard.app import php
 
-def make_regex(var):
-    return re.compile('^\$' + var + r'''\s*=\s*((["\']).*\2;)$''', re.M)
-
-wizard_to_var = \
-        {'WIZARD_IP': 'IP' # obsolete
-        ,'WIZARD_SITENAME': 'wgSitename'
-        ,'WIZARD_SCRIPTPATH': 'wgScriptPath'
-        ,'WIZARD_EMERGENCYCONTACT': 'wgEmergencyContact'
-        ,'WIZARD_DBSERVER': 'wgDBserver'
-        ,'WIZARD_DBNAME': 'wgDBname'
-        ,'WIZARD_DBUSER': 'wgDBuser'
-        ,'WIZARD_DBPASSWORD': 'wgDBpassword'
-        ,'WIZARD_PROXYKEY': 'wgProxyKey'}
+@app.filename_regex_extractor
+def make_extractor(var):
+    return 'LocalSettings.php', re.compile('^\$' + re.escape(var) + r'''\s*=\s*((["\']).*\2);$''', re.M)
 
 class Application(deploy.Application):
-    pass
+    @property
+    def extractors(self):
+        if not self._extractors:
+            self._extractors = util.dictmap(make_extractor,
+                {'WIZARD_IP': 'IP' # obsolete
+                ,'WIZARD_SITENAME': 'wgSitename'
+                ,'WIZARD_SCRIPTPATH': 'wgScriptPath'
+                ,'WIZARD_EMERGENCYCONTACT': 'wgEmergencyContact'
+                ,'WIZARD_DBSERVER': 'wgDBserver'
+                ,'WIZARD_DBNAME': 'wgDBname'
+                ,'WIZARD_DBUSER': 'wgDBuser'
+                ,'WIZARD_DBPASSWORD': 'wgDBpassword'
+                ,'WIZARD_PROXYKEY': 'wgProxyKey'
+                })
+            self._extractors.update(php.extractors)
+        return self._extractors
diff --git a/wizard/app/php.py b/wizard/app/php.py
index 0d9fcfd..9093edf 100644
--- a/wizard/app/php.py
+++ b/wizard/app/php.py
@@ -1,3 +1,13 @@
-php_ini_regexes = \
-        {'WIZARD_SESSIONNAME': None,
-         'WIZARD_TMPDIR': None}
+import re
+
+from wizard import app, util
+
+@app.filename_regex_extractor
+def make_extractor(var):
+    return 'php.ini', re.compile('^' + re.escape(var) + r'\s*=\s*(.*)$', re.M)
+
+extractors = util.dictmap(make_extractor,
+        {'WIZARD_SESSIONNAME': 'session.name'
+        ,'WIZARD_TMPDIR': 'upload_tmp_dir'
+        })
+
diff --git a/wizard/app/tests/mediawiki/LocalSettings.php b/wizard/app/tests/mediawiki/LocalSettings.php
new file mode 100644
index 0000000..433adcb
--- /dev/null
+++ b/wizard/app/tests/mediawiki/LocalSettings.php
@@ -0,0 +1,128 @@
+<?php
+
+# This file was automatically generated by the MediaWiki installer.
+# If you make manual changes, please keep track in case you need to
+# recreate them later.
+#
+# See includes/DefaultSettings.php for all configurable settings
+# and their default values, but don't forget to make changes in _this_
+# file, not there.
+
+# If you customize your file layout, set $IP to the directory that contains
+# the other MediaWiki files. It will be used as a base to locate files.
+if( defined( 'MW_INSTALL_PATH' ) ) {
+	$IP = MW_INSTALL_PATH;
+} else {
+	$IP = dirname( __FILE__ );
+}
+
+$path = array( $IP, "$IP/includes", "$IP/languages" );
+set_include_path( implode( PATH_SEPARATOR, $path ) . PATH_SEPARATOR . get_include_path() );
+
+require_once( "$IP/includes/DefaultSettings.php" );
+
+# If PHP's memory limit is very low, some operations may fail.
+# ini_set( 'memory_limit', '20M' );
+
+if ( $wgCommandLineMode ) {
+	if ( isset( $_SERVER ) && array_key_exists( 'REQUEST_METHOD', $_SERVER ) ) {
+		die( "This script must be run from the command line\n" );
+	}
+}
+## Uncomment this to disable output compression
+# $wgDisableOutputCompression = true;
+
+$wgSitename         = "wizard_sitename";
+
+## The URL base path to the directory containing the wiki;
+## defaults for all runtime URL paths are based off of this.
+$wgScriptPath       = "/wizard_scriptpath";
+$wgScriptExtension  = ".php";
+
+## For more information on customizing the URLs please see:
+## http://www.mediawiki.org/wiki/Manual:Short_URL
+
+$wgEnableEmail      = true;
+$wgEnableUserEmail  = true;
+
+$wgEmergencyContact = "wizard_emergencycontact";
+$wgPasswordSender = "wizard_emergencycontact";
+
+## For a detailed description of the following switches see
+## http://www.mediawiki.org/wiki/Extension:Email_notification 
+## and http://www.mediawiki.org/wiki/Extension:Email_notification
+## There are many more options for fine tuning available see
+## /includes/DefaultSettings.php
+## UPO means: this is also a user preference option
+$wgEnotifUserTalk = true; # UPO
+$wgEnotifWatchlist = true; # UPO
+$wgEmailAuthentication = true;
+
+$wgDBtype           = "mysql";
+$wgDBserver         = "wizard_dbserver";
+$wgDBname           = "wizard_dbname";
+$wgDBuser           = "wizard_dbuser";
+$wgDBpassword       = "wizard_dbpassword";
+
+# MySQL specific settings
+$wgDBprefix         = "";
+
+# MySQL table options to use during installation or update
+$wgDBTableOptions   = "TYPE=InnoDB";
+
+# Experimental charset support for MySQL 4.1/5.0.
+$wgDBmysql5 = false;
+
+# Postgres specific settings
+$wgDBport           = "5432";
+$wgDBmwschema       = "mediawiki";
+$wgDBts2schema      = "public";
+
+## Shared memory settings
+$wgMainCacheType = CACHE_NONE;
+$wgMemCachedServers = array();
+
+## To enable image uploads, make sure the 'images' directory
+## is writable, then set this to true:
+$wgEnableUploads       = false;
+$wgUseImageMagick = true;
+$wgImageMagickConvertCommand = "/usr/bin/convert";
+
+## If you want to use image uploads under safe mode,
+## create the directories images/archive, images/thumb and
+## images/temp, and make them all writable. Then uncomment
+## this, if it's not already uncommented:
+# $wgHashedUploadDirectory = false;
+
+## If you have the appropriate support software installed
+## you can enable inline LaTeX equations:
+$wgUseTeX           = false;
+
+$wgLocalInterwiki   = $wgSitename;
+
+$wgLanguageCode = "en";
+
+$wgProxyKey = "wizard_proxykey";
+
+## Default skin: you can change the default skin. Use the internal symbolic
+## names, ie 'standard', 'nostalgia', 'cologneblue', 'monobook':
+$wgDefaultSkin = 'monobook';
+
+## For attaching licensing metadata to pages, and displaying an
+## appropriate copyright notice / icon. GNU Free Documentation
+## License and Creative Commons licenses are supported so far.
+# $wgEnableCreativeCommonsRdf = true;
+$wgRightsPage = ""; # Set to the title of a wiki page that describes your license/copyright
+$wgRightsUrl = "";
+$wgRightsText = "";
+$wgRightsIcon = "";
+# $wgRightsCode = ""; # Not yet used
+
+$wgDiff3 = "/usr/bin/diff3";
+
+# When you make changes to this configuration file, this will make
+# sure that cached pages are cleared.
+$configdate = gmdate( 'YmdHis', @filemtime( __FILE__ ) );
+$wgCacheEpoch = max( $wgCacheEpoch, $configdate );
+define('MEDIAWIKI_NO_OUTPUT_COMPRESSION', true);
+$wgUseGzip = false;
diff --git a/wizard/app/tests/mediawiki/php.ini b/wizard/app/tests/mediawiki/php.ini
new file mode 100644
index 0000000..8a0cb29
--- /dev/null
+++ b/wizard/app/tests/mediawiki/php.ini
@@ -0,0 +1,26 @@
+extension = gd.so
+extension = mysql.so
+zlib.output_compression = on
+zlib.output_compression_level = 6
+session.name = wizard_SID
+session.save_path = /mit/wizard/web_scripts_tmp
+upload_tmp_dir = /mit/wizard/web_scripts_tmp
+register_long_arrays = on
+cgi.fix_pathinfo = 1
+memory_limit = 1024M
+max_execution_time = 60
+expose_php = off
+output_buffering = 4096
+implicit_flush = Off
+error_reporting  =  E_ALL & ~E_NOTICE & ~E_WARNING
+display_errors = On
+display_startup_errors = Off
+register_globals = Off
+register_long_arrays = Off
+register_argc_argv = On
+post_max_size = 1024M
+magic_quotes_gpc = Off
+upload_max_filesize = 1024M
+log_errors = Off
+[MySQL]
+mysql.default_host = sql.mit.edu
diff --git a/wizard/app/tests/test_mediawiki.py b/wizard/app/tests/test_mediawiki.py
new file mode 100644
index 0000000..6834df5
--- /dev/null
+++ b/wizard/app/tests/test_mediawiki.py
@@ -0,0 +1,21 @@
+import os.path
+
+from wizard import deploy
+from wizard.app import mediawiki
+
+def test_extract():
+    app = mediawiki.Application('mediawiki')
+    d = deploy.Deployment(os.path.join(os.path.dirname(os.path.abspath(__file__)), "mediawiki"))
+    result = app.extract(d)
+    assert result['WIZARD_PROXYKEY'] == '"wizard_proxykey"'
+    assert result['WIZARD_SCRIPTPATH'] == '"/wizard_scriptpath"'
+    assert result['WIZARD_SESSIONNAME'] == 'wizard_SID'
+    assert result['WIZARD_DBSERVER'] == '"wizard_dbserver"'
+    assert result['WIZARD_DBUSER'] == '"wizard_dbuser"'
+    assert result['WIZARD_SITENAME'] == '"wizard_sitename"'
+    assert result['WIZARD_DBPASSWORD'] == '"wizard_dbpassword"'
+    assert result['WIZARD_EMERGENCYCONTACT'] == '"wizard_emergencycontact"'
+    assert result['WIZARD_IP'] is None
+    assert result['WIZARD_TMPDIR'] == '/mit/wizard/web_scripts_tmp'
+    assert result['WIZARD_DBNAME'] == '"wizard_dbname"'
+
diff --git a/wizard/deploy.py b/wizard/deploy.py
index b6d6f66..7908f03 100644
--- a/wizard/deploy.py
+++ b/wizard/deploy.py
@@ -27,6 +27,14 @@ class Deployment(object):
         self.location = location
         self._version = version
         self._log = log
+        self._read_cache = {}
+    def read(self, file, force = False):
+        """Reads a file's contents and stuffs it in a cache"""
+        if force or file not in self._read_cache:
+            f = open(os.path.join(self.location, file))
+            self._read_cache[file] = f.read()
+            f.close()
+        return self._read_cache[file]
     @property
     def version_file(self):
         return os.path.join(self.location, '.scripts-version')
@@ -65,6 +73,7 @@ class Application(object):
     def __init__(self, name):
         self.name = name
         self.versions = {}
+        self._extractors = {}
     @property
     def repository(self):
         """Returns the Git repository that would contain this application."""
@@ -76,6 +85,15 @@ class Application(object):
         if version not in self.versions:
             self.versions[version] = ApplicationVersion(distutils.version.LooseVersion(version), self)
         return self.versions[version]
+    def extract(self, deployment):
+        """Extracts wizard variables from a deployment."""
+        result = {}
+        for k,extractor in self.extractors.items():
+            result[k] = extractor(deployment)
+        return result
+    @property
+    def extractors(self):
+        return {}
     @staticmethod
     def make(name):
         """Makes an application, but uses the correct subtype if available."""
diff --git a/wizard/tests/log_test.py b/wizard/tests/log_test.py
index a59de34..f84fb3b 100644
--- a/wizard/tests/log_test.py
+++ b/wizard/tests/log_test.py
@@ -5,7 +5,7 @@ from datetime import datetime
 from wizard import deploy, log
 
 def getTestFile(file):
-    return os.path.realpath(os.path.join(__file__, "..", file))
+    return os.path.join(os.path.dirname(os.path.abspath(__file__)), file)
 
 def test_deploy_log_load():
     # this also is test_deploy_source_parse() and test_application_version_parse()
diff --git a/wizard/tests/util_test.py b/wizard/tests/util_test.py
index c34abba..7907bff 100644
--- a/wizard/tests/util_test.py
+++ b/wizard/tests/util_test.py
@@ -9,6 +9,9 @@ class MyError(Exception):
 ERROR: Foo
 """
 
+def test_dictmap():
+    assert dictmap(lambda x: x + 1, {'a': 0, 'b': 1}) == {'a': 1, 'b': 2}
+
 def test_get_dir_uid():
     if os.getuid(): return # only run if on a scripts server. This is crude
     assert get_dir_uid("/mit/ezyang/web_scripts/test-wiki") == 537864399
diff --git a/wizard/util.py b/wizard/util.py
index 5c5a631..08a6a66 100644
--- a/wizard/util.py
+++ b/wizard/util.py
@@ -17,6 +17,11 @@ class ChangeDirectory(object):
     def __exit__(self, *args):
         os.chdir(self.olddir)
 
+def dictmap(f, d):
+    """A map function for dictionaries.  Does not allow changing keys, only
+    values"""
+    return dict((k,f(v)) for k,v in d.items())
+
 def get_exception_name(output):
     """Reads the stderr output of another Python command and grabs the
     fully qualified exception name"""