source: trunk/server/doc/install-howto.sh @ 1699

Last change on this file since 1699 was 1699, checked in by ezyang, 12 years ago
Word of warning about PHP sessions.
File size: 16.2 KB
Line 
1# This document is a how-to for installing a Fedora scripts.mit.edu server.
2# It is semi-vaguely in the form of a shell script, but is not really
3# runnable as it stands.
4
5set -e -x
6
7# Some commands should be run as the scripts-build user, not root.
8
9alias asbuild="sudo -u scripts-build"
10
11# Old versions of this install document advised setting
12# NSS_NONLOCAL_IGNORE=1 anytime you're setting up anything, e.g. using
13# yum, warning that useradd will query LDAP in a stupid way that makes
14# it hang forever.  As of Fedora 13, this does not seem to be a problem,
15# so it's been removed from the instructions.  If an install is hanging,
16# though, try adding NSS_NONLOCAL_IGNORE.
17
18# This is actually just "pick an active scripts server".  It can't be
19# scripts.mit.edu because our networking config points that domain
20# at localhost, and if our server is not setup at that point things
21# will break.
22source_server="cats-whiskers.mit.edu"
23
24# 'branch' is the current svn branch you are on.  You want to
25# use trunk if your just installing a new server, and branches/fcXX-dev
26# if your preparing a server on a new Fedora release.
27branch="trunk"
28
29# 'server' is the public hostname of your server, for SCP'ing files
30# to and from.
31server=YOUR-SERVER-NAME-HERE
32
33# Start with a Scripts kickstarted install of Fedora (install-fedora)
34
35# Take updates, reboot if there's a kernel update.
36    yum update -y
37
38# Get rid of network manager
39    yum remove NetworkManager
40
41# Copy over root's dotfiles from one of the other machines.
42# Perhaps a useful change is to remove the default aliases
43    cd /root
44    ls -l .bashrc
45    ls -l .ldapvirc
46    ls -l .screenrc
47    ls -l .ssh
48    ls -l .vimrc
49    ls -l .k5login
50    # Trying to scp from server to server won't work, as scp
51    # will attempt to negotiate a server-to-server connection.
52    # Instead, scp to your trusted machine as a temporary file,
53    # and then push to the other server
54scp -r root@$source_server:~/{.bashrc,.ldapvirc,.screenrc,.ssh,.vimrc,.k5login} .
55scp -r {.bashrc,.ldapvirc,.screenrc,.ssh,.vimrc,.k5login} root@$server:~
56
57# Install the initial set of credentials (to get Kerberized logins once
58# krb5 is installed).  Otherwise, SCP'ing things in will be annoying.
59#   o You probably installed the machine keytab long ago
60    ls -l /etc/krb5.keytab
61#     Use ktutil to combine the host/scripts.mit.edu and
62#     host/scripts-vhosts.mit.edu keys with host/this-server.mit.edu in
63#     the keytab.  Do not use 'k5srvutil change' on the combined keytab
64#     or you'll break the other servers. (real servers only).  Be
65#     careful about writing out the keytab: if you write it to an
66#     existing file the keys will just get appended.  The correct
67#     credential list should look like:
68#       ktutil:  l
69#       slot KVNO Principal
70#       ---- ---- ---------------------------------------------------------------------
71#          1    5 host/old-faithful.mit.edu@ATHENA.MIT.EDU
72#          2    3 host/scripts-vhosts.mit.edu@ATHENA.MIT.EDU
73#          3    2      host/scripts.mit.edu@ATHENA.MIT.EDU
74#   o Replace the ssh host keys with the ones common to all scripts servers (real servers only)
75    ls -l /etc/ssh/*key*
76#     You can do that with:
77scp root@$source_server:/etc/ssh/*key* .
78scp *key* root@$server:/etc/ssh/
79    service sshd reload
80
81# Check out the scripts /etc configuration
82    # backslash to make us not use the alias
83    cd /root
84    \cp -a etc /
85    chmod 0440 /etc/sudoers
86
87# If this is the first time you've installed this hostname, you will
88# need to update a bunch of files to add support for it. These include:
89#   o Adding all aliases to /etc/httpd/conf.d/scripts-vhost-names.conf
90#     (usually this is hostname, hostname.mit.edu, h-n, h-n.mit.edu,
91#     scriptsN, scriptsN.mit.edu, and the IP address.)
92#   o Adding routing rules for the static IP in
93#     /etc/sysconfig/network-scripts/route-eth1
94#   o Adding the IP address to the hosts file (same hosts as for
95#     scripts-vhost-names)
96#   o Put the hostname information in LDAP so SVN and Git work
97#   o Set up Nagios monitoring on sipb-noc for the host
98#   o Set up the host as in the pool on r-b/r-b /etc/heartbeat/ldirectord.cf
99    XXX TODO COMMANDS
100
101# NOTE: You will have just lost DNS resolution and the ability
102# to do password SSH in.  If you managed to botch this step without
103# having named setup, you can do a quick fix by frobbing /etc/resolv.conf
104# with a non 127.0.0.1 address for the DNS server.  Be sure to revert it once
105# you have named.
106
107# NOTE: You can get password SSH back by editing /etc/ssh/sshd_config (allow
108# password auth) and /etc/pam.d/sshd (comment out the first three auth
109# lines).  However, you should have the Kerberos credentials in place
110# so as soon as you install the full set of Scripts packages, you'll get
111# Kerberized logins.
112
113# Make sure network is working.  If this is a new server name, you'll
114# need to add it to /etc/hosts and
115# /etc/sysconfig/network-scripts/route-eth1.  Kickstart should have
116# configured eth0 and eth1 correctly; use service network restart
117# to add the new routes in route-eth1.
118    service network restart
119    route
120    ifconfig
121    cat /etc/hosts
122    cat /etc/sysconfig/network-scripts/route-eth1
123
124# This is the point at which you should start updating scriptsified
125# packages for a new Fedora release.  Consult 'upgrade-tips' for more
126# information.
127    yum install -y scripts-base
128    # Some of these packages are naughty and clobber some of our files
129    cd /etc
130    svn revert resolv.conf hosts sysconfig/openafs
131
132# Replace rsyslog with syslog-ng by doing:
133    rpm -e --nodeps rsyslog
134    yum install -y syslog-ng
135    chkconfig syslog-ng on
136
137# Fix the openafs /usr/vice/etc <-> /etc/openafs mapping.
138    echo "/afs:/usr/vice/cache:10000000" > /usr/vice/etc/cacheinfo
139    echo "athena.mit.edu" > /usr/vice/etc/ThisCell
140
141# [TEST SERVER] If you're installing a test server, this needs to be
142# much smaller; the max filesize on XVM is 10GB.  Pick something like
143# 500000. Also, some of the AFS parameters are kind of retarded (and if
144# you're low on disk space, will actually exhaust our inodes).  Edit
145# these parameters in /etc/sysconfig/openafs
146
147# Test that zephyr is working
148    chkconfig zhm on
149    service zhm start
150    echo 'Test!' | zwrite -d -c scripts -i test
151
152# Install the full list of RPMs that users expect to be on the
153# scripts.mit.edu servers.
154rpm -qa --queryformat "%{Name}.%{Arch}\n" | sort > packages.txt
155# arrange for packages.txt to be passed to the server, then run:
156# --skip-broken will (usually) prevent you from having to sit through
157# several minutes of dependency resolution until it decides that
158# it can't install /one/ package.
159    yum install -y --skip-broken $(cat packages.txt)
160
161# Make sure sendmail isn't installed
162    yum remove sendmail
163
164# Check which packages are installed on your new server that are not
165# in the snapshot, and remove ones that aren't needed for some reason
166# on the new machine.  Otherwise, aside from bloat, you may end up
167# with undesirable things for security, like sendmail.
168    rpm -qa --queryformat "%{Name}.%{Arch}\n" | grep -v kernel | sort > newpackages.txt
169    diff -u packages.txt newpackages.txt | grep -v kernel | less
170    # here's a cute script that removes all extra packages
171    yum erase -y $(grep -Fxvf packages.txt newpackages.txt)
172
173# We need an upstream version of cgi which we've packaged ourselves, but
174# it doesn't work with the haskell-platform package which expects
175# explicit versions.  So temporarily rpm -e the package, and then
176# install it again after you install haskell-platform.  [Note: You
177# probably won't need this in Fedora 15 or something, when the Haskell
178# Platform gets updated.]
179    rpm -e ghc-cgi-devel ghc-cgi
180    yum install -y haskell-platform
181    yumdownloader ghc-cgi
182    yumdownloader ghc-cgi-devel
183    rpm -i ghc-cgi*1.8.1*.rpm
184
185# Check out the scripts /usr/vice/etc configuration
186    cd /root/vice
187    \cp -a etc /usr/vice
188
189# Install the full list of perl modules that users expect to be on the
190# scripts.mit.edu servers.
191    cd /root
192    export PERL_MM_USE_DEFAULT=1
193    cpan # this is interactive, enter the next two lines
194        o conf prerequisites_policy follow
195        o conf commit
196# on a reference server
197perldoc -u perllocal | grep head2 | cut -f 3 -d '<' | cut -f 1 -d '|' | sort -u | perl -ne 'chomp; print "notest install $_\n" if system("rpm -q --whatprovides \"perl($_)\" >/dev/null 2>/dev/null")' > perl-packages.txt
198# arrange for perl-packages.txt to be transferred to server
199    cat perl-packages.txt | perl -MCPAN -e shell
200
201# Install the Python eggs and Ruby gems and PEAR/PECL doohickeys that are on
202# the other scripts.mit.edu servers and do not have RPMs.
203# The general mode of operation will be to run the "list" command
204# on both servers, see what the differences are, check if those diffs
205# are packaged up as rpms, and install them (rpm if possible, native otherwise)
206# - Look at /usr/lib/python2.6/site-packages and
207#           /usr/lib64/python2.6/site-packages for Python eggs and modules.
208#   There will be a lot of gunk that was installed from packages;
209#   easy-install.pth in /usr/lib/ will tell you what was easy_installed.
210#   First use 'yum search' to see if the relevant package is now available
211#   as an RPM, and install that if it is.  If not, then use easy_install.
212#   Pass -Z to easy_install to install them unzipped, as some zipped eggs
213#   want to be able to write to ~/.python-eggs.  (Also makes sourcediving
214#   easier.)
215cat /usr/lib/python2.6/site-packages/easy-install.pth | grep "^./" | cut -c3- | cut -f1 -d- . egg.txt
216    cat egg.txt | xargs easy_install -Z
217# - Look at `gem list` for Ruby gems.
218#   Again, use 'yum search' and prefer RPMs, but failing that, 'gem install'.
219#       ezyang: rspec-rails depends on rspec, and will override the Yum
220#       package, so... don't use that RPM yet
221gem list --no-version > gem.txt
222    gem install $(gem list --no-version | grep -Fxvf - gem.txt)
223# - Look at `pear list` for Pear fruits (or whatever they're called).
224#   Yet again, 'yum search' for RPMs before resorting to 'pear install'.  Note
225#   that for things in the beta repo, you'll need 'pear install package-beta'.
226#   (you might get complaints about the php_scripts module; ignore them)
227pear list | tail -n +4 | cut -f 1 -d " " > pear.txt
228    pear config-set preferred_state beta
229    pear channel-update pear.php.net
230    pear install $(pear list | tail -n +4 | cut -f 1 -d " " | grep -Fxvf - pear.txt)
231# - Look at `pecl list` for PECL things.  'yum search', and if you must,
232#   'pecl install' needed items. If it doesn't work, try 'pear install
233#   pecl/foo' or 'pecl install foo-beta' or those two combined.
234pecl list | tail -n +4 | cut -f 1 -d " " > pecl.txt
235    pecl install --nodeps $(pecl list | tail -n +4 | cut -f 1 -d " " | grep -Fxvf - pecl.txt)
236
237# Setup some Python config
238    echo 'import site, os.path; site.addsitedir(os.path.expanduser("~/lib/python2.6/site-packages"))' > /usr/lib/python2.6/site-packages/00scripts-home.pth
239
240# Install the credentials.  There are a lot of things to remember here.
241# Be sure to make sure the permissions match up (ls -l on an existing
242# server!).
243scp root@$source_server:{/etc/{sql-mit-edu.cfg.php,daemon.keytab,pki/tls/private/scripts.key,signup-ldap-pw,whoisd-password},/home/logview/.k5login} .
244scp daemon.keytab signup-ldap-pw whoisd-password sql-mit-edu.cfg.php root@$server:/etc
245scp scripts.key root@$server:/etc/pki/tls/private
246scp .k5login root@$server:/home/logview
247    chown afsagent:afsagent /etc/daemon.keytab
248#   o The daemon.scripts keytab (will be daemon.scripts-test for test)
249    ls -l /etc/daemon.keytab
250#   o The SSL cert private key (real servers only)
251    ls -l /etc/pki/tls/private/scripts.key
252#   o The LDAP password for the signup process (real servers only)
253    ls -l /etc/signup-ldap-pw
254#   o The whoisd password (real servers only)
255    ls -l /etc/whoisd-password
256#   o Make sure logview's .k5login is correct (real servers only)
257    cat /home/logview/.k5login
258
259# Spin up OpenAFS.  This will fail if there's been a new kernel since
260# when you last tried.  In that case, you can hold on till later to
261# start OpenAFS.  This will take a little bit of time;
262    service openafs-client start
263
264# Check that fs sysname is correct.  You should see, among others,
265# 'amd64_fedoraX_scripts' (vary X) and 'scripts'. If it's not, you
266# probably did a distro upgrade and should update /etc/sysconfig/openafs.
267    fs sysname
268
269# [TEST SERVER] If you are setting up a test server, pay attention to
270# /etc/sysconfig/network-scripts and do not bind scripts' IP address.
271# You will also need to modify:
272#   o /etc/ldap.conf
273#       add: host scripts.mit.edu
274#   o /etc/nss-ldapd.conf
275#       replace: uri *****
276#       with: uri ldap://scripts.mit.edu/
277#   o /etc/openldap/ldap.conf
278#       add: URI ldap://scripts.mit.edu/
279#            BASE dc=scripts,dc=mit,dc=edu
280#   o /etc/httpd/conf.d/vhost_ldap.conf
281#       replace: VhostLDAPUrl ****
282#       with: VhostLDAPUrl "ldap://scripts.mit.edu/ou=VirtualHosts,dc=scripts,dc=mit,dc=edu"
283#   o /etc/postfix/virtual-alias-{domains,maps}-ldap.cf
284#       replace: server_host *****
285#       with: server_host = ldap://scripts.mit.edu
286# to use scripts.mit.edu instead of localhost.
287# XXX: someone should write sed scripts to do this
288
289# [TEST SERVER] If you are setting up a test server, afsagent's cronjob
290# will attempt to be renewing with the wrong credentials
291# (daemon.scripts). Change this:
292    vim /home/afsagent/renew # replace all mentions of daemon.scripts.mit.edu
293
294# Set up replication (see ./install-ldap).
295# You'll need the LDAP keytab for this server: be sure to chown it
296# fedora-ds after you create the fedora-ds user
297    ls -l /etc/dirsrv/keytab
298    cat install-ldap
299
300# Make the services dirsrv, nslcd, nscd, postfix, and httpd start at
301# boot. Run chkconfig to make sure the set of services to be run is
302# correct.
303    service nslcd start
304    service nscd start
305    service postfix start
306    service httpd start
307    chkconfig dirsrv on
308    chkconfig nslcd on
309    chkconfig nscd on
310    chkconfig postfix on
311    chkconfig httpd on
312
313# nrpe is required for nagios alerts
314    chkconfig nrpe on
315
316# Check sql user credentials (needs to be done after LDAP is setup)
317    chown sql /etc/sql-mit-edu.cfg.php
318
319# Postfix doesn't actually deliver mail; fix this
320    cd /etc/postfix
321    postmap virtual
322
323# Munin might not be monitoring packages that were installed after it
324    munin-node-configure --suggest --shell | sh
325
326# Run fmtutil-sys --all, which does something that makes TeX work.
327# (Note: this errors on XeTeX which is ok.)
328    fmtutil-sys --all
329
330# Ensure that PHP isn't broken:
331    mkdir /tmp/sessions
332    chmod 01777 /tmp/sessions
333    # XXX: this seems to get deleted if tmp gets cleaned up, so we
334    # might need something a little better (maybe init script.)
335
336# Ensure fcgid isn't broken (should be 755)
337    ls -ld /var/run/mod_fcgid
338
339# Fix etc by making sure none of our config files got overwritten
340    cd /etc
341    svn status -q
342    # Some usual candidates for clobbering include nsswitch.conf and
343    # sysconfig/openafs
344
345# ThisCell got clobbered, replace it with athena.mit.edu
346    echo "athena.mit.edu" > /usr/vice/etc/ThisCell
347
348# Reboot the machine to restore a consistent state, in case you
349# changed anything. (Note: Starting kdump fails (this is ok))
350
351# [OPTIONAL] Your machine's hostname is baked in at install time;
352# in the rare case you need to change it: it appears to be in:
353#   o /etc/sysconfig/network
354#   o your lvm thingies; probably don't need to edit
355
356# [TEST SERVER] More stuff for test servers
357#   - You need a self-signed SSL cert.  Generate with:
358    openssl req -new -x509 -keyout /etc/pki/tls/private/scripts.key -out /etc/pki/tls/certs/scripts.cert -nodes
359#     Also make /etc/pki/tls/certs/ca.pem match up
360#   - Make (/etc/aliases) root mail go to /dev/null, so we don't spam people
361#   - Edit /etc/httpd/conf.d/scripts-vhost-names.conf to have scripts-fX-test.xvm.mit.edu
362#     be an accepted vhost name
363#   - Look at the old test server and see what config changes are floating around
364
365# XXX: our SVN checkout should be updated to use scripts.mit.edu
366# (repository and etc) once serving actually works.
367    cd /etc
368    svn switch --relocate svn://$source_server/ svn://scripts.mit.edu/
369    cd /usr/vice/etc
370    svn switch --relocate svn://$source_server/ svn://scripts.mit.edu/
371    cd /srv/repository
372    asbuild svn switch --relocate svn://$source_server/ svn://scripts.mit.edu/
373    asbuild svn up # verify scripts.mit.edu works
Note: See TracBrowser for help on using the repository browser.