source: branches/fc15-dev/server/doc/install-howto.sh @ 1982

Last change on this file since 1982 was 1982, checked in by ezyang, 13 years ago
Readd 'make install-deps' to instructions.
File size: 18.7 KB
Line 
1# This document is a how-to for installing a Fedora scripts.mit.edu server.
2# It is semi-vaguely in the form of a shell script, but is not really
3# runnable as it stands.
4
5# Notation
6# [PRODUCTION] Production server that will be put into the pool
7# [WIZARD]     Semi-production server that will only have
8#              daemon.scripts-security-upd bits, among other
9#              restricted permissions bits, among other
10#              restricted permissions bits, among other
11#              restricted permissions bits, among other
12#              restricted permissions
13# [TESTSERVER] Completely untrusted server
14
15set -e -x
16
17# Some commands should be run as the scripts-build user, not root.
18
19alias asbuild="sudo -u scripts-build"
20
21# Old versions of this install document advised setting
22# NSS_NONLOCAL_IGNORE=1 anytime you're setting up anything, e.g. using
23# yum, warning that useradd will query LDAP in a stupid way that makes
24# it hang forever.  As of Fedora 13, this does not seem to be a problem,
25# so it's been removed from the instructions.  If an install is hanging,
26# though, try adding NSS_NONLOCAL_IGNORE.
27
28# This is actually just "pick an active scripts server".  It can't be
29# scripts.mit.edu because our networking config points that domain
30# at localhost, and if our server is not setup at that point things
31# will break.
32source_server="shining-armor.mit.edu"
33
34# 'branch' is the current svn branch you are on.  You want to
35# use trunk if your just installing a new server, and branches/fcXX-dev
36# if your preparing a server on a new Fedora release.
37branch="trunk"
38
39# 'server' is the public hostname of your server, for SCP'ing files
40# to and from.
41server=YOUR-SERVER-NAME-HERE
42
43# Start with a Scripts kickstarted install of Fedora (install-fedora)
44
45# Take updates, reboot if there's a kernel update.
46    yum update -y
47
48# Install compilation deps
49    cd /srv/repository/server/fedora
50    make install-deps
51
52# Get rid of network manager
53    yum remove NetworkManager
54
55# Copy over root's dotfiles from one of the other machines.
56# Perhaps a useful change is to remove the default aliases
57    cd /root
58    ls -l .bashrc
59    ls -l .screenrc
60    ls -l .ssh
61    ls -l .vimrc
62    ls -l .k5login
63    # [PRODUCTION] This rc file has sensitive data on it and should only
64    # be pushed onto production servers.
65    ls -l .ldapvirc
66    # Trying to scp from server to server won't work, as scp
67    # will attempt to negotiate a server-to-server connection.
68    # Instead, scp to your trusted machine as a temporary file,
69    # and then push to the other server
70scp -r root@$source_server:~/{.bashrc,.screenrc,.ssh,.vimrc,.k5login} .
71scp -r {.bashrc,.screenrc,.ssh,.vimrc,.k5login} root@$server:~
72# [PRODUCTION]
73scp root@$source_server:~/.ldapvirc .
74scp .ldapvirc root@$server:~
75
76# Install the initial set of credentials (to get Kerberized logins once
77# krb5 is installed).  Otherwise, SCP'ing things in will be annoying.
78# Note that if you don't feel like forcefully ignoring "public key
79# changed" errors, all you need to transfer is /etc/ssh/ssh_host_rsa_key
80# initially.
81#   o Install the machine keytab.
82    ls -l /etc/krb5.keytab
83#     Use ktutil to combine the host/scripts.mit.edu and
84#     host/scripts-vhosts.mit.edu keys with host/this-server.mit.edu in
85#     the keytab.  Do not use 'k5srvutil change' on the combined keytab
86#     or you'll break the other servers. (real servers only).  Be
87#     careful about writing out the keytab: if you write it to an
88#     existing file the keys will just get appended.  The correct
89#     credential list should look like:
90#       ktutil:  l
91#       slot KVNO Principal
92#       ---- ---- ---------------------------------------------------------------------
93#          1    5 host/old-faithful.mit.edu@ATHENA.MIT.EDU
94#          2    3 host/scripts-vhosts.mit.edu@ATHENA.MIT.EDU
95#          3    2      host/scripts.mit.edu@ATHENA.MIT.EDU
96#   o [PRODUCTION] Replace the ssh host keys with the ones common to all
97#     scripts servers (real servers only)
98    ls -l /etc/ssh/*key*
99#     You can do that with:
100scp root@$source_server:/etc/ssh/*key* .
101scp *key* root@$server:/etc/ssh/
102    # Actually, this appears to be unnecessary
103    service sshd reload
104
105# Check out the scripts /etc configuration
106    # backslash to make us not use the alias
107    cd /root
108    \cp -a etc /
109    chmod 0440 /etc/sudoers
110
111# [PRODUCTION] If this is the first time you've installed this hostname,
112# you will need to update a bunch of files to add support for it. These
113# include:
114#   o Adding all aliases to /etc/httpd/conf.d/scripts-vhost-names.conf
115#     (usually this is hostname, hostname.mit.edu, h-n, h-n.mit.edu,
116#     scriptsN, scriptsN.mit.edu, and the IP address.)
117#   o Adding routing rules for the static IP in
118#     /etc/sysconfig/network-scripts/route-eth1
119#   o Adding the IP address to the hosts file (same hosts as for
120#     scripts-vhost-names)
121#   o Update SSH config at
122#       - server/fedora/config/etc/ssh/shosts.equiv
123#       - server/fedora/config/etc/ssh/ssh_known_hosts
124#       - server/fedora/config/etc/ssh/sshd_config : DenyUsers
125#     (the last part is critical to ensure that rooting one server
126#     doesn't give you root to all the other servers)
127#   o Put the hostname information in LDAP so SVN and Git work
128#   o Set up Nagios monitoring on sipb-noc for the host
129#   o Set up the host as in the pool on r-b/r-b /etc/heartbeat/ldirectord.cf
130    XXX TODO COMMANDS
131
132# NOTE: You will have just lost DNS resolution and the ability
133# to do password SSH in.  If you managed to botch this step without
134# having named setup, you can do a quick fix by frobbing /etc/resolv.conf
135# with a non 127.0.0.1 address for the DNS server.  Be sure to revert it once
136# you have named.
137
138# NOTE: You can get password SSH back by editing /etc/ssh/sshd_config (allow
139# password auth) and /etc/pam.d/sshd (comment out the first three auth
140# lines).  However, you should have the Kerberos credentials in place
141# so as soon as you install the full set of Scripts packages, you'll get
142# Kerberized logins.
143
144# Make sure network is working.  If this is a new server name, you'll
145# need to add it to /etc/hosts and
146# /etc/sysconfig/network-scripts/route-eth1.  Kickstart should have
147# configured eth0 and eth1 correctly; use service network restart
148# to add the new routes in route-eth1.
149    service network restart
150    route
151    ifconfig
152    cat /etc/hosts
153    cat /etc/sysconfig/network-scripts/route-eth1
154
155# This is the point at which you should start updating scriptsified
156# packages for a new Fedora release.  Consult 'upgrade-tips' for more
157# information.
158    yum install -y scripts-base
159    # Some of these packages are naughty and clobber some of our files
160    cd /etc
161    svn revert resolv.conf hosts sysconfig/openafs
162
163# Replace rsyslog with syslog-ng by doing:
164    rpm -e --nodeps rsyslog
165    yum install -y syslog-ng
166    chkconfig syslog-ng on
167
168# [PRODUCTION/WIZARD] Fix the openafs /usr/vice/etc <-> /etc/openafs
169# mapping.
170    echo "/afs:/usr/vice/cache:10000000" > /usr/vice/etc/cacheinfo
171    echo "athena.mit.edu" > /usr/vice/etc/ThisCell
172
173# [TESTSERVER] If you're installing a test server, this needs to be
174# much smaller; the max filesize on XVM is 10GB.  Pick something like
175# 500000. Also, some of the AFS parameters are kind of retarded (and if
176# you're low on disk space, will actually exhaust our inodes).  Edit
177# these parameters in /etc/sysconfig/openafs
178    echo "/afs:/usr/vice/cache:500000" > /usr/vice/etc/cacheinfo
179    XXX TODO COMMANDS
180
181# Test that zephyr is working
182    chkconfig zhm on
183    service zhm start
184    echo 'Test!' | zwrite -d -c scripts -i test
185
186# Install the full list of RPMs that users expect to be on the
187# scripts.mit.edu servers.
188rpm -qa --queryformat "%{Name}.%{Arch}\n" | sort > packages.txt
189# arrange for packages.txt to be passed to the server, then run:
190# --skip-broken will (usually) prevent you from having to sit through
191# several minutes of dependency resolution until it decides that
192# it can't install /one/ package.
193    yum install -y --skip-broken $(cat packages.txt)
194
195# Make sure sendmail isn't installed
196    yum remove sendmail
197
198# Check which packages are installed on your new server that are not
199# in the snapshot, and remove ones that aren't needed for some reason
200# on the new machine.  Otherwise, aside from bloat, you may end up
201# with undesirable things for security, like sendmail.
202    rpm -qa --queryformat "%{Name}.%{Arch}\n" | grep -v kernel | sort > newpackages.txt
203    diff -u packages.txt newpackages.txt | grep -v kernel | less
204    # here's a cute script that removes all extra packages
205    yum erase -y $(grep -Fxvf packages.txt newpackages.txt)
206    # 20101208 - Mysteriously we manage to get these extra packages
207    # from kickstart: mcelog mobile-broadband-provider-info
208    # ModemManager PackageKit
209
210# We need an upstream version of cgi which we've packaged ourselves, but
211# it doesn't work with the haskell-platform package which expects
212# explicit versions.  So temporarily rpm -e the package, and then
213# install it again after you install haskell-platform.  [Note: You
214# probably won't need this in Fedora 15 or something, when the Haskell
215# Platform gets updated.]
216    rpm -e ghc-cgi-devel ghc-cgi
217    yum install -y haskell-platform
218    yumdownloader ghc-cgi
219    yumdownloader ghc-cgi-devel
220    rpm -i ghc-cgi*1.8.1*.rpm
221
222# Check out the scripts /usr/vice/etc configuration
223    cd /root/vice
224    \cp -a etc /usr/vice
225
226# Install the full list of perl modules that users expect to be on the
227# scripts.mit.edu servers.
228    cd /root
229    export PERL_MM_USE_DEFAULT=1
230    cpan # this is interactive, enter the next two lines
231        o conf prerequisites_policy follow
232        o conf commit
233# on a reference server
234perldoc -u perllocal | grep head2 | cut -f 3 -d '<' | cut -f 1 -d '|' | sort -u | perl -ne 'chomp; print "notest install $_\n" if system("rpm -q --whatprovides \"perl($_)\" >/dev/null 2>/dev/null")' > perl-packages.txt
235# arrange for perl-packages.txt to be transferred to server
236    cat perl-packages.txt | perl -MCPAN -e shell
237
238# Install the Python eggs and Ruby gems and PEAR/PECL doohickeys that are on
239# the other scripts.mit.edu servers and do not have RPMs.
240# The general mode of operation will be to run the "list" command
241# on both servers, see what the differences are, check if those diffs
242# are packaged up as rpms, and install them (rpm if possible, native otherwise)
243# - Look at /usr/lib/python2.6/site-packages and
244#           /usr/lib64/python2.6/site-packages for Python eggs and modules.
245#   There will be a lot of gunk that was installed from packages;
246#   easy-install.pth in /usr/lib/ will tell you what was easy_installed.
247#   First use 'yum search' to see if the relevant package is now available
248#   as an RPM, and install that if it is.  If not, then use easy_install.
249#   Pass -Z to easy_install to install them unzipped, as some zipped eggs
250#   want to be able to write to ~/.python-eggs.  (Also makes sourcediving
251#   easier.)
252cat /usr/lib/python2.6/site-packages/easy-install.pth | grep "^./" | cut -c3- | cut -f1 -d- > egg.txt
253    cat egg.txt | xargs easy_install -Z
254# - Look at `gem list` for Ruby gems.
255#   Again, use 'yum search' and prefer RPMs, but failing that, 'gem install'.
256#       ezyang: rspec-rails depends on rspec, and will override the Yum
257#       package, so... don't use that RPM yet
258gem list --no-version > gem.txt
259    gem install $(gem list --no-version | grep -Fxvf - gem.txt)
260# - Look at `pear list` for Pear fruits (or whatever they're called).
261#   Yet again, 'yum search' for RPMs before resorting to 'pear install'.  Note
262#   that for things in the beta repo, you'll need 'pear install package-beta'.
263#   (you might get complaints about the php_scripts module; ignore them)
264pear list | tail -n +4 | cut -f 1 -d " " > pear.txt
265    pear config-set preferred_state beta
266    pear channel-update pear.php.net
267    pear install $(pear list | tail -n +4 | cut -f 1 -d " " | grep -Fxvf - pear.txt)
268# - Look at `pecl list` for PECL things.  'yum search', and if you must,
269#   'pecl install' needed items. If it doesn't work, try 'pear install
270#   pecl/foo' or 'pecl install foo-beta' or those two combined.
271pecl list | tail -n +4 | cut -f 1 -d " " > pecl.txt
272    pecl install --nodeps $(pecl list | tail -n +4 | cut -f 1 -d " " | grep -Fxvf - pecl.txt)
273
274# [PRODUCTION] Install the credentials.  There are a lot of things to
275# remember here.  Be sure to make sure the permissions match up (ls -l
276# on an existing server!).
277scp root@$source_server:{/etc/{sql-mit-edu.cfg.php,pki/tls/private/scripts.key,signup-ldap-pw,whoisd-password},/home/logview/.k5login} .
278scp signup-ldap-pw whoisd-password sql-mit-edu.cfg.php root@$server:/etc
279scp scripts.key root@$server:/etc/pki/tls/private
280scp .k5login root@$server:/home/logview
281#   o The SSL cert private key (real servers only)
282    ls -l /etc/pki/tls/private/scripts.key
283#   o The LDAP password for the signup process (real servers only)
284    ls -l /etc/signup-ldap-pw
285#   o The whoisd password (real servers only)
286    ls -l /etc/whoisd-password
287#   o Make sure logview's .k5login is correct (real servers only)
288    cat /home/logview/.k5login
289
290# All types of servers will have an /etc/daemon.keytab file, however,
291# different types of server will have different credentials in this
292# keytab.
293#   [PRODUCTION] daemon.scripts
294#   [WIZARD]     daemon.scripts-security-upd
295#   [TESTSERVER] daemon.scripts-test
296k5srvutil list -f daemon.keytab
297scp daemon.keytab root@$server:/etc
298    chown afsagent:afsagent /etc/daemon.keytab
299#   o The daemon.scripts keytab (will be daemon.scripts-test for test)
300    ls -l /etc/daemon.keytab
301
302# Spin up OpenAFS.  This will fail if there's been a new kernel since
303# when you last tried.  In that case, you can hold on till later to
304# start OpenAFS.  This will take a little bit of time;
305    service openafs-client start
306# Then, check that fs sysname is correct.  You should see, among others,
307# 'amd64_fedoraX_scripts' (vary X) and 'scripts'. If it's not, you
308# probably did a distro upgrade and should update /etc/sysconfig/openafs.
309    fs sysname
310
311# [WIZARD/TESTSERVER] If you are setting up a non-production server,
312# there are some services that it won't provide, and you will need to
313# make it talk to a real server instead.  In particular:
314#   - We don't serve the web, so don't bind scripts.mit.edu
315#   - We don't serve LDAP, so use another server
316# This involves editing the following files:
317#   o /etc/sysconfig/network-scripts/ifcfg-lo:0
318#   o /etc/sysconfig/network-scripts/ifcfg-lo:1
319#   o /etc/sysconfig/network-scripts/ifcfg-lo:2
320#   o /etc/sysconfig/network-scripts/ifcfg-lo:3
321       \rm /etc/sysconfig/network-scripts/ifcfg-lo:{0,1,2,3}
322#   o /etc/ldap.conf
323#       add: host scripts.mit.edu
324#   o /etc/{nss-ldapd,nslcd}.conf
325#       replace: uri ldapi://%2fvar%2frun%2fdirsrv%2fslapd-scripts.socket/
326#       with: uri ldap://scripts.mit.edu/
327#   o /etc/openldap/ldap.conf
328#       add: URI ldap://scripts.mit.edu/
329#            BASE dc=scripts,dc=mit,dc=edu
330#   o /etc/httpd/conf.d/vhost_ldap.conf
331#       replace: VhostLDAPUrl "ldap://127.0.0.1/ou=VirtualHosts,dc=scripts,dc=mit,dc=edu"
332#       with: VhostLDAPUrl "ldap://scripts.mit.edu/ou=VirtualHosts,dc=scripts,dc=mit,dc=edu"
333#   o /etc/postfix/virtual-alias-{domains,maps}-ldap.cf
334#       replace: server_host ldapi://%2fvar%2frun%2fdirsrv%2fslapd-scripts.socket/
335#       with: server_host = ldap://scripts.mit.edu
336# to use scripts.mit.edu instead of localhost.
337# XXX: someone should write sed scripts to do this
338
339# [WIZARD/TESTSERVER] If you are setting up a non-production server,
340# afsagent's cronjob will attempt to be renewing with the wrong
341# credentials (daemon.scripts). Change this:
342    vim /home/afsagent/renew # replace all mentions of daemon.scripts.mit.edu
343
344# [PRODUCTION] Set up replication (see ./install-ldap).
345# You'll need the LDAP keytab for this server: be sure to chown it
346# fedora-ds after you create the fedora-ds user
347    ls -l /etc/dirsrv/keytab
348    cat install-ldap
349
350# Make the services dirsrv, nslcd, nscd, postfix, and httpd start at
351# boot. Run chkconfig to make sure the set of services to be run is
352# correct.
353    service nslcd start
354    service nscd start
355    service postfix start
356    chkconfig nslcd on
357    chkconfig nscd on
358    chkconfig postfix on
359
360# [PRODUCTION]
361    chkconfig dirsrv on
362
363# [PRODUCTION/TESTSERVER]
364# (Maybe WIZARD too once we start doing strange things to autoupgrade
365# installs behind firewalls.)
366    service httpd start # will fail if AFS is not running
367    chkconfig httpd on
368
369# nrpe is required for nagios alerts
370    chkconfig nrpe on
371
372# [PRODUCTION] Check sql user credentials (needs to be done after LDAP
373# is setup)
374    chown sql /etc/sql-mit-edu.cfg.php
375
376# Postfix doesn't actually deliver mail; fix this
377    cd /etc/postfix
378    postmap virtual
379
380# Munin might not be monitoring packages that were installed after it
381    munin-node-configure --suggest --shell | sh
382
383# Run fmtutil-sys --all, which does something that makes TeX work.
384# (Note: this errors on XeTeX which is ok.)
385    fmtutil-sys --all
386
387# Ensure that PHP isn't broken:
388    mkdir /tmp/sessions
389    chmod 01777 /tmp/sessions
390    # XXX: this seems to get deleted if tmp gets cleaned up, so we
391    # might need something a little better (maybe init script.)
392
393# Ensure fcgid isn't broken (should be 755)
394    ls -ld /var/run/mod_fcgid
395
396# Fix etc by making sure none of our config files got overwritten
397    cd /etc
398    svn status -q
399    # Some usual candidates for clobbering include nsswitch.conf and
400    # sysconfig/openafs
401    # [WIZARD/TEST] Remember that changes you made should not get
402    # reverted!
403
404# ThisCell got clobbered, replace it with athena.mit.edu
405    echo "athena.mit.edu" > /usr/vice/etc/ThisCell
406
407# Reboot the machine to restore a consistent state, in case you
408# changed anything. (Note: Starting kdump fails (this is ok))
409
410# [OPTIONAL] Your machine's hostname is baked in at install time;
411# in the rare case you need to change it: it appears to be in:
412#   o /etc/sysconfig/network
413#   o your lvm thingies; probably don't need to edit
414
415# [TESTERVER]
416#   - You need a self-signed SSL cert or Apache will refuse to start
417#     or do SSL.  Generate with:
418    openssl req -new -x509 -keyout /etc/pki/tls/private/scripts.key -out /etc/pki/tls/certs/scripts.cert -nodes
419#     Also make /etc/pki/tls/certs/ca.pem match up (XXX what's the
420#     incant for that?)
421
422# [TESTSERVER] More stuff for test servers
423#   - Make (/etc/aliases) root mail go to /dev/null, so we don't spam people
424#   - Edit /etc/httpd/conf.d/scripts-vhost-names.conf to have scripts-fX-test.xvm.mit.edu
425#     be an accepted vhost name
426#   - Look at the old test server and see what config changes are floating around
427
428# XXX: our SVN checkout should be updated to use scripts.mit.edu
429# (repository and etc) once serving actually works.
430    cd /etc
431    svn switch --relocate svn://$source_server/ svn://scripts.mit.edu/
432    cd /usr/vice/etc
433    svn switch --relocate svn://$source_server/ svn://scripts.mit.edu/
434    cd /srv/repository
435    asbuild svn switch --relocate svn://$source_server/ svn://scripts.mit.edu/
436    asbuild svn up # verify scripts.mit.edu works
Note: See TracBrowser for help on using the repository browser.