source: branches/fc15-dev/server/doc/ @ 1830

Last change on this file since 1830 was 1704, checked in by ezyang, 14 years ago
Info about adding servers to SSH config.
File size: 16.5 KB
1# This document is a how-to for installing a Fedora server.
2# It is semi-vaguely in the form of a shell script, but is not really
3# runnable as it stands.
5set -e -x
7# Some commands should be run as the scripts-build user, not root.
9alias asbuild="sudo -u scripts-build"
11# Old versions of this install document advised setting
12# NSS_NONLOCAL_IGNORE=1 anytime you're setting up anything, e.g. using
13# yum, warning that useradd will query LDAP in a stupid way that makes
14# it hang forever.  As of Fedora 13, this does not seem to be a problem,
15# so it's been removed from the instructions.  If an install is hanging,
16# though, try adding NSS_NONLOCAL_IGNORE.
18# This is actually just "pick an active scripts server".  It can't be
19# because our networking config points that domain
20# at localhost, and if our server is not setup at that point things
21# will break.
24# 'branch' is the current svn branch you are on.  You want to
25# use trunk if your just installing a new server, and branches/fcXX-dev
26# if your preparing a server on a new Fedora release.
29# 'server' is the public hostname of your server, for SCP'ing files
30# to and from.
33# Start with a Scripts kickstarted install of Fedora (install-fedora)
35# Take updates, reboot if there's a kernel update.
36    yum update -y
38# Get rid of network manager
39    yum remove NetworkManager
41# Copy over root's dotfiles from one of the other machines.
42# Perhaps a useful change is to remove the default aliases
43    cd /root
44    ls -l .bashrc
45    ls -l .ldapvirc
46    ls -l .screenrc
47    ls -l .ssh
48    ls -l .vimrc
49    ls -l .k5login
50    # Trying to scp from server to server won't work, as scp
51    # will attempt to negotiate a server-to-server connection.
52    # Instead, scp to your trusted machine as a temporary file,
53    # and then push to the other server
54scp -r root@$source_server:~/{.bashrc,.ldapvirc,.screenrc,.ssh,.vimrc,.k5login} .
55scp -r {.bashrc,.ldapvirc,.screenrc,.ssh,.vimrc,.k5login} root@$server:~
57# Install the initial set of credentials (to get Kerberized logins once
58# krb5 is installed).  Otherwise, SCP'ing things in will be annoying.
59#   o You probably installed the machine keytab long ago
60    ls -l /etc/krb5.keytab
61#     Use ktutil to combine the host/ and
62#     host/ keys with host/ in
63#     the keytab.  Do not use 'k5srvutil change' on the combined keytab
64#     or you'll break the other servers. (real servers only).  Be
65#     careful about writing out the keytab: if you write it to an
66#     existing file the keys will just get appended.  The correct
67#     credential list should look like:
68#       ktutil:  l
69#       slot KVNO Principal
70#       ---- ---- ---------------------------------------------------------------------
71#          1    5 host/
72#          2    3 host/
73#          3    2      host/
74#   o Replace the ssh host keys with the ones common to all scripts servers (real servers only)
75    ls -l /etc/ssh/*key*
76#     You can do that with:
77scp root@$source_server:/etc/ssh/*key* .
78scp *key* root@$server:/etc/ssh/
79    service sshd reload
81# Check out the scripts /etc configuration
82    # backslash to make us not use the alias
83    cd /root
84    \cp -a etc /
85    chmod 0440 /etc/sudoers
87# If this is the first time you've installed this hostname, you will
88# need to update a bunch of files to add support for it. These include:
89#   o Adding all aliases to /etc/httpd/conf.d/scripts-vhost-names.conf
90#     (usually this is hostname,, h-n,,
91#     scriptsN,, and the IP address.)
92#   o Adding routing rules for the static IP in
93#     /etc/sysconfig/network-scripts/route-eth1
94#   o Adding the IP address to the hosts file (same hosts as for
95#     scripts-vhost-names)
96#   o Update SSH config at
97#       - server/fedora/config/etc/ssh/shosts.equiv
98#       - server/fedora/config/etc/ssh/ssh_known_hosts
99#       - server/fedora/config/etc/ssh/sshd_config : DenyUsers
100#     (the last part is critical to ensure that rooting one server
101#     doesn't give you root to all the other servers)
102#   o Put the hostname information in LDAP so SVN and Git work
103#   o Set up Nagios monitoring on sipb-noc for the host
104#   o Set up the host as in the pool on r-b/r-b /etc/heartbeat/
107# NOTE: You will have just lost DNS resolution and the ability
108# to do password SSH in.  If you managed to botch this step without
109# having named setup, you can do a quick fix by frobbing /etc/resolv.conf
110# with a non address for the DNS server.  Be sure to revert it once
111# you have named.
113# NOTE: You can get password SSH back by editing /etc/ssh/sshd_config (allow
114# password auth) and /etc/pam.d/sshd (comment out the first three auth
115# lines).  However, you should have the Kerberos credentials in place
116# so as soon as you install the full set of Scripts packages, you'll get
117# Kerberized logins.
119# Make sure network is working.  If this is a new server name, you'll
120# need to add it to /etc/hosts and
121# /etc/sysconfig/network-scripts/route-eth1.  Kickstart should have
122# configured eth0 and eth1 correctly; use service network restart
123# to add the new routes in route-eth1.
124    service network restart
125    route
126    ifconfig
127    cat /etc/hosts
128    cat /etc/sysconfig/network-scripts/route-eth1
130# This is the point at which you should start updating scriptsified
131# packages for a new Fedora release.  Consult 'upgrade-tips' for more
132# information.
133    yum install -y scripts-base
134    # Some of these packages are naughty and clobber some of our files
135    cd /etc
136    svn revert resolv.conf hosts sysconfig/openafs
138# Replace rsyslog with syslog-ng by doing:
139    rpm -e --nodeps rsyslog
140    yum install -y syslog-ng
141    chkconfig syslog-ng on
143# Fix the openafs /usr/vice/etc <-> /etc/openafs mapping.
144    echo "/afs:/usr/vice/cache:10000000" > /usr/vice/etc/cacheinfo
145    echo "" > /usr/vice/etc/ThisCell
147# [TEST SERVER] If you're installing a test server, this needs to be
148# much smaller; the max filesize on XVM is 10GB.  Pick something like
149# 500000. Also, some of the AFS parameters are kind of retarded (and if
150# you're low on disk space, will actually exhaust our inodes).  Edit
151# these parameters in /etc/sysconfig/openafs
153# Test that zephyr is working
154    chkconfig zhm on
155    service zhm start
156    echo 'Test!' | zwrite -d -c scripts -i test
158# Install the full list of RPMs that users expect to be on the
159# servers.
160rpm -qa --queryformat "%{Name}.%{Arch}\n" | sort > packages.txt
161# arrange for packages.txt to be passed to the server, then run:
162# --skip-broken will (usually) prevent you from having to sit through
163# several minutes of dependency resolution until it decides that
164# it can't install /one/ package.
165    yum install -y --skip-broken $(cat packages.txt)
167# Make sure sendmail isn't installed
168    yum remove sendmail
170# Check which packages are installed on your new server that are not
171# in the snapshot, and remove ones that aren't needed for some reason
172# on the new machine.  Otherwise, aside from bloat, you may end up
173# with undesirable things for security, like sendmail.
174    rpm -qa --queryformat "%{Name}.%{Arch}\n" | grep -v kernel | sort > newpackages.txt
175    diff -u packages.txt newpackages.txt | grep -v kernel | less
176    # here's a cute script that removes all extra packages
177    yum erase -y $(grep -Fxvf packages.txt newpackages.txt)
179# We need an upstream version of cgi which we've packaged ourselves, but
180# it doesn't work with the haskell-platform package which expects
181# explicit versions.  So temporarily rpm -e the package, and then
182# install it again after you install haskell-platform.  [Note: You
183# probably won't need this in Fedora 15 or something, when the Haskell
184# Platform gets updated.]
185    rpm -e ghc-cgi-devel ghc-cgi
186    yum install -y haskell-platform
187    yumdownloader ghc-cgi
188    yumdownloader ghc-cgi-devel
189    rpm -i ghc-cgi*1.8.1*.rpm
191# Check out the scripts /usr/vice/etc configuration
192    cd /root/vice
193    \cp -a etc /usr/vice
195# Install the full list of perl modules that users expect to be on the
196# servers.
197    cd /root
198    export PERL_MM_USE_DEFAULT=1
199    cpan # this is interactive, enter the next two lines
200        o conf prerequisites_policy follow
201        o conf commit
202# on a reference server
203perldoc -u perllocal | grep head2 | cut -f 3 -d '<' | cut -f 1 -d '|' | sort -u | perl -ne 'chomp; print "notest install $_\n" if system("rpm -q --whatprovides \"perl($_)\" >/dev/null 2>/dev/null")' > perl-packages.txt
204# arrange for perl-packages.txt to be transferred to server
205    cat perl-packages.txt | perl -MCPAN -e shell
207# Install the Python eggs and Ruby gems and PEAR/PECL doohickeys that are on
208# the other servers and do not have RPMs.
209# The general mode of operation will be to run the "list" command
210# on both servers, see what the differences are, check if those diffs
211# are packaged up as rpms, and install them (rpm if possible, native otherwise)
212# - Look at /usr/lib/python2.6/site-packages and
213#           /usr/lib64/python2.6/site-packages for Python eggs and modules.
214#   There will be a lot of gunk that was installed from packages;
215#   easy-install.pth in /usr/lib/ will tell you what was easy_installed.
216#   First use 'yum search' to see if the relevant package is now available
217#   as an RPM, and install that if it is.  If not, then use easy_install.
218#   Pass -Z to easy_install to install them unzipped, as some zipped eggs
219#   want to be able to write to ~/.python-eggs.  (Also makes sourcediving
220#   easier.)
221cat /usr/lib/python2.6/site-packages/easy-install.pth | grep "^./" | cut -c3- | cut -f1 -d- . egg.txt
222    cat egg.txt | xargs easy_install -Z
223# - Look at `gem list` for Ruby gems.
224#   Again, use 'yum search' and prefer RPMs, but failing that, 'gem install'.
225#       ezyang: rspec-rails depends on rspec, and will override the Yum
226#       package, so... don't use that RPM yet
227gem list --no-version > gem.txt
228    gem install $(gem list --no-version | grep -Fxvf - gem.txt)
229# - Look at `pear list` for Pear fruits (or whatever they're called).
230#   Yet again, 'yum search' for RPMs before resorting to 'pear install'.  Note
231#   that for things in the beta repo, you'll need 'pear install package-beta'.
232#   (you might get complaints about the php_scripts module; ignore them)
233pear list | tail -n +4 | cut -f 1 -d " " > pear.txt
234    pear config-set preferred_state beta
235    pear channel-update
236    pear install $(pear list | tail -n +4 | cut -f 1 -d " " | grep -Fxvf - pear.txt)
237# - Look at `pecl list` for PECL things.  'yum search', and if you must,
238#   'pecl install' needed items. If it doesn't work, try 'pear install
239#   pecl/foo' or 'pecl install foo-beta' or those two combined.
240pecl list | tail -n +4 | cut -f 1 -d " " > pecl.txt
241    pecl install --nodeps $(pecl list | tail -n +4 | cut -f 1 -d " " | grep -Fxvf - pecl.txt)
243# Setup some Python config
244    echo 'import site, os.path; site.addsitedir(os.path.expanduser("~/lib/python2.6/site-packages"))' > /usr/lib/python2.6/site-packages/00scripts-home.pth
246# Install the credentials.  There are a lot of things to remember here.
247# Be sure to make sure the permissions match up (ls -l on an existing
248# server!).
249scp root@$source_server:{/etc/{sql-mit-edu.cfg.php,daemon.keytab,pki/tls/private/scripts.key,signup-ldap-pw,whoisd-password},/home/logview/.k5login} .
250scp daemon.keytab signup-ldap-pw whoisd-password sql-mit-edu.cfg.php root@$server:/etc
251scp scripts.key root@$server:/etc/pki/tls/private
252scp .k5login root@$server:/home/logview
253    chown afsagent:afsagent /etc/daemon.keytab
254#   o The daemon.scripts keytab (will be daemon.scripts-test for test)
255    ls -l /etc/daemon.keytab
256#   o The SSL cert private key (real servers only)
257    ls -l /etc/pki/tls/private/scripts.key
258#   o The LDAP password for the signup process (real servers only)
259    ls -l /etc/signup-ldap-pw
260#   o The whoisd password (real servers only)
261    ls -l /etc/whoisd-password
262#   o Make sure logview's .k5login is correct (real servers only)
263    cat /home/logview/.k5login
265# Spin up OpenAFS.  This will fail if there's been a new kernel since
266# when you last tried.  In that case, you can hold on till later to
267# start OpenAFS.  This will take a little bit of time;
268    service openafs-client start
270# Check that fs sysname is correct.  You should see, among others,
271# 'amd64_fedoraX_scripts' (vary X) and 'scripts'. If it's not, you
272# probably did a distro upgrade and should update /etc/sysconfig/openafs.
273    fs sysname
275# [TEST SERVER] If you are setting up a test server, pay attention to
276# /etc/sysconfig/network-scripts and do not bind scripts' IP address.
277# You will also need to modify:
278#   o /etc/ldap.conf
279#       add: host
280#   o /etc/nss-ldapd.conf
281#       replace: uri *****
282#       with: uri ldap://
283#   o /etc/openldap/ldap.conf
284#       add: URI ldap://
285#            BASE dc=scripts,dc=mit,dc=edu
286#   o /etc/httpd/conf.d/vhost_ldap.conf
287#       replace: VhostLDAPUrl ****
288#       with: VhostLDAPUrl "ldap://,dc=scripts,dc=mit,dc=edu"
289#   o /etc/postfix/virtual-alias-{domains,maps}
290#       replace: server_host *****
291#       with: server_host = ldap://
292# to use instead of localhost.
293# XXX: someone should write sed scripts to do this
295# [TEST SERVER] If you are setting up a test server, afsagent's cronjob
296# will attempt to be renewing with the wrong credentials
297# (daemon.scripts). Change this:
298    vim /home/afsagent/renew # replace all mentions of
300# Set up replication (see ./install-ldap).
301# You'll need the LDAP keytab for this server: be sure to chown it
302# fedora-ds after you create the fedora-ds user
303    ls -l /etc/dirsrv/keytab
304    cat install-ldap
306# Make the services dirsrv, nslcd, nscd, postfix, and httpd start at
307# boot. Run chkconfig to make sure the set of services to be run is
308# correct.
309    service nslcd start
310    service nscd start
311    service postfix start
312    service httpd start
313    chkconfig dirsrv on
314    chkconfig nslcd on
315    chkconfig nscd on
316    chkconfig postfix on
317    chkconfig httpd on
319# nrpe is required for nagios alerts
320    chkconfig nrpe on
322# Check sql user credentials (needs to be done after LDAP is setup)
323    chown sql /etc/sql-mit-edu.cfg.php
325# Postfix doesn't actually deliver mail; fix this
326    cd /etc/postfix
327    postmap virtual
329# Munin might not be monitoring packages that were installed after it
330    munin-node-configure --suggest --shell | sh
332# Run fmtutil-sys --all, which does something that makes TeX work.
333# (Note: this errors on XeTeX which is ok.)
334    fmtutil-sys --all
336# Ensure that PHP isn't broken:
337    mkdir /tmp/sessions
338    chmod 01777 /tmp/sessions
339    # XXX: this seems to get deleted if tmp gets cleaned up, so we
340    # might need something a little better (maybe init script.)
342# Ensure fcgid isn't broken (should be 755)
343    ls -ld /var/run/mod_fcgid
345# Fix etc by making sure none of our config files got overwritten
346    cd /etc
347    svn status -q
348    # Some usual candidates for clobbering include nsswitch.conf and
349    # sysconfig/openafs
351# ThisCell got clobbered, replace it with
352    echo "" > /usr/vice/etc/ThisCell
354# Reboot the machine to restore a consistent state, in case you
355# changed anything. (Note: Starting kdump fails (this is ok))
357# [OPTIONAL] Your machine's hostname is baked in at install time;
358# in the rare case you need to change it: it appears to be in:
359#   o /etc/sysconfig/network
360#   o your lvm thingies; probably don't need to edit
362# [TEST SERVER] More stuff for test servers
363#   - You need a self-signed SSL cert.  Generate with:
364    openssl req -new -x509 -keyout /etc/pki/tls/private/scripts.key -out /etc/pki/tls/certs/scripts.cert -nodes
365#     Also make /etc/pki/tls/certs/ca.pem match up
366#   - Make (/etc/aliases) root mail go to /dev/null, so we don't spam people
367#   - Edit /etc/httpd/conf.d/scripts-vhost-names.conf to have
368#     be an accepted vhost name
369#   - Look at the old test server and see what config changes are floating around
371# XXX: our SVN checkout should be updated to use
372# (repository and etc) once serving actually works.
373    cd /etc
374    svn switch --relocate svn://$source_server/ svn://
375    cd /usr/vice/etc
376    svn switch --relocate svn://$source_server/ svn://
377    cd /srv/repository
378    asbuild svn switch --relocate svn://$source_server/ svn://
379    asbuild svn up # verify works
Note: See TracBrowser for help on using the repository browser.