source: trunk/server/doc/ @ 1696

Last change on this file since 1696 was 1696, checked in by ezyang, 13 years ago
Add notes about first-time installs of hostnames.
File size: 16.0 KB
1# This document is a how-to for installing a Fedora server.
2# It is semi-vaguely in the form of a shell script, but is not really
3# runnable as it stands.
5set -e -x
7# Some commands should be run as the scripts-build user, not root.
9alias asbuild="sudo -u scripts-build"
11# Old versions of this install document advised setting
12# NSS_NONLOCAL_IGNORE=1 anytime you're setting up anything, e.g. using
13# yum, warning that useradd will query LDAP in a stupid way that makes
14# it hang forever.  As of Fedora 13, this does not seem to be a problem,
15# so it's been removed from the instructions.  If an install is hanging,
16# though, try adding NSS_NONLOCAL_IGNORE.
18# This is actually just "pick an active scripts server".  It can't be
19# because our networking config points that domain
20# at localhost, and if our server is not setup at that point things
21# will break.
24# 'branch' is the current svn branch you are on.  You want to
25# use trunk if your just installing a new server, and branches/fcXX-dev
26# if your preparing a server on a new Fedora release.
29# 'server' is the public hostname of your server, for SCP'ing files
30# to and from.
33# Start with a Scripts kickstarted install of Fedora (install-fedora)
35# Take updates, reboot if there's a kernel update.
37    yum update
39# Get rid of network manager
40    yum remove NetworkManager
42# Check out the svn repository. Configure svn not to cache
43# credentials.
45# Copy over root's dotfiles from one of the other machines.
46# Perhaps a useful change is to remove the default aliases
47    cd /root
48    ls -l .bashrc
49    ls -l .ldapvirc
50    ls -l .screenrc
51    ls -l .ssh
52    ls -l .vimrc
53    ls -l .k5login
54    # Trying to scp from server to server won't work, as scp
55    # will attempt to negotiate a server-to-server connection.
56    # Instead, scp to your trusted machine as a temporary file,
57    # and then push to the other server
58scp -r root@$source_server:~/{.bashrc,.ldapvirc,.screenrc,.ssh,.vimrc,.k5login} .
59scp -r {.bashrc,.ldapvirc,.screenrc,.ssh,.vimrc,.k5login} root@$server:~
61# Install the initial set of credentials (to get Kerberized logins once
62# krb5 is installed).  Otherwise, SCP'ing things in will be annoying.
63#   o You probably installed the machine keytab long ago
64    ls -l /etc/krb5.keytab
65#     Use ktutil to combine the host/ and
66#     host/ keys with host/ in
67#     the keytab.  Do not use 'k5srvutil change' on the combined keytab
68#     or you'll break the other servers. (real servers only).  Be
69#     careful about writing out the keytab: if you write it to an
70#     existing file the keys will just get appended.  The correct
71#     credential list should look like:
72#       ktutil:  l
73#       slot KVNO Principal
74#       ---- ---- ---------------------------------------------------------------------
75#          1    5 host/
76#          2    3 host/
77#          3    2      host/
78#   o Replace the ssh host keys with the ones common to all scripts servers (real servers only)
79    ls -l /etc/ssh/*key*
80#     You can do that with:
81scp root@$source_server:/etc/ssh/*key* .
82scp *key* root@$server:/etc/ssh/
83    service sshd reload
85# Check out the scripts /etc configuration
86    # backslash to make us not use the alias
87    cd /root
88    \cp -a etc /
89    chmod 0440 /etc/sudoers
91# If this is the first time you've installed this hostname, you will
92# need to update a bunch of files to add support for it. These include:
93#   o Adding all aliases to /etc/httpd/conf.d/scripts-vhost-names.conf
94#     (usually this is hostname,, h-n,,
95#     scriptsN,, and the IP address.)
96#   o Adding routing rules for the static IP in
97#     /etc/sysconfig/network-scripts/route-eth1
98#   o Adding the IP address to the hosts file (same hosts as for
99#     scripts-vhost-names)
100#   o Put the hostname information in LDAP so SVN and Git work
101#   o Set up Nagios monitoring on sipb-noc for the host
102#   o Set up the host as in the pool on r-b/r-b /etc/heartbeat/
105# NOTE: You will have just lost DNS resolution and the ability
106# to do password SSH in.  If you managed to botch this step without
107# having named setup, you can do a quick fix by frobbing /etc/resolv.conf
108# with a non address for the DNS server.  Be sure to revert it once
109# you have named.
111# NOTE: You can get password SSH back by editing /etc/ssh/sshd_config (allow
112# password auth) and /etc/pam.d/sshd (comment out the first three auth
113# lines).  However, you should have the Kerberos credentials in place
114# so as soon as you install the full set of Scripts packages, you'll get
115# Kerberized logins.
117# Make sure network is working.  If this is a new server name, you'll
118# need to add it to /etc/hosts and
119# /etc/sysconfig/network-scripts/route-eth1.  Kickstart should have
120# configured eth0 and eth1 correctly; use service network restart
121# to add the new routes in route-eth1.
122    service network restart
123    route
124    ifconfig
125    cat /etc/hosts
126    cat /etc/sysconfig/network-scripts/route-eth1
128# This is the point at which you should start updating scriptsified
129# packages for a new Fedora release.  Consult 'upgrade-tips' for more
130# information.
131    yum install -y scripts-base
132    # Some of these packages are naughty and clobber some of our files
133    cd /etc
134    svn revert resolv.conf hosts sysconfig/openafs
136# Replace rsyslog with syslog-ng by doing:
137    rpm -e --nodeps rsyslog
138    yum install -y syslog-ng
139    chkconfig syslog-ng on
141# Fix the openafs /usr/vice/etc <-> /etc/openafs mapping.
142    echo "/afs:/usr/vice/cache:10000000" > /usr/vice/etc/cacheinfo
143    echo "" > /usr/vice/etc/ThisCell
145# [TEST SERVER] If you're installing a test server, this needs to be
146# much smaller; the max filesize on XVM is 10GB.  Pick something like
147# 500000. Also, some of the AFS parameters are kind of retarded (and if
148# you're low on disk space, will actually exhaust our inodes).  Edit
149# these parameters in /etc/sysconfig/openafs
151# Test that zephyr is working
152    chkconfig zhm on
153    service zhm start
154    echo 'Test!' | zwrite -d -c scripts -i test
156# Install the full list of RPMs that users expect to be on the
157# servers.
158rpm -qa --queryformat "%{Name}.%{Arch}\n" | sort > packages.txt
159# arrange for packages.txt to be passed to the server, then run:
160# --skip-broken will (usually) prevent you from having to sit through
161# several minutes of dependency resolution until it decides that
162# it can't install /one/ package.
163    yum install -y --skip-broken $(cat packages.txt)
165# Check which packages are installed on your new server that are not
166# in the snapshot, and remove ones that aren't needed for some reason
167# on the new machine.  Otherwise, aside from bloat, you may end up
168# with undesirable things for security, like sendmail.
169    rpm -qa --queryformat "%{Name}.%{Arch}\n" | grep -v kernel | sort > newpackages.txt
170    diff -u packages.txt newpackages.txt | grep -v kernel | less
171    # here's a cute script that removes all extra packages
172    yum erase -y $(grep -Fxvf packages.txt newpackages.txt)
174# We need an upstream version of cgi which we've packaged ourselves, but
175# it doesn't work with the haskell-platform package which expects
176# explicit versions.  So temporarily rpm -e the package, and then
177# install it again after you install haskell-platform.  [Note: You
178# probably won't need this in Fedora 15 or something, when the Haskell
179# Platform gets updated.]
180    rpm -e ghc-cgi-devel ghc-cgi
181    yum install -y haskell-platform
182    yumdownloader ghc-cgi
183    yumdownloader ghc-cgi-devel
184    rpm -i ghc-cgi*1.8.1*.rpm
186# Check out the scripts /usr/vice/etc configuration
187    cd /root/vice
188    \cp -a etc /usr/vice
190# Install the full list of perl modules that users expect to be on the
191# servers.
192    cd /root
193    export PERL_MM_USE_DEFAULT=1
194    cpan # this is interactive, enter the next two lines
195        o conf prerequisites_policy follow
196        o conf commit
197# on a reference server
198perldoc -u perllocal | grep head2 | cut -f 3 -d '<' | cut -f 1 -d '|' | sort -u | perl -ne 'chomp; print "notest install $_\n" if system("rpm -q --whatprovides \"perl($_)\" >/dev/null 2>/dev/null")' > perl-packages.txt
199# arrange for perl-packages.txt to be transferred to server
200    cat perl-packages.txt | perl -MCPAN -e shell
202# Install the Python eggs and Ruby gems and PEAR/PECL doohickeys that are on
203# the other servers and do not have RPMs.
204# The general mode of operation will be to run the "list" command
205# on both servers, see what the differences are, check if those diffs
206# are packaged up as rpms, and install them (rpm if possible, native otherwise)
207# - Look at /usr/lib/python2.6/site-packages and
208#           /usr/lib64/python2.6/site-packages for Python eggs and modules.
209#   There will be a lot of gunk that was installed from packages;
210#   easy-install.pth in /usr/lib/ will tell you what was easy_installed.
211#   First use 'yum search' to see if the relevant package is now available
212#   as an RPM, and install that if it is.  If not, then use easy_install.
213#   Pass -Z to easy_install to install them unzipped, as some zipped eggs
214#   want to be able to write to ~/.python-eggs.  (Also makes sourcediving
215#   easier.)
216    cat /usr/lib/python2.6/site-packages/easy-install.pth
217# - Look at `gem list` for Ruby gems.
218#   Again, use 'yum search' and prefer RPMs, but failing that, 'gem install'.
219#       ezyang: rspec-rails depends on rspec, and will override the Yum
220#       package, so... don't use that RPM yet
221gem list --no-version > gem.txt
222    gem install $(gem list --no-version | grep -Fxvf - gem.txt)
223# - Look at `pear list` for Pear fruits (or whatever they're called).
224#   Yet again, 'yum search' for RPMs before resorting to 'pear install'.  Note
225#   that for things in the beta repo, you'll need 'pear install package-beta'.
226#   (you might get complaints about the php_scripts module; ignore them)
227pear list | tail -n +4 | cut -f 1 -d " " > pear.txt
228    pear config-set preferred_state beta
229    pear channel-update
230    pear install $(pear list | tail -n +4 | cut -f 1 -d " " | grep -Fxvf - pear.txt)
231# - Look at `pecl list` for PECL things.  'yum search', and if you must,
232#   'pecl install' needed items. If it doesn't work, try 'pear install
233#   pecl/foo' or 'pecl install foo-beta' or those two combined.
234pecl list | tail -n +4 | cut -f 1 -d " " > pecl.txt
235    pecl install --nodeps $(pecl list | tail -n +4 | cut -f 1 -d " " | grep -Fxvf - pecl.txt)
237# Setup some Python config
238    echo 'import site, os.path; site.addsitedir(os.path.expanduser("~/lib/python2.6/site-packages"))' > /usr/lib/python2.6/site-packages/00scripts-home.pth
240# Install the credentials.  There are a lot of things to remember here.
241# Be sure to make sure the permissions match up (ls -l on an existing
242# server!).
243scp root@$source_server:{/etc/{sql-mit-edu.cfg.php,daemon.keytab,pki/tls/private/scripts.key,signup-ldap-pw,whoisd-password},/home/logview/.k5login} .
244scp daemon.keytab signup-ldap-pw whoisd-password sql-mit-edu.cfg.php root@$server:/etc
245scp scripts.key root@$server:/etc/pki/tls/private
246scp .k5login root@$server:/home/logview
247    chown afsagent:afsagent /etc/daemon.keytab
248#   o The daemon.scripts keytab (will be daemon.scripts-test for test)
249    ls -l /etc/daemon.keytab
250#   o The SSL cert private key (real servers only)
251    ls -l /etc/pki/tls/private/scripts.key
252#   o The LDAP password for the signup process (real servers only)
253    ls -l /etc/signup-ldap-pw
254#   o The whoisd password (real servers only)
255    ls -l /etc/whoisd-password
256#   o Make sure logview's .k5login is correct (real servers only)
257    cat /home/logview/.k5login
259# Spin up OpenAFS.  This will fail if there's been a new kernel since
260# when you last tried.  In that case, you can hold on till later to
261# start OpenAFS.  This will take a little bit of time;
262    service openafs-client start
264# Check that fs sysname is correct.  You should see, among others,
265# 'amd64_fedoraX_scripts' (vary X) and 'scripts'. If it's not, you
266# probably did a distro upgrade and should update /etc/sysconfig/openafs.
267    fs sysname
269# [TEST SERVER] If you are setting up a test server, pay attention to
270# /etc/sysconfig/network-scripts and do not bind scripts' IP address.
271# You will also need to modify:
272#   o /etc/ldap.conf
273#       add: host
274#   o /etc/nss-ldapd.conf
275#       replace: uri *****
276#       with: uri ldap://
277#   o /etc/openldap/ldap.conf
278#       add: URI ldap://
279#            BASE dc=scripts,dc=mit,dc=edu
280#   o /etc/httpd/conf.d/vhost_ldap.conf
281#       replace: VhostLDAPUrl ****
282#       with: VhostLDAPUrl "ldap://,dc=scripts,dc=mit,dc=edu"
283#   o /etc/postfix/virtual-alias-{domains,maps}
284#       replace: server_host *****
285#       with: server_host = ldap://
286# to use instead of localhost.
287# XXX: someone should write sed scripts to do this
289# [TEST SERVER] If you are setting up a test server, afsagent's cronjob
290# will attempt to be renewing with the wrong credentials
291# (daemon.scripts). Change this:
292    vim /home/afsagent/renew # replace all mentions of
294# Set up replication (see ./install-ldap).
295# You'll need the LDAP keytab for this server: be sure to chown it
296# fedora-ds after you create the fedora-ds user
297    ls -l /etc/dirsrv/keytab
298    cat install-ldap
300# Make the services dirsrv, nslcd, nscd, postfix, and httpd start at
301# boot. Run chkconfig to make sure the set of services to be run is
302# correct.
303    service nslcd start
304    service nscd start
305    service postfix start
306    service httpd start
307    chkconfig dirsrv on
308    chkconfig nslcd on
309    chkconfig nscd on
310    chkconfig postfix on
311    chkconfig httpd on
313# nrpe is required for nagios alerts
314    chkconfig nrpe on
316# Check sql user credentials (needs to be done after LDAP is setup)
317    chown sql /etc/sql-mit-edu.cfg.php
319# Postfix doesn't actually deliver mail; fix this
320    cd /etc/postfix
321    postmap virtual
323# Munin might not be monitoring packages that were installed after it
324    munin-node-configure --suggest --shell | sh
326# Run fmtutil-sys --all, which does something that makes TeX work.
327# (Note: this errors on XeTeX which is ok.)
328    fmtutil-sys --all
330# Ensure that PHP isn't broken:
331    mkdir /tmp/sessions
332    chmod 01777 /tmp/sessions
334# Ensure fcgid isn't broken (should be 755)
335    ls -ld /var/run/mod_fcgid
337# Fix etc by making sure none of our config files got overwritten
338    cd /etc
339    svn status -q
340    # Some usual candidates for clobbering include nsswitch.conf and
341    # sysconfig/openafs
343# ThisCell got clobbered, replace it with
344    echo "" > /usr/vice/etc/ThisCell
346# Reboot the machine to restore a consistent state, in case you
347# changed anything. (Note: Starting kdump fails (this is ok))
349# [OPTIONAL] Your machine's hostname is baked in at install time;
350# in the rare case you need to change it: it appears to be in:
351#   o /etc/sysconfig/network
352#   o your lvm thingies; probably don't need to edit
354# [TEST SERVER] More stuff for test servers
355#   - You need a self-signed SSL cert.  Generate with:
356    openssl req -new -x509 -keyout /etc/pki/tls/private/scripts.key -out /etc/pki/tls/certs/scripts.cert -nodes
357#     Also make /etc/pki/tls/certs/ca.pem match up
358#   - Make (/etc/aliases) root mail go to /dev/null, so we don't spam people
359#   - Edit /etc/httpd/conf.d/scripts-vhost-names.conf to have
360#     be an accepted vhost name
361#   - Look at the old test server and see what config changes are floating around
363# XXX: our SVN checkout should be updated to use
364# (repository and etc) once serving actually works.
365    cd /etc
366    svn switch --relocate svn://$source_server/ svn://
367    cd /usr/vice/etc
368    svn switch --relocate svn://$source_server/ svn://
369    cd /srv/repository
370    asbuild svn switch --relocate svn://$source_server/ svn://
371    asbuild svn up # verify works
Note: See TracBrowser for help on using the repository browser.