xenBackup - the source

here's the source for the backup script. please feel free to use, modify, plagiarize, mock, torture or hack up any of this bash code as your mood takes you.

#!/bin/bash
#
#   Copyright John Quinn, 2008
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see .

#
# xenBackup - Backup Xen Domains
#
#             Version:    1.0:     Created:  John D Quinn, http://www.johnandcailin.com/john
#

# initialize our variables
domains="null"                           # the list of domains to backup
allDomains="null"                        # backup all domains?
targetLocation="/tmp"                    # the default backup target directory
mountPoint="/mnt/xen"                    # the mount point to use to mount disk areas
xenDiskArea="/dev/skx-vg"                # the LVM volume group to use
shutdownDomains=false                    # don't shutdown domains by default
quiet=false                              # keep the chatter down
backupEngine=tar                         # the default backup engine
rsyncExe=/usr/bin/rsync                  # rsync executable
rdiffbackupExe=/usr/bin/rdiff-backup     # rdiff-backup executable
tarExe=/usr/bin/tar                      # tar executable
xmExe=/usr/sbin/xm                       # xm executable
purgeAge="null"                          # age at which to purge increments
globalBackupResult=0                     # success status of overall job

# settings for logging (syslog)
loggerArgs=""                            # what extra arguments to the logger to use
loggerTag="xenBackup"                    # the tag for our log statements
loggerFacility="local3"                  # the syslog facility to log to

# trap user exit and cleanup
trap 'cleanup;exit 1' 1 2

cleanup()
{
   ${logDebug} "Cleaning up"
   cd / ; umount ${mountPoint}

   # restart the domain
   if test ${shutdownDomains} = "true"
   then
      ${logDebug} "Restarting domain"
      ${xmExe} create ${domain}.cfg > /dev/null
   fi
}

# function to print a usage message and bail
usageAndBail()
{
   cat << EOT
Usage: xenBackup [OPTION]...
Backup xen domains to a target area. different backup engines may be specified to
produce a tarfile, an exact mirror of the disk area or a mirror with incremental backup.

   -d      backup only the specified DOMAINs (comma seperated list)
   -t      target LOCATION for the backup e.g. /tmp or root@www.example.com:/tmp
           (not used for tar engine)
   -a      backup all domains
   -s      shutdown domains before backup (and restart them afterwards)
   -q      run in quiet mode, output still goes to syslog
   -e      backup ENGINE to use, either tar, rsync or rdiff-backup
   -p      purge increments older than TIME_SPEC. this option only applies
           to rdiff-backup, e.g. 3W for 3 weeks. see "man rdiff-backup" for
           more information

Example 1
   Backup all domains to the /tmp directgory
   $ xenBackup -a -t /tmp

Example 2
   Backup domain: "wiki" using rsync to directory /var/xenImages on machine backupServer,
   $ xenBackup -e rsync -d wiki -t root@backupServer:/var/xenImages

Example 3
   Backup domains "domainOne" and "domainTwo" using rdiff-backup purging old increments older than 5 days
   $ xenBackup -e rdiff-backup -d "domainOne, domainTwo" -p 5D

EOT

   exit 1;
}

# parse the command line arguments
while getopts p:e:qsad:t:h o
do     case "$o" in
        q)     quiet="true";;
        s)     shutdownDomains="true";;
        a)     allDomains="true";;
        d)     domains="$OPTARG";;
        t)     targetLocation="$OPTARG";;
        e)     backupEngine="$OPTARG";;
        p)     purgeAge="$OPTARG";;
        h)     usageAndBail;;
        [?])   usageAndBail
       esac
done

# if quiet don't output logging to standard error
if test ${quiet} = "false"
then
   loggerArgs="-s"
fi

# setup logging subsystem. using syslog via logger
logCritical="logger -t ${loggerTag} ${loggerArgs} -p ${loggerFacility}.crit"
logWarning="logger -t ${loggerTag} ${loggerArgs} -p ${loggerFacility}.warning"
logDebug="logger -t ${loggerTag} ${loggerArgs} -p ${loggerFacility}.debug"

# make sure only root can run our script
test $(id -u) = 0 || { ${logCritical} "This script must be run as root"; exit 1; }

# make sure that the guest manager is available
test -x ${xmExe} || { ${logCritical} "xen guest manager (${xmExe}) not found"; exit 1; }

# assemble the list of domains to backup
if test ${allDomains} = "true"
then
   domainList=`${xmExe} list | cut -f1 -d" " | egrep -v "Name|Domain-0"`
else
   # make sure we've got some domains specified
   if test "${domains}" = "null"
   then
      usageAndBail
   fi

   # create the domain list by mapping commas to spaces
   domainList=`echo ${domains} | tr -d " " | tr , " "`
fi

# function to do a "rdiff-backup" of domain
backupDomainUsingrdiff-backup ()
{
   domain=$1
   test -x ${rdiffbackupExe} || { ${logCritical} "rdiff-backup executable (${rdiffbackupExe}) not found"; exit 1; }

   if test ${quiet} = "false"
   then
      verbosity="3"
   else
      verbosity="0"
   fi

   targetSubDir=${targetLocation}/${domain}.rdiff-backup.mirror

   # make the targetSubDir if it doesn't already exist
   mkdir ${targetSubDir} > /dev/null 2>&1
   ${logDebug} "backing up domain ${domain} to ${targetSubDir} using rdiff-backup"

   # rdiff-backup to the target directory
   ${rdiffbackupExe} --verbosity ${verbosity} ${mountPoint}/ ${targetSubDir}
   backupResult=$?

   # purge old increments
   if test ${purgeAge} != "null"
   then
      # purge old increments
      ${logDebug} "purging increments older than ${purgeAge} from ${targetSubDir}"
      ${rdiffbackupExe} --verbosity ${verbosity} --force --remove-older-than ${purgeAge} ${targetSubDir}
   fi

   return ${backupResult}
}

# function to do a "rsync" backup of domain
backupDomainUsingrsync ()
{
   domain=$1
   test -x ${rsyncExe} || { ${logCritical} "rsync executable (${rsyncExe}) not found"; exit 1; }

   targetSubDir=${targetLocation}/${domain}.rsync.mirror

   # make the targetSubDir if it doesn't already exist
   mkdir ${targetSubDir} > /dev/null 2>&1
   ${logDebug} "backing up domain ${domain} to ${targetSubDir} using rsync"

   # rsync to the target directory
   ${rsyncExe} -essh -avz --delete ${mountPoint}/ ${targetSubDir}
   backupResult=$?

   return ${backupResult}
}

# function to a "tar" backup of domain
backupDomainUsingtar ()
{
   domain=$1

   # make sure we can write to the target directory
   test -w ${targetLocation} || { ${logCritical} "target directory (${targetLocation}) is not writeable"; exit 1; }

   targetFile=${targetLocation}/${domain}.`date '+%d%b%y'`.$$.tar.gz
   ${logDebug} "backing up domain ${domain} to ${targetFile} using tar"

   # tar to the target directory
   cd ${mountPoint}

   ${tarExe} pcfz ${targetFile} * > /dev/null
   backupResult=$?

   return ${backupResult}
}

# backup the specified domains
for domain in ${domainList}
do
   ${logDebug} "backing up domain: ${domain}"

   # make sure that the domain is shutdown if required
   if test ${shutdownDomains} = "true"
   then
      ${logDebug} "shutting down domain ${domain}"
      ${xmExe} shutdown -w ${domain} > /dev/null
   fi

   # unmount mount point if already mounted
   umount ${mountPoint} > /dev/null 2>&1

   # mount the xen disk read-only
   xenDisk=${xenDiskArea}/${domain}-disk
   test -r ${xenDisk} || { ${logCritical} "xen disk area not readable. are you sure that the domain \"${domain}\" exists?"; exit 1; }
   ${logDebug} "Mounting ${xenDisk} read-only"
   mount -r ${xenDisk} ${mountPoint} || { ${logCritical} "mount failed, does mount point (${mountPoint}) exist?"; exit 1; }

   # do the backup according to the chosen backup engine
   backupDomainUsing${backupEngine} ${domain}

   # make sure that the backup was successful
   if test $? -ne 0
   then
      ${logCritical} "FAILURE: error backing up domain ${domain}"
      globalBackupResult=1
   else
      ${logDebug} "SUCCESS: domain ${domain} backed up"
   fi
     
   # clean up
   cleanup;
done

if test ${globalBackupResult} -eq 0
then
   ${logDebug} "SUCCESS: backup of all domains completed successfully"
else
   ${logCritical} "FAILURE: backup completed with some failures"
fi

exit ${globalBackupResult}

 

backing up your xen domains

backups are boring, but we all know how important they are. backups can also be quite powerful when working with xen virtualization, since xen allows for convenient back-up and restore of entire systems.
i've recently been working on a flexible, general-purpose script enabling incremental backups of complete xen guests, optimized for secure, distributed environments; xenBackup. if you're working with xen, you might find it useful.
the xenBackup script leverages open-source components like ssh, rsync, and rdiff-backup to create a simple, efficient and functional solution.
all code and configurations have been tested on debian etch but should be useful for other *nix flavors with subtle modifications. if you're unfamiliar with xen, you might consider starting with an earlier how-to on setting up xen on your debian etch box

a general approach

the approach you take to backups obviously depends on what your guests are doing. let's consider one of the more difficult cases, backing up a xen guest with an application server and a database running on it. ideally, you'd typically:

• take regular backups of the database.
• take a regular incremental backup of the entire machine.
• write the backups onto a different server on your network, and hopefully to a different geographical location too.
• do all of this without any interruption of your service.

in this article we'll discuss a simple way of doing this.

the xenBackup script

the xenBackup script, which i've included at the end of this article, helps implement a xen backup strategy. it automates the backup of single or multiple xen guests using one of three backup methods, tar, rsync or rdiff-backup. the usage message for xenBackup is as follows:

Usage: xenBackup [OPTION]...
Backup xen domains to a target area. different backup engines may be specified to
produce a tarfile, an exact mirror of the disk area or a mirror with incremental backup.

   -d      backup only the specified DOMAIN
   -t      target LOCATION for the backup e.g. /tmp or root@www.example.com:/tmp
           (not used for tar engine)
   -a      backup all domains
   -s      shutdown domains before backup (and restart them afterwards)
   -q      run in quiet mode
   -e      backup ENGINE to use, either tar, rsync or rdiff-backup
   -p      purge increments older than TIME_SPEC. this option only applies
           to rdiff-backup, e.g. 3W for 3 weeks. see "man rdiff-backup" for
           more information

to illustrate how it could be used, let's consider a typical scenario.

scenario: a single xen server with multiple xen guests

consider a xen server with multiple guests running on it, where the database on each guest backs up locally using a database specific backup technique e.g. a regularly scheduled hot backup writing to the local file system.
xenBackup could be used to periodically backup each of the local guests from the dom0. this is safe to do on a running server since the database backup does not rely on datafile consistency, but instead on the hot backups.
alternatively, the hot backup could be avoided if each of the guests was cleanly shutdown before the backup. xenBackup supports both these modes of operation but the former is recommended.
the xenBackup command on dom0 to incrementally backup all guests to /var/backup would simply be:

$ sudo xenBackup -a -e rdiff-backup -t /var/backup

this arrangement is shown in the diagram above. additionally, this backup could be periodically pulled from another backup server using rsync over ssh. this backup server could be located on or off-site.
this could be simplified by writing the backup directly onto another server in a single xenBackup command. this is arguably less secure since you need to push the backup rather than pull it, but could be done with:

$ sudo xenBackup -a -e rdiff-backup -t root@backupserver:/var/backup

running xenBackup on multiple dom0's the following arrangement can easily be achieved:

the backups

one of the great things about xen is that the backup allows you to reconstitute a fully working xen guest from the backup area, simply with a command like:

$ sudo xen-create-image --copy=/var/backup/mymachine.rdiff-backup.mirror --ip=192.168.1.10 --hostname=mymachine

if rdiff-backup is used as the backup engine, the xen guest can easily be restored to a historical state, to as far back as increments are kept (controlled by the xenBackup purge flag, -p). see man rdiff-backup for more information on using --restore-as-of on your backup directory.

dependencies

the xenBackup script has dependencies on rsync and rdiff-backup, if you choose to use those engines. if you do, you should install those packages:

$ sudo apt-get install rsync rdiff-backup

automation

to automate your backups, consider adding a cron entry to automatically run xenBackup on your backup server. typically you should do this at a quiet time. an example cron entry is:

00 1 * * * /usr/bin/xenBackup -q -a -t /var/backup -e rdiff-backup

you should consider adding the backup server's identity file to the authorized keys of the backup user on machines to push backups to. to allow the automation of encrypted, automated backups to be pushed over ssh. read up on ssh-copy-id and ssh-keygen for more information. give very careful consideration to security when determining the user to use and what machines can access what.

a word on syslog

xenBackup logs all backup output to syslog's local3 facility. all xenBackup's log output is available in the main syslog log, /var/log/syslog. in addition, a dedicated xenBackup log can be created by adding the following to your /etc/syslog.conf file:

# xenBackup logging: log all local3's messages to /var/log/xenBackup
local3.*                        /var/log/xenBackup

if you'd like to be informed about critical backup problems by email, please refer to my earlier blog how to setup real-time email-notification for critical syslog events.

setting up xen on your debian etch box

xen is a free software virtual machine monitor for IA-32, x86-64, IA-64 and PowerPC architectures. it runs on a host operating system and allows several guest operating systems to be run on top of the host on the same computer hardware at the same time.
there are many ways to setup xen, but i've put together a simple step-by-step guide to get a working xen system based on debian etch. easy as pie.

install your host system

install a copy of debian etch. you should leave a partition available for lvm, that your virtual machines will use for disk.

create a logical volume group

1. Get the linux logical volume manager; apt-get install lvm2
2. Initialize your partition (or disk) for lvm; pvcreate /dev/myLvmPartition
3. Create a logical volume group on your partition; vgcreate skx-vg /dev/myLvmPartition

install xen

you can install Xen from the debian packages. Find a list with apt-cache search xen-linux-system. you'll do something like:

# apt-get install xen-tools xen-linux-system-2.6.18-4-xen-686 xen-docs-3.0 libc6-xen

you should end up with something like the following, depending on what you chose:

# dpkg --list | grep xen
ii  libc6-xen                         2.3.6.ds1-13etch2
ii  linux-image-2.6.18-4-xen-686      2.6.18.dfsg.1-12etch2
ii  linux-modules-2.6.18-4-xen-686    2.6.18.dfsg.1-12etch2
ii  xen-docs-3.0                      3.0.3-0-2
ii  xen-hypervisor-3.0.3-1-i386-pae   3.0.3-0-2
ii  xen-linux-system-2.6.18-4-xen-686 2.6.18.dfsg.1-12etch2
ii  xen-tools                         2.8-2
ii  xen-utils-3.0.3-1                 3.0.3-0-2
ii  xen-utils-common                  3.0.3-0-2

reboot

reboot your system and make sure that you're now running the xen kernel

# uname -a
Linux yourhostmachine 2.6.18-4-xen-686 #1 SMP Thu May 10 03:24:35 UTC 2007 i686 GNU/Linux

configure a network bridge

get the bridge utils package

# apt-get install bridge-utils

add a bridging interface to /etc/network/interfaces

auto xenbr0
iface xenbr0 inet static
   pre-up brctl addbr xenbr0
   post-down brctl delbr xenbr0
   post-up iptables -t nat -F
   post-up iptables -t nat -A POSTROUTING -o eth0 -s 192.168.1.0/24 -j MASQUERADE
   address 192.168.1.1
   netmask 255.255.255.0
   bridge_fd 0
   bridge_hello 0
   bridge_stp off

bring up this new interface:

# ifup xenbr0

edit /etc/sysctl.conf and uncomment the following line:

net.ipv4.conf.default.forwarding=1

enable this by:

# sysctl -p
#  echo 1 > /proc/sys/net/ipv4/conf/all/forwarding

configure your default guest system using xen-tools

you can use xen-tools to configure a default guest system. It's here where you specify what OS you want to use, how networking is configured, how disk is configured etc. This can be overridden when you create a specific guest system, but it's a good idea to configure your starting point.

try creating a guest system

you can create a guest system as follows:

# xen-create-image --ip=192.168.1.6 --hostname=mymachine

this takes a minute or two. you can follow along with the progress by tailing the log file: # tail -f /var/log/xen-tools/mymachine.log you can later delete this image using:

# xen-delete-image mymachine

you can list all your images using:

# xen-list-images

boot up that sucker

you can quickly test-boot your new system as follows.

# xm create -c mymachine.cfg

this attaches a console to it and is useful for making sure that it works o.k. when you've got everything working you'll probably want to use a start / stop technique described later.

port forward (optional)

if you want external machines to access ports on your virtual machines you can setup port forwards using IP tables e.g. if you wanted to install apache on one of your virtual machines and have it answer on http://yourhostmachine:80, you'd do the following (which forwards HTTP traffic on your eth0 interface to a virtual machine at address 192.168.1.8). add the following two lines to your network/interfaces file:

   post-up iptables -A PREROUTING -t nat -i eth0 -p tcp --dport 80 -j DNAT --to 192.168.1.8:80
   post-up iptables -A INPUT -p tcp -m state --state NEW --dport 80 -i eth0 -j ACCEPT

i.e. your complete bridge definition might look like:

auto xenbr0
iface xenbr0 inet static
   pre-up brctl addbr xenbr0
   post-down brctl delbr xenbr0
   post-up iptables -t nat -F
   post-up iptables -t nat -A POSTROUTING -o eth0 -s 192.168.1.0/24 -j MASQUERADE
   post-up iptables -A PREROUTING -t nat -i eth0 -p tcp --dport 80 -j DNAT --to 192.168.1.8:80
   post-up iptables -A INPUT -p tcp -m state --state NEW --dport 80 -i eth0 -j ACCEPT
   address 192.168.1.1
   netmask 255.255.255.0
   bridge_fd 0
   bridge_hello 0
   bridge_stp off

cloning a machine

one of the great things about Xen, is that it makes it really simple to build a machine exactly the way that you want it, then clone it and distribute it to everyone that needs it. allowing you to:

• Easily create development sandboxes
• Create and distribute a standardized development environment
• Create a machine and then build a cluster
• Upgrade machines by duplicating them, patching the duplicates and if everything goes well, switching over to the new machines or rolling back.

anyway, here's an easy way that you can do it.

create an tarfile of an existing virtual machine

1. create a place to store your image # mkdir /var/xen-images
2. shutdown the machine that you're planning to clone (duh)
3. create a mount point to mount of of your existing images # mkdir /mnt/xen
4. mount the image you want to copy #  mount /dev/skx-vg/mymachine-disk /mnt/xen
5. go to the mount point and tar everything up # cd /mnt/xen ; tar pcfzv /var/xen-images/myImage.tar.gz *
6. take a peek at your nice new tar file # tar tvfz /var/xen-images/myImage.tar.gz
7. get out of the mount point and unmount. # cd / ; umount /mnt/xen

i've created a bash script to automate this, posted at the end of this article

creating a virtual machine from a tarfile (like the one created above)

1. temporarily comment out any installation method in /etc/xen-tools/xen-tools.conf e.g. this line debootstrap = 1
2. create your image with whatever flags you want e.g. # xen-create-image --tar=/var/xen-images/myImage.tar.gz --ip=192.168.1.10 --hostname=flossyTheClonedMachine
3. off you go to happy land.

starting and stopping on boot

If you want to automatically start / stop your machines on bootup, link the machine configuration in /etc/xen/auto e.g.

# mkdir /etc/xen/auto
# ln -s /etc/xen/mymachine.cfg /etc/xen/auto/

manually starting and stopping

You can easily start and stop all your xen domains with the handy /etc/init.d/xendomains script e.g. by:

# /etc/init.d/xendomains stop

You can use the usual stop, start, restart commands

utilities

take a look at XenMan (apt-get install xenman ), is a nifty little x-windows tool for managing the virtual machines running on your host.

cleaning up the debian install

if you install a debian guest, you should consider some post install steps including:

• setup locales:
  # apt-get install locales
# dpkg-reconfigure locales
  picking e.g.en_US.UTF-8 UTF-8
• set the timezone:
  # tzconfig
  (note: say yes and follow the prompts even if it looks right)
• by default your domU clock is the dom0 clock. this is probably the way you should leave it i.e. install ntp on dom0 and have your domU's use the dom0 synchronized clock. if you want your domU to operate independenly, you'll want to try: echo 1 > /proc/sys/xen/independent_wallclock

notes

If you are seeing errors like "4Gb seg fixup" spewed to the console, you need to apt-get install libc6-xen

backing up your xen guests

if you need to backup your xen guests, please take a look at my article backing up your xen domains for a discussion on the subject. a flexible script that you can use, xenBackup, is also provided.

setting up a bridging interface

in the configuration above the xen guests are only visible to the xen-host, and any services on the xen-hosts must be accesses via port forwarding, tunneling etc. for some applications, a bridging configuration works better. you can set this up by following the instructions in setting up a xen bridging interface

Installing and configuring Exim 4 on Debian

1. First, install all the necessary Debian packages are on the system as the root user. (The exim4 package will REPLACE the exim package.)
   NOTE: If you are using the stable branch, it is suggested to use the debian volatile packages (along with the security packages) so that your system is using the most up-to-date critical packages (like ClamAV) for security purposes. For production servers, you may not want to run a mixed stable/testing/unstable system (though I know some of you do!). To use these packages, see http://volatile.debian.net/ for more information. For those of you who are impatient and don't want to find the correct mirror, here's is what I added to my /etc/apt/sources.list file:
   

   deb http://volatile.debian.net/debian-volatile sarge/volatile main contrib

   I used aptitude to install these packages, but you could also use the old apt-get method:
   

   apt-get install clamav-daemon \
   clamav-freshclam exim4-daemon-heavy exim4 \
   courier-base courier-authdaemon courier-imap \
   courier-pop spamassassin wget spamc sa-exim

   When going through the exim4 config, be sure to select the multiple file configuration layout. If you didn't (or weren't prompted for it), simply set dc_use_split_config to true in the /etc/exim4/update-exim.conf.conf file. (Thanks Mike!)
   
2. Create your Maildir directory
   

   maildirmake ~/Maildir/

3. Now we want to make exim4 use Maildir format mailboxes. Modify the file /etc/exim4/update-exim4.conf.conf so that it contains:
   

   dc_localdelivery='maildir_home'

4. We need to Edit /etc/default/spamassassin to enable spamd.
   
5. Each user can set up their own filters by creating a .forward file in their home directory. If the first line of this file reads # Exim filter then Exim4 will treat it as a filter.
   Here is an example of an Exim filter that checks the headers that SpamAssassin adds and puts the mail in the appropriate Maildir folder:
   

   # Exim filter
   if $h_X-Spam-Status: CONTAINS "Yes"
        or
     $h_X-Spam-Flag: CONTAINS "Yes"
   then
     save $home/Maildir/.Spam/
     finish
   endif

   Exim's Interface To Mail Filtering (PDF format) - Local copy
   
6. Many system administrators like to set up the Maildir directories and .forward filter file in the /etc/skel directory so that when they make a new user on the system, everything is automatically copied over. I suggest that you do this as well as it makes things easier.
   
7. Before going live with the mail server, we will want to test it!
   
   1. Generate the new configuration:
      

      update-exim4.conf

      If you made it through this, then your config files don't have any syntax errors.
      

      exim4 -bV

      If that works, then there are no config issues
      
   2. Next, start exim by issuing:
      

      /etc/init.d/exim4 start

      Above assumes that you are running exim4 as a daemon, and not through inetd
      
   3. Now, check a local address:
      

      exim4 -bt local_user@example.com

   4. Check sending an email:
      

      exim4 -v mailbox_you_can_check@dom.ain
         From: user@your.domain
         To: mailbox_you_can_check@dom.ain
         Subject: Testing exim
               
         Testing exim
         .

      You should now see some messages to let you know that the email was sent or information about what went wrong.
      
   5. To test with full debug output using a specific config file, use something like:
      

      exim4 -C /etc/exim/exim_example.conf -d -bt user@example.com

   6. To test the config coming from a specified ip address, use:

      exim4 -bh 192.168.1.10

      HELO example.com
         MAIL FROM: 
         RCPT TO: <local_user@example.com>
         DATA
         Subject: something
         your message here
         .
         QUIT

8. Add the following to your /etc/exim4/conf.d/main/01_exim4-config_listmacrosdefs file:
   

   # This tells what virus scanner to use
   av_scanner = clamd:/var/run/clamav/clamd.ctl

9. Edit /etc/exim4/conf.d/acl/40_exim4-config_check_data to inlude the following before the "# accept otherwise" line:
   

   # Reject messages that have serious MIME errors.
      # This calls the demime condition again, but it
      # will return cached results.
      deny message = Serious MIME defect detected ($demime_reason)
      demime = *
      condition = ${if >{$demime_errorlevel}{2}{1}{0}}
      
      # Reject file extensions used by worms.
      # Note that the extension list may be incomplete.
      deny message = This domain has a policy of not accepting certain types of attachments \
                     in mail as they may contain a virus.  This mail has a file with a .$found_extension \
                     attachment and is not accepted.  If you have a legitimate need to send \
                     this particular attachment, send it in a compressed archive, and it will \
                     then be forwarded to the recipient.
      demime = exe:com:vbs:bat:pif:scr
      
      # Reject messages containing malware.
      deny message = This message contains a virus ($malware_name) and has been rejected
      malware = *

10. Then, you need to enable ClamAV.
    
    1. Firstly, you will want to be sure that it is running against messages. In /etc/exim4/sa-exim.conf, search for SAEximRunCond:
       

       SAEximRunCond: ${if and {{def:sender_host_address} {!eq {$sender_host_address}{127.0.0.1}} {!eq {$h_X-SA-Do-Not-Run:}{Yes}} } {1}{0}}

       That is simply skipping the scan on anything from the local machine or if the X-SA-Do-Not-Run header in the message is set to Yes. If you just want exim to run ClamAV on all messages, use this:
       

       SAEximRunCond: 1

    2. Before restarting ClamAV, we need to be sure that all of the access rights are in place so that the scans actually happen. The best way to handle this is to add the clamav user to the Debian-exim group. Either manually edit /etc/group, or simple run:
       

       adduser clamav Debian-exim

    3. Be sure that /etc/clamav/clamd.conf contains a line that reads:
       

       AllowSupplementaryGroups

    4. Set the file permissions for the /var/run/clamav directory to allow for the correct user to use it:
       

       chown Debian-exim.Debian-exim /var/run/clamav
       chmod g+w /var/run/clamav

    5. A restart of ClamAV is necessary for the changes to take effect:
       

       /etc/init.d/clamav-daemon restart

11. You should now be able to get your mail via IMAP with a mail client like Mozilla. Check your headers (View Source) and see that SpamAssassin has added its headers. SMTP-end virus scanning should also be taking place. Check your /var/log/clamav/clamav.log to monitor this.
    

Multiple Domain Alias Files

The steps below are used to enable support for having multiple virtual domains each with its own alias file.

1. Exim will need to have the alias files for each domain.
   
   1. Create the /etc/exim4/virtual directory.
      
   2. For each virtual domain, create a file that contains the aliases to be used named as the domain.
      For example, if I example.com was one of my domains, I'd do the following:
      
      1. Create the /etc/exim4/virtual/example.com file.
         
      2. If my system users were sys1, sys2, and sys3, and their email addresses were to be joe, john, jason, I'd put the following into the domain alias file:
         

         joe:    sys1@localhost
         john:   sys2@localhost
         jason:  sys3@localhost

         If john was also to get all mail addressed to info@example.com, you would add this entry:
         

         info:   sys2@localhost

         If you wanted all mail to user1@example.com to go to another email account outside of this domain, you would enter:
         

         user1:  a.user@some.domain

         If you wanted all mail directed at any address other than what is defined in the alias file to go to joe, you'd enter:
         

         *:      sys1@localhost

         In the above examples, the "@localhost" suffix to the user names forces the delivery to a system user. I found that if you do not include this in the alias files and your machine's host name is within one of the domains handled by exim, every system user would need an entry in the machine's domain in order to be delivered corectly. For instance, if your host name was mail.example1.com and example1.com was handled by this server this would be needed. This would allow delivery to all the system user names at example1.com. The reason is simple, and I will try to illustrate it for you here:
         
         1. exim receives a message delivered to joe.blow@example3.com
         2. The alias file for this domain has joe.blow: jblow in it.
         3. This would translate to jblow@domain-of-the-system
         4. The process would be repeated using jblow@domain-of-the-system
         5. If there was no entry in the domain-of-the-system alias file for jblow, the message would be undeliverable (or non-routable)
         You could even have special redirects like the following:
         

         script: "| /path/to/some/script"
         prev:   :fail: $local_part left!
         kill:   :blackhole:

2. Edit /etc/exim4/conf.d/main/01_exim4-config_listmacrosdefs by replacing the current local_domains line with:
   

   domainlist local_domains = @:localhost:dsearch;/etc/exim4/virtual

3. Create /etc/exim4/conf.d/router/350_exim4-config_vdom_aliases with the following content:
   

   vdom_aliases:
         driver = redirect
         allow_defer
         allow_fail
         domains = dsearch;/etc/exim4/virtual
         data = ${expand:${lookup{$local_part}lsearch*@{/etc/exim4/virtual/$domain}}}
         retry_use_local_part
         pipe_transport   = address_pipe
         file_transport   = address_file

4. Now, regenerate your exim4 config:
   

   update-exim4.conf

5. If there were no errors, restart exim4:
   

   /etc/init.d/exim4 restart

Domain Dependent Maximum Message Size

The next step for my server is to give each domain a configurable message size limit. Then, when the server get's a message that is larger than the target domain's size limit, I want to send a message back to the original sender telling them why the message was not delivered. However, I also want to have that message customized for each domain. That way, the domain owners can provide detailed instructions on how to send large messages to their domain if it is necessary. Of course, there will also need to be some kind of default size limit and message for domains that do not need the customization.

1. Create /etc/exim4/domain-size-limits to contain the list of domains and their maximum message size limits. You can also add a wildcard at the end entry if you want to set a default limit. The file may look something like the following:
   

   example.com: 20M
   example1.com: 5M
   *: 15M

   This provides you a quick way to edit the values. The values will also take effect as soon as the file is saved - no need to restart exim!
   
2. OK, now we know what domains we want to customize the size for. Now it's time to create a message to send for those domains. Create /etc/exim4/domain-size-limit-messages with content similar to:
   

   exmaple.com: The largest acceptable message size for Example.com is\
                ${expand:${lookup{$domain}lsearch*@{/etc/exim4/domain-size-limits}}}.\
                Your message was $message_size. If you feel that $local_part@$domain\
                should really get your message, then visit http://www.example.com/files/\
                where you can upload any large files. If you select $local_part@$domain\
                from the "notify" list, they will receive a message with a link directly\
                to your file.
   *:           The largest acceptable message size for $domain is\
                ${expand:${lookup{$domain}lsearch*@{/etc/exim4/domain-size-limits}}}.\
                Your message size was $message_size. Please revise your message so it\
                does not exceed this maximum file size and resend. If this is not\
                possible, contact the recipient in another way.

   As you see, we have one domain that has a custom message sent out, and have defined a default message for all other domains. These messages can be edited at any time and do not need an exim restart to take effect.
   
3. Now for the fun part! We need a way to catch the messages that are too large for the domain! First, create /etc/exim4/conf.d/router/325_exim4-config_large_messages with the following:
   

   large_messages:
       driver = accept
       domains = dsearch;/etc/exim4/virtual
       condition = ${if >{$message_size}{${expand:${lookup{$domain}lsearch*@{/etc/exim4/domain-size-limits}}}} {yes}{no}}
       transport = bounce_large_messages
       no_verify

   This router dynamically checks which domains are available and what their limits are set to.
   
4. Now create /etc/exim4/conf.d/transport/40_exim4-config_bounce_large_messages with the following content:
   

   # This bounces a message to people who send files too large for that domain
   bounce_large_messages:
     driver = autoreply
     from = $local_part@$domain
     to = $sender_address
     subject = Re: ${escape:$h_subject:}
     text = ${expand:${lookup{$domain}lsearch*@{/etc/exim4/domain-size-limit-messages}}}

   This transport then sends the original sender a message using the text looked up from the domain-size-limit-messages file for that domain. The From: field is filled in with the intended recipient of the message - appearing to be a reply.
   

This was actually very simple to put together once I realized what I needed to do. The above is based on what I found in the Exim FAQ

Configuration Tips

Maybe this is something I should have said in the begining, but at the time or writing this document, I had never set up an exim4 server, and the only exim3 server I had was used with the default debconf install. Therefore, if you see something on this page that could be done in a more elegant, more efficient or just plain better way, please send me a note.
I will post suggestions and updates to this page as they come in.

• UPDATE 2/23/2009: I received a report of this installation going off on top of Lenny as well. In addition, I updated the stuff about /var/run/clamav to also add write permissions to the group.
  
• UPDATE 9/13/2006: I have successfully installed this on an Ubuntu Dapper machine as well. The only thing that I needed to do was to enable the universe repository in /etc/apt/sources.list
  
• UPDATE 5/11/2006: Updated paths for alias files and such to match the configuration of my last install (makes things easier for me next time).
  
• UPDATE 1/11/2006: I just made some minor edits to explain a couple things that got left out of the article. Thanks to those that brought them to my attention.
  
• UPDATE 9/13/2005: I just configured a new mail server last month using the Sarge release of Debian. I did things a little different, and have updated this article to reflect the changes. Changes include using aptitude to install the packages rather than apt-get. Aptitude handles conflicting and broken packages much better. Also, I didn't go through the trouble of creating all the transports and such for spamassassin or Rules Du Jour. Instead, I installed the sa-exim package, and got rid of Rules Du Jour completely. I also am using the freshclam package to handle all the anti-virus updates.
  
• UPDATE 2/15/2005: Alex Tingle submitted some updates to the article that I have now added. He also supplied a link for a Spamassassin and Exim on Debian HOWTO that some of you may find interesting.
  

Exim4 + Maildir + Procmail on Debian etch mini-HOWTO

What is Procmail? From Wikipedia (http://en.wikipedia.org/wiki/Procmail):

Procmail is a mail delivery agent (MDA) or mail filter, a program to process incoming emails on a computer, widely used on Unix systems. It is typically invoked from an MTA like Sendmail; this makes the mail processing event-driven. The companion-tool formail allows procmail to be used in batch-processing on mail that already is in your mailbox.
Common operations carried out with procmail include filtering and sorting of emails into different folders according to keywords in from, to, subject, text of the mail, or sending autoreplies, but more sophisticated operations are also possible.
A common practice is to let procmail call an external spam filter program, such as SpamAssassin. This method can allow for spam to be filtered or even deleted.

We can combine Procmail with Exim4, filter mail with some extra rules, and deliver incoming mail into other location. Working this on Debian etch is not too difficult.
First of all, we will need the Procmail package (you usually have this already):

apt-get install procmail

Then we will need the /etc/procmailrc, which defined the rules that will be apply. We can use the example as references:

cat /usr/share/doc/procmail/examples/3procmailrc  > /etc/procmailrc

NOTICE!! This sample is target for mailbox format, and so we will need to take the following changes for Maildir support:

MAILDIR =       $HOME/Maildir/  # You'd better make sure it exists
DEFAULT =       $MAILDIR
#LOGFILE =      $MAILDIR/from
LOCKFILE=       $HOME/.lockmail

By default, Exim4 will route incoming mail to procmail if /etc/procmailrc exists, and transport it though procmail_pipe. you don't need to take extra handling for this ;-)

Ntpdate has no effect

ntpdate(1) has no effect

Xen by default just uses the dom0's clock, which isn't updated within the domU's. Either set /proc/xen/independent_wallclock to 1 (so that this domU has an indepedent clock from the host dom0, or set the clock in the dom0.

Making a tape drive available to a guest via iSCSI

Making a tape drive available to a guest via iSCSI

This is specifically for Citrix XenServer, although the principles will of course work in other Xen implementations

I recently had a scenario where I was replacing two Windows servers with XenServer guests. This was fine, but we needed a way to backup to the existing SCSI DDS4 DAT drive. After failing to make PCI passthrough work, I settled on the much nicer method of providing the tape drive via an iSCSI target on the XenServer Host (Dom0). Here is how I achieved this.

Note 1: This is totally unsupported by Citrix

Note 2: I've used the XenServer terminology "host" instead of Dom0, as this applies to the Citrix commercial implementation of Xen. It will probably work fine on OSS Xen, but you can just install the normal kernel dev packages and ignore the DDK stuff.

Note 3: This is for XenServer 4.1.0, but the principles are the same for previous versions. Just ensure you understand each step rather than following blindly.

Note 4: You'll need to enable yum repositories. Do this by editing /etc/yum.repos.d/CentOS-Base.repo, and set "enabled=1" for the Base, Updates and Addons repositories

Note 5: Thanks to the wonderful work of Blake-r the rawio patch has now been updated to work against iscsitarget-0.4.17 - when using this against a kernel newer than 2.6.22, you'll need to edit kernel/target_raw.c and replace all the psg.page occurrences with psg.page_link due to changes in the scatterlist struct. To take advantage of this, just substitute the newer versions of iscsitarget and the rawio patch in these instructions. You should be able to keep all the instructions the same, but I've not tested this yet.

• Download the Xen 4.1 DDK from citrix.com
• The DDK is an ISO containing a VM with a development environment. Import it to your Xen host, and start it.
• Download iscsitarget 0.4.14 (you MUST use this version, as the patch for rawio support will not cleanly apply to 0.4.15) from http://optusnet.dl.sourceforge.net/sourceforge/iscsitarget/iscsitarget-0.4.14.tar.gz into your DDK VM.
• Download the patch for rawio support from http://sourceforge.net/mailarchive/attachment.php?list_name=iscsitarget-devel&message_id=1170171101.2822.23.camel@localhost.localdomain&counter=1 into your DDK VM (assuming you're saving it in /tmp)
• Now do the following:

yum install kernel-devel bison flex
tar -zxvf iscsitarget-0.4.14.tar.gz
cd iscsitarget-0.4.14
patch -p0 < /tmp/raw.p
make

• scp the entire iscsitarget-0.4.14 directory to your destination Xen host, and on that host (after enabling the base repo in /etc/yum.repos.d/CentOS-Base.repo) do:

yum install make gcc
cd iscsitarget-0.4.14
make install
mkdir /lib/modules/`uname -r`/kernel/iscsi
cp kernel/iscsi_trgt.ko /lib/modules/`uname -r`/kernel/iscsi
depmod -aq

The last three steps are required because make install will not copy the kernel module correctly outside the target environment.

• Now edit your /etc/ietd.conf and configure the tape as per the following example snippet (cat /proc/scsi/scsi for the correct HCIL values for your SCSI tape drive, this is an example only):

Target iqn.2007-04.com.example:tape0
     Lun 0 H=1,C=0,I=6,L=0,Type=rawio
     Type 1

• Save and do /etc/init.d/iscsi-target start
• Modify /etc/sysconfig/iptables to allow port 3260 tcp from the IP addresses running the initiator.
• Attach to the target using the initiator of your choice.

Working around pesky controllers which change HCIL at boot

See the following, requested by Simon Larsen at http://fragged.nu/2009/05/making-a-tape-drive-available-to-a-guest-via-iscsi/

Simon, if you read this, your captcha on your blog is broken, preventing comments, and there's no contact email address. Hope you find this, and find it useful. ;)

Just bung it in your rc.local.

#!/bin/bash
#
# This script is a (very) primitive method of determining the current HCIL info
# for /etc/ietd.conf (config file for iscsi-target) to work around some
# controllers which change this ID at boot time. You'll probably want to add
# some data validation of those vars as this is purely for my environments.
# Feel free to adapt or use it in any way you see fit.
# Greig McGill. August, 2009

# First set some vars

ietd="/etc/ietd.conf"
iqn="iqn.2009-08.nz.org.aol.internal:tape0"

# Get the whole HCIL string representing the tape drive.
# IMPORTANT NOTE: I am assuming there is only one sequential access device.
# If I am wrong, you WILL need to rewrite this, or badness will happen.
# You have been warned.

HCIL="`cat /proc/scsi/scsi | grep -B2 Sequential | head -1`"

# Now get each component

H="`echo $HCIL | cut -c 11`"
C="`echo $HCIL | cut -c 23`"
I="`echo $HCIL | cut -c 30`"
L="`echo $HCIL | cut -c 38`"

# Got all that, now generate ietd.conf and restart iscsi-target

cat <<EOT >$ietd
Target $iqn
        Lun 0 H=$H,C=$C,I=$I,L=$L,Type=rawio
        Type 1
EOT

/etc/init.d/iscsi-target restart

# exit with no error

exit 0

See iSCSINotes as well for more information

Mutt & Maildir Mini-HOWTO

Introduction

This document describes how to use the Maildir format with the Mutt MUA.
It has technical and performance advantages over the mbox format.


It is supported by MTA's like Exim, Postfix and qmail, MDA's like maildrop and procmail and IMAP4/POP3
servers like Dovecot and Courier to just name a few.


This document uses the Maildir++ directory layout for Maildir subfolders.

Using Maildir format with Mutt

Basic setup

You should already have setup your system to use Maildir.
This assumes your Maildir being ~/Maildir and that it contains Drafts and Sent subfolders as commonly used by other MUAs via IMAP4.

Mutt configuration

This section describes the statements to put into your muttrc.

First tell Mutt to use the Maildir format:

set mbox_type=Maildir

Next configure the locations of the common folders:

set folder="~/Maildir"
set mask="!^\\.[^.]"
set mbox="~/Maildir"
set record="+.Sent"
set postponed="+.Drafts"
set spoolfile="~/Maildir"

Set up mailboxes by scanning for all subfolders in ~/Maildir:

mailboxes `echo -n "+ "; find ~/Maildir -maxdepth 1 -type d -name ".*" -printf "+'%f' "`

Note that this requires find(1) from the GNU findutils, which on BSD systems usually is installed as find(1).


Add macros to make the folder browser usable by always using the mailboxes setup above:

macro index c "<change-folder>?<toggle-mailboxes>" "open a different folder"
macro pager c "<change-folder>?<toggle-mailboxes>" "open a different folder"

To additionally get straight to the folder browser when copying, moving and attaching mail,
add the following macros:

macro index C "<copy-message>?<toggle-mailboxes>" "copy a message to a mailbox"
macro index M "<save-message>?<toggle-mailboxes>" "move a message to a mailbox"

macro compose A "<attach-message>?<toggle-mailboxes>" "attach message(s) to this message"

License

This document is free documentation; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software Foundation;
either version 2 of the License, or (at your option) any later version.

Acknowledgment

Thanks to Alexandre Beelen for suggesting a fix to the original mailboxes
command to handle folder names that contain white space, to Martin Steigerwald for
suggesting to simplify the mailboxes command by using find(1),
to Dave Kiddell for pointing out that the latter depends on a
GNU findutils specific feature and to Abel Daniel for pointing out some minor errors.