re-mqueue.pl revision 182352 
1#!/usr/bin/perl
2#
3# re-mqueue -- requeue messages from queueA to queueB based on age.
4#
5#	Contributed by Paul Pomes <ppomes@Qualcomm.COM>.
6#		http://www.qualcomm.com/~ppomes/
7#
8# Usage: re-mqueue [-d] queueA queueB seconds
9#
10#  -d		enable debugging
11#  queueA	source directory
12#  queueB	destination directory
13#  seconds	select files older than this number of seconds
14#
15# Example: re-mqueue /var/spool/mqueue /var/spool/mqueue2 2700
16#
17# Moves the qf* and df* files for a message from /var/spool/mqueue to
18# /var/spool/mqueue2 if the df* file is over 2700 seconds old.
19#
20# The qf* file can't be used for age checking as it's partially re-written
21# with the results of the last queue run.
22#
23# Rationale: With a limited number of sendmail processes allowed to run,
24# messages that can't be delivered immediately slow down the ones that can.
25# This becomes especially important when messages are being queued instead
26# of delivered right away, or when the queue becomes excessively deep.
27# By putting messages that have already failed one or more delivery attempts
28# into another queue, the primary queue can be kept small and fast.
29#
30# On postoffice.cso.uiuc.edu, the primary sendmail daemon runs the queue
31# every thirty minutes.  Messages over 45 minutues old are moved to
32# /var/spool/mqueue2 where sendmail runs every hour.  Messages more than
33# 3.25 hours old are moved to /var/spool/mqueue3 where sendmail runs every
34# four hours.  Messages more than a day old are moved to /var/spool/mqueue4
35# where sendmail runs three times a day.  The idea is that a message is
36# tried at least twice in the first three queues before being moved to the
37# old-age ghetto.
38#
39# (Each must be re-formed into a single line before using in crontab)
40#
41# 08 * * * *	/usr/local/libexec/re-mqueue /var/spool/mqueue ##						/var/spool/mqueue2 2700
42# 11 * * * *	/usr/lib/sendmail -oQ/var/spool/mqueue2 -q > ##							> /var/log/mqueue2 2>&1
43# 38 * * * *	/usr/local/libexec/re-mqueue /var/spool/mqueue2
44#					/var/spool/mqueue3 11700
45# 41 1,5,9,13,17,21 * * * /usr/lib/sendmail -oQ/var/spool/mqueue3 -q ##							> /var/log/mqueue3 2>&1
46# 48 * * * *	/usr/local/libexec/re-mqueue /var/spool/mqueue3
47#					/var/spool/mqueue4 100000
48#53 3,11,19 * * * /usr/lib/sendmail -oQ/var/spool/mqueue4 -q > ##							> /var/log/mqueue4 2>&1
49#
50#
51# N.B., the moves are done with link().  This has two effects: 1) the mqueue*
52# directories must all be on the same filesystem, and 2) the file modification
53# times are not changed.  All times must be cumulative from when the df*
54# file was created.
55#
56# Copyright (c) 1995 University of Illinois Board of Trustees and Paul Pomes
57# All rights reserved.
58#
59# Redistribution and use in source and binary forms, with or without
60# modification, are permitted provided that the following conditions
61# are met:
62# 1. Redistributions of source code must retain the above copyright
63#    notice, this list of conditions and the following disclaimer.
64# 2. Redistributions in binary form must reproduce the above copyright
65#    notice, this list of conditions and the following disclaimer in the
66#    documentation and/or other materials provided with the distribution.
67# 3. All advertising materials mentioning features or use of this software
68#    must display the following acknowledgement:
69#       This product includes software developed by the University of
70#       Illinois at Urbana and their contributors.
71# 4. Neither the name of the University nor the names of their contributors
72#    may be used to endorse or promote products derived from this software
73#    without specific prior written permission.
74#
75# THIS SOFTWARE IS PROVIDED BY THE TRUSTEES AND CONTRIBUTORS ``AS IS'' AND
76# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
77# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
78# ARE DISCLAIMED.  IN NO EVENT SHALL THE TRUSTEES OR CONTRIBUTORS BE LIABLE
79# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
80# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
81# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
82# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
83# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
84# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
85# SUCH DAMAGE.
86#
87# @(#)$OrigId: re-mqueue,v 1.3 1995/05/25 18:14:53 p-pomes Exp $
88#
89# Updated by Graeme Hewson <ghewson@uk.oracle.com> May 1999
90#
91#	'use Sys::Syslog' for Perl 5
92#	Move transcript (xf) files if they exist
93#	Allow zero-length df files (empty message body)
94#	Preserve $! for error messages
95#
96# Updated by Graeme Hewson <ghewson@uk.oracle.com> April 2000
97#
98#	Improve handling of race between re-mqueue and sendmail
99#
100# Updated by Graeme Hewson <graeme.hewson@oracle.com> June 2000
101#
102#	Don't exit(0) at end so can be called as subroutine
103#
104# NB This program can't handle separate qf/df/xf subdirectories
105# as introduced in sendmail 8.10.0.
106#
107
108use Sys::Syslog;
109
110$LOCK_EX = 2;
111$LOCK_NB = 4;
112$LOCK_UN = 8;
113
114# Count arguments, exit if wrong in any way.
115die "Usage: $0 [-d] queueA queueB seconds\n" if ($#ARGV < 2);
116
117while ($_ = $ARGV[0], /^-/) {
118    shift;
119    last if /^--$/;
120    /^-d/ && $debug++;
121}
122
123$queueA = shift;
124$queueB = shift;
125$age = shift;
126
127die "$0: $queueA not a directory\n" if (! -d $queueA);
128die "$0: $queueB not a directory\n" if (! -d $queueB);
129die "$0: $age isn't a valid number of seconds for age\n" if ($age =~ /\D/);
130
131# chdir to $queueA and read the directory.  When a df* file is found, stat it.
132# If it's older than $age, lock the corresponding qf* file.  If the lock
133# fails, give up and move on.  Once the lock is obtained, verify that files
134# of the same name *don't* already exist in $queueB and move on if they do.
135# Otherwise re-link the qf* and df* files into $queueB then release the lock.
136
137chdir "$queueA" || die "$0: can't cd to $queueA: $!\n";
138opendir (QA, ".") || die "$0: can't open directory $queueA for reading: $!\n";
139@dfiles = grep(/^df/, readdir(QA));
140$now = time();
141($program = $0) =~ s,.*/,,;
142&openlog($program, 'pid', 'mail');
143
144# Loop through the dfiles
145while ($dfile = pop(@dfiles)) {
146    print "Checking $dfile\n" if ($debug);
147    ($qfile = $dfile) =~ s/^d/q/;
148    ($xfile = $dfile) =~ s/^d/x/;
149    ($mfile = $dfile) =~ s/^df//;
150    if (! -e $qfile || -z $qfile) {
151	print "$qfile is gone or zero bytes - skipping\n" if ($debug);
152	next;
153    }
154
155    ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
156     $atime,$mtime,$ctime,$blksize,$blocks) = stat($dfile);
157    if (! defined $mtime) {
158	print "$dfile is gone - skipping\n" if ($debug);
159	next;
160    }
161
162    # Compare timestamps
163    if (($mtime + $age) > $now) {
164	printf ("%s is %d seconds old - skipping\n", $dfile, $now-$mtime) if ($debug);
165	next;
166    }
167
168    # See if files of the same name already exist in $queueB
169    if (-e "$queueB/$dfile") {
170	print "$queueb/$dfile already exists - skipping\n" if ($debug);
171	next;
172    }
173    if (-e "$queueB/$qfile") {
174	print "$queueb/$qfile already exists - skipping\n" if ($debug);
175	next;
176    }
177    if (-e "$queueB/$xfile") {
178	print "$queueb/$xfile already exists - skipping\n" if ($debug);
179	next;
180    }
181
182    # Try and lock qf* file
183    unless (open(QF, ">>$qfile")) {
184	print "$qfile: $!\n" if ($debug);
185	next;
186    }
187    $retval = flock(QF, $LOCK_EX|$LOCK_NB) || ($retval = -1);
188    if ($retval == -1) {
189	print "$qfile already flock()ed - skipping\n" if ($debug);
190	close(QF);
191	next;
192    }
193    print "$qfile now flock()ed\n" if ($debug);
194
195    # Check df* file again in case sendmail got in
196    if (! -e $dfile) {
197	print "$mfile sent - skipping\n" if ($debug);
198	# qf* file created by ourselves at open? (Almost certainly)
199	if (-z $qfile) {
200	   unlink($qfile);
201	}
202	close(QF);
203	next;
204    }
205
206    # Show time!  Do the link()s
207    if (link("$dfile", "$queueB/$dfile") == 0) {
208	$bang = $!;
209	&syslog('err', 'link(%s, %s/%s): %s', $dfile, $queueB, $dfile, $bang);
210	print STDERR "$0: link($dfile, $queueB/$dfile): $bang\n";
211	exit (1);
212    }
213    if (link("$qfile", "$queueB/$qfile") == 0) {
214	$bang = $!;
215	&syslog('err', 'link(%s, %s/%s): %s', $qfile, $queueB, $qfile, $bang);
216	print STDERR "$0: link($qfile, $queueB/$qfile): $bang\n";
217	unlink("$queueB/$dfile");
218	exit (1);
219    }
220    if (-e "$xfile") {
221	if (link("$xfile", "$queueB/$xfile") == 0) {
222	    $bang = $!;
223	    &syslog('err', 'link(%s, %s/%s): %s', $xfile, $queueB, $xfile, $bang);
224	    print STDERR "$0: link($xfile, $queueB/$xfile): $bang\n";
225	    unlink("$queueB/$dfile");
226	    unlink("$queueB/$qfile");
227	    exit (1);
228	}
229    }
230
231    # Links created successfully.  Unlink the original files, release the
232    # lock, and close the file.
233    print "links ok\n" if ($debug);
234    if (unlink($qfile) == 0) {
235	$bang = $!;
236	&syslog('err', 'unlink(%s): %s', $qfile, $bang);
237	print STDERR "$0: unlink($qfile): $bang\n";
238	exit (1);
239    }
240    if (unlink($dfile) == 0) {
241	$bang = $!;
242	&syslog('err', 'unlink(%s): %s', $dfile, $bang);
243	print STDERR "$0: unlink($dfile): $bang\n";
244	exit (1);
245    }
246    if (-e "$xfile") {
247	if (unlink($xfile) == 0) {
248	    $bang = $!;
249	    &syslog('err', 'unlink(%s): %s', $xfile, $bang);
250	    print STDERR "$0: unlink($xfile): $bang\n";
251	    exit (1);
252	}
253    }
254    flock(QF, $LOCK_UN);
255    close(QF);
256    &syslog('info', '%s moved to %s', $mfile, $queueB);
257    print "Done with $dfile $qfile\n\n" if ($debug);
258}
259