1;;; mh-junk.el --- MH-E interface to anti-spam measures
2
3;; Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc.
4
5;; Author: Satyaki Das <satyaki@theforce.stanford.edu>,
6;;         Bill Wohler <wohler@newt.com>
7;; Maintainer: Bill Wohler <wohler@newt.com>
8;; Keywords:  mail, spam
9
10;; This file is part of GNU Emacs.
11
12;; GNU Emacs is free software; you can redistribute it and/or modify
13;; it under the terms of the GNU General Public License as published by
14;; the Free Software Foundation; either version 2, or (at your option)
15;; any later version.
16
17;; GNU Emacs is distributed in the hope that it will be useful,
18;; but WITHOUT ANY WARRANTY; without even the implied warranty of
19;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20;; GNU General Public License for more details.
21
22;; You should have received a copy of the GNU General Public License
23;; along with GNU Emacs; see the file COPYING.  If not, write to the
24;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
25;; Boston, MA 02110-1301, USA.
26
27;;; Commentary:
28
29;; Spam handling in MH-E.
30
31;;; Change Log:
32
33;;; Code:
34
35(require 'mh-e)
36(require 'mh-scan)
37(mh-require-cl)
38
39;;;###mh-autoload
40(defun mh-junk-blacklist (range)
41  "Blacklist RANGE as spam.
42
43This command trains the spam program in use (see the option
44`mh-junk-program') with the content of RANGE and then handles the
45message(s) as specified by the option `mh-junk-disposition'.
46
47Check the documentation of `mh-interactive-range' to see how RANGE is
48read in interactive use.
49
50For more information about using your particular spam fighting
51program, see:
52
53  - `mh-spamassassin-blacklist'
54  - `mh-bogofilter-blacklist'
55  - `mh-spamprobe-blacklist'"
56  (interactive (list (mh-interactive-range "Blacklist")))
57  (let ((blacklist-func (nth 1 (assoc mh-junk-choice mh-junk-function-alist))))
58    (unless blacklist-func
59      (error "Customize `mh-junk-program' appropriately"))
60    (let ((dest (cond ((null mh-junk-disposition) nil)
61                      ((equal mh-junk-disposition "") "+")
62                      ((eq (aref mh-junk-disposition 0) ?+)
63                       mh-junk-disposition)
64                      ((eq (aref mh-junk-disposition 0) ?@)
65                       (concat mh-current-folder "/"
66                               (substring mh-junk-disposition 1)))
67                      (t (concat "+" mh-junk-disposition)))))
68      (mh-iterate-on-range msg range
69        (message "Blacklisting message %d..." msg)
70        (funcall (symbol-function blacklist-func) msg)
71        (message "Blacklisting message %d...done" msg)
72        (if (not (memq msg mh-seen-list))
73            (setq mh-seen-list (cons msg mh-seen-list)))
74        (if dest
75            (mh-refile-a-msg nil (intern dest))
76          (mh-delete-a-msg nil)))
77      (mh-next-msg))))
78
79;;;###mh-autoload
80(defun mh-junk-whitelist (range)
81  "Whitelist RANGE as ham.
82
83This command reclassifies the RANGE as ham if it were incorrectly
84classified as spam (see the option `mh-junk-program'). It then
85refiles the message into the \"+inbox\" folder.
86
87Check the documentation of `mh-interactive-range' to see how
88RANGE is read in interactive use."
89  (interactive (list (mh-interactive-range "Whitelist")))
90  (let ((whitelist-func (nth 2 (assoc mh-junk-choice mh-junk-function-alist))))
91    (unless whitelist-func
92      (error "Customize `mh-junk-program' appropriately"))
93    (mh-iterate-on-range msg range
94      (message "Whitelisting message %d..." msg)
95      (funcall (symbol-function whitelist-func) msg)
96      (message "Whitelisting message %d...done" msg)
97      (mh-refile-a-msg nil (intern mh-inbox)))
98    (mh-next-msg)))
99
100
101
102;; Spamassassin Interface
103
104(defvar mh-spamassassin-executable (executable-find "spamassassin"))
105(defvar mh-sa-learn-executable (executable-find "sa-learn"))
106
107;;;###mh-autoload
108(defun mh-spamassassin-blacklist (msg)
109  "Blacklist MSG with SpamAssassin.
110
111SpamAssassin is one of the more popular spam filtering programs.
112Get it from your local distribution or from
113http://spamassassin.org/.
114
115To use SpamAssassin, add the following recipes to
116\".procmailrc\":
117
118    PATH=$PATH:/usr/bin/mh
119    MAILDIR=$HOME/`mhparam Path`
120
121    # Fight spam with SpamAssassin.
122    :0fw
123    | spamc
124
125    # Anything with a spam level of 10 or more is junked immediately.
126    :0:
127    * ^X-Spam-Level: ..........
128    /dev/null
129
130    :0:
131    * ^X-Spam-Status: Yes
132    spam/.
133
134If you don't use \"spamc\", use \"spamassassin -P -a\".
135
136Note that one of the recipes above throws away messages with a
137score greater than or equal to 10. Here's how you can determine a
138value that works best for you.
139
140First, run \"spamassassin -t\" on every mail message in your
141archive and use Gnumeric to verify that the average plus the
142standard deviation of good mail is under 5, the SpamAssassin
143default for \"spam\".
144
145Using Gnumeric, sort the messages by score and view the messages
146with the highest score. Determine the score which encompasses all
147of your interesting messages and add a couple of points to be
148conservative. Add that many dots to the \"X-Spam-Level:\" header
149field above to send messages with that score down the drain.
150
151In the example above, messages with a score of 5-9 are set aside
152in the \"+spam\" folder for later review. The major weakness of
153rules-based filters is a plethora of false positives so it is
154worthwhile to check.
155
156If SpamAssassin classifies a message incorrectly, or is unsure,
157you can use the MH-E commands \\[mh-junk-blacklist] and
158\\[mh-junk-whitelist].
159
160The command \\[mh-junk-blacklist] adds a \"blacklist_from\" entry
161to \"~/spamassassin/user_prefs\", deletes the message, and sends
162the message to the Razor, so that others might not see this spam.
163If the \"sa-learn\" command is available, the message is also
164recategorized as spam.
165
166The command \\[mh-junk-whitelist] adds a \"whitelist_from\" rule
167to the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\"
168command is available, the message is also recategorized as ham.
169
170Over time, you'll observe that the same host or domain occurs
171repeatedly in the \"blacklist_from\" entries, so you might think
172that you could avoid future spam by blacklisting all mail from a
173particular domain. The utility function
174`mh-spamassassin-identify-spammers' helps you do precisely that.
175This function displays a frequency count of the hosts and domains
176in the \"blacklist_from\" entries from the last blank line in
177\"~/.spamassassin/user_prefs\" to the end of the file. This
178information can be used so that you can replace multiple
179\"blacklist_from\" entries with a single wildcard entry such as:
180
181    blacklist_from *@*amazingoffersdirect2u.com
182
183In versions of SpamAssassin (2.50 and on) that support a Bayesian
184classifier, \\[mh-junk-blacklist] uses the program \"sa-learn\"
185to recategorize the message as spam. Neither MH-E, nor
186SpamAssassin, rebuilds the database after adding words, so you
187will need to run \"sa-learn --rebuild\" periodically. This can be
188done by adding the following to your crontab:
189
190    0 * * * *   sa-learn --rebuild > /dev/null 2>&1"
191  (unless mh-spamassassin-executable
192    (error "Unable to find the spamassassin executable"))
193  (let ((current-folder mh-current-folder)
194        (msg-file (mh-msg-filename msg mh-current-folder))
195        (sender))
196    (message "Reporting message %d..." msg)
197    (mh-truncate-log-buffer)
198    ;; Put call-process output in log buffer if we are saving it
199    ;; (this happens if mh-junk-background is t).
200    (with-current-buffer mh-log-buffer
201      (call-process mh-spamassassin-executable msg-file mh-junk-background nil
202                    ;;"--report" "--remove-from-whitelist"
203                    "-r" "-R")          ; spamassassin V2.20
204    (when mh-sa-learn-executable
205      (message "Recategorizing message %d as spam..." msg)
206      (mh-truncate-log-buffer)
207      (call-process mh-sa-learn-executable msg-file mh-junk-background nil
208                    "--single" "--spam" "--local" "--no-rebuild")))
209    (message "Blacklisting sender of message %d..." msg)
210    (with-current-buffer (get-buffer-create mh-temp-buffer)
211      (erase-buffer)
212      (call-process (expand-file-name mh-scan-prog mh-progs)
213                    nil t nil
214                    (format "%d" msg) current-folder
215                    "-format" "%<(mymbox{from})%|%(addr{from})%>")
216      (goto-char (point-min))
217      (if (search-forward-regexp "^\\(.+\\)$" nil t)
218          (progn
219            (setq sender (match-string 0))
220            (mh-spamassassin-add-rule "blacklist_from" sender)
221            (message "Blacklisting sender of message %d...done" msg))
222        (message "Blacklisting sender of message %d...not done (from my address)" msg)))))
223
224;;;###mh-autoload
225(defun mh-spamassassin-whitelist (msg)
226  "Whitelist MSG with SpamAssassin.
227
228The \\[mh-junk-whitelist] command adds a \"whitelist_from\" rule to
229the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\" command
230is available, the message is also recategorized as ham.
231
232See `mh-spamassassin-blacklist' for more information."
233  (unless mh-spamassassin-executable
234    (error "Unable to find the spamassassin executable"))
235  (let ((msg-file (mh-msg-filename msg mh-current-folder))
236        (show-buffer (get-buffer mh-show-buffer))
237        from)
238    (with-current-buffer (get-buffer-create mh-temp-buffer)
239      (erase-buffer)
240      (message "Removing spamassassin markup from message %d..." msg)
241      (call-process mh-spamassassin-executable msg-file t nil
242                    ;; "--remove-markup"
243                    "-d")               ; spamassassin V2.20
244      (if show-buffer
245          (kill-buffer show-buffer))
246      (write-file msg-file)
247      (when mh-sa-learn-executable
248        (message "Recategorizing message %d as ham..." msg)
249        (mh-truncate-log-buffer)
250        ;; Put call-process output in log buffer if we are saving it
251        ;; (this happens if mh-junk-background is t).
252        (with-current-buffer mh-log-buffer
253          (call-process mh-sa-learn-executable msg-file mh-junk-background nil
254                        "--single" "--ham" "--local" "--no-rebuild")))
255      (message "Whitelisting sender of message %d..." msg)
256      (setq from
257            (car (mh-funcall-if-exists
258                  ietf-drums-parse-address (mh-get-header-field "From:"))))
259      (kill-buffer nil)
260      (unless (or (null from) (equal from ""))
261        (mh-spamassassin-add-rule "whitelist_from" from))
262      (message "Whitelisting sender of message %d...done" msg))))
263
264(defun mh-spamassassin-add-rule (rule body)
265  "Add a new rule to \"~/.spamassassin/user_prefs\".
266The name of the rule is RULE and its body is BODY."
267  (save-window-excursion
268    (let* ((line (format "%s\t%s\n" rule body))
269           (case-fold-search t)
270           (file (expand-file-name "~/.spamassassin/user_prefs"))
271           (buffer-exists (find-buffer-visiting file)))
272      (find-file file)
273      (if (not (search-forward (format "\n%s" line) nil t))
274          (progn
275            (goto-char (point-max))
276            (insert (if (bolp) "" "\n") line)
277            (save-buffer)))
278      (if (not buffer-exists)
279          (kill-buffer nil)))))
280
281;;;###mh-autoload
282(defun mh-spamassassin-identify-spammers ()
283  "Identify spammers who are repeat offenders.
284
285This function displays a frequency count of the hosts and domains
286in the \"blacklist_from\" entries from the last blank line in
287\"~/.spamassassin/user_prefs\" to the end of the file. This
288information can be used so that you can replace multiple
289\"blacklist_from\" entries with a single wildcard entry such as:
290
291    blacklist_from *@*amazingoffersdirect2u.com"
292  (interactive)
293  (let* ((file (expand-file-name "~/.spamassassin/user_prefs"))
294         (domains (make-hash-table :test 'equal)))
295    (find-file file)
296    ;; Only consider entries between last blank line and end of file.
297    (goto-char (1- (point-max)))
298    (search-backward-regexp "^$")
299    ;; Perform frequency count.
300    (save-excursion
301      (while (search-forward-regexp "^blacklist_from\\s-*\\(.*\\)@\\(.*\\)$"
302                                    nil t)
303        (let ((host (match-string 2))
304              value)
305          ;; Remove top-level-domain from hostname.
306          (setq host (cdr (reverse (split-string host "\\."))))
307          ;; Add counts for each host and domain part.
308          (while host
309            (setq value (gethash (car host) domains))
310            (setf (gethash (car host) domains) (1+ (if (not value) 0 value)))
311            (setq host (cdr host))))))
312
313    ;; Output
314    (delete-other-windows)
315    (pop-to-buffer (get-buffer-create "*MH-E Spammer Frequencies*"))
316    (erase-buffer)
317    (maphash '(lambda (key value) ""
318                (if (> value 2)
319                    (insert (format "%s %s\n" key value))))
320             domains)
321    (sort-numeric-fields 2 (point-min) (point-max))
322    (reverse-region (point-min) (point-max))
323    (goto-char (point-min))))
324
325
326
327;; Bogofilter Interface
328
329(defvar mh-bogofilter-executable (executable-find "bogofilter"))
330
331;;;###mh-autoload
332(defun mh-bogofilter-blacklist (msg)
333  "Blacklist MSG with bogofilter.
334
335Bogofilter is a Bayesian spam filtering program. Get it from your
336local distribution or from http://bogofilter.sourceforge.net/.
337
338Bogofilter is taught by running:
339
340    bogofilter -n < good-message
341
342on every good message, and
343
344    bogofilter -s < spam-message
345
346on every spam message. This is called a full training; three other
347training methods are described in the FAQ that is distributed with
348bogofilter. Note that most Bayesian filters need 1000 to 5000 of each
349type of message to start doing a good job.
350
351To use bogofilter, add the following recipes to \".procmailrc\":
352
353    PATH=$PATH:/usr/bin/mh
354    MAILDIR=$HOME/`mhparam Path`
355
356    # Fight spam with bogofilter.
357    :0fw
358    | bogofilter -3 -e -p
359
360    :0:
361    * ^X-Bogosity: Yes, tests=bogofilter
362    spam/.
363
364    :0:
365    * ^X-Bogosity: Unsure, tests=bogofilter
366    spam/unsure/.
367
368If bogofilter classifies a message incorrectly, or is unsure, you can
369use the MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist]
370to update bogofilter's training.
371
372The \"Bogofilter FAQ\" suggests that you run the following
373occasionally to shrink the database:
374
375    bogoutil -d wordlist.db | bogoutil -l wordlist.db.new
376    mv wordlist.db wordlist.db.prv
377    mv wordlist.db.new wordlist.db
378
379The \"Bogofilter tuning HOWTO\" describes how you can fine-tune Bogofilter."
380  (unless mh-bogofilter-executable
381    (error "Unable to find the bogofilter executable"))
382  (let ((msg-file (mh-msg-filename msg mh-current-folder)))
383    (mh-truncate-log-buffer)
384    ;; Put call-process output in log buffer if we are saving it
385    ;; (this happens if mh-junk-background is t).
386    (with-current-buffer mh-log-buffer
387      (call-process mh-bogofilter-executable msg-file mh-junk-background
388                    nil "-s"))))
389
390;;;###mh-autoload
391(defun mh-bogofilter-whitelist (msg)
392  "Whitelist MSG with bogofilter.
393
394See `mh-bogofilter-blacklist' for more information."
395  (unless mh-bogofilter-executable
396    (error "Unable to find the bogofilter executable"))
397  (let ((msg-file (mh-msg-filename msg mh-current-folder)))
398    (mh-truncate-log-buffer)
399    ;; Put call-process output in log buffer if we are saving it
400    ;; (this happens if mh-junk-background is t).
401    (with-current-buffer mh-log-buffer
402      (call-process mh-bogofilter-executable msg-file mh-junk-background
403                    nil "-n"))))
404
405
406
407;; Spamprobe Interface
408
409(defvar mh-spamprobe-executable (executable-find "spamprobe"))
410
411;;;###mh-autoload
412(defun mh-spamprobe-blacklist (msg)
413  "Blacklist MSG with SpamProbe.
414
415SpamProbe is a Bayesian spam filtering program. Get it from your local
416distribution or from http://spamprobe.sourceforge.net.
417
418To use SpamProbe, add the following recipes to \".procmailrc\":
419
420    PATH=$PATH:/usr/bin/mh
421    MAILDIR=$HOME/`mhparam Path`
422
423    # Fight spam with SpamProbe.
424    :0
425    SCORE=| spamprobe receive
426
427    :0 wf
428    | formail -I \"X-SpamProbe: $SCORE\"
429
430    :0:
431    *^X-SpamProbe: SPAM
432    spam/.
433
434If SpamProbe classifies a message incorrectly, you can use the
435MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist] to
436update SpamProbe's training."
437  (unless mh-spamprobe-executable
438    (error "Unable to find the spamprobe executable"))
439  (let ((msg-file (mh-msg-filename msg mh-current-folder)))
440    (mh-truncate-log-buffer)
441    ;; Put call-process output in log buffer if we are saving it
442    ;; (this happens if mh-junk-background is t).
443    (with-current-buffer mh-log-buffer
444      (call-process mh-spamprobe-executable msg-file mh-junk-background
445                    nil "spam"))))
446
447;;;###mh-autoload
448(defun mh-spamprobe-whitelist (msg)
449  "Whitelist MSG with SpamProbe.
450
451See `mh-spamprobe-blacklist' for more information."
452  (unless mh-spamprobe-executable
453    (error "Unable to find the spamprobe executable"))
454  (let ((msg-file (mh-msg-filename msg mh-current-folder)))
455    (mh-truncate-log-buffer)
456    ;; Put call-process output in log buffer if we are saving it
457    ;; (this happens if mh-junk-background is t).
458    (with-current-buffer mh-log-buffer
459      (call-process mh-spamprobe-executable msg-file mh-junk-background
460                    nil "good"))))
461
462(provide 'mh-junk)
463
464;; Local Variables:
465;; indent-tabs-mode: nil
466;; sentence-end-double-space: nil
467;; End:
468
469;; arch-tag: 603335f1-77ff-4306-8828-5d3dad51abe1
470;;; mh-junk.el ends here
471