1;;; mh-junk.el --- MH-E interface to anti-spam measures 2 3;; Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc. 4 5;; Author: Satyaki Das <satyaki@theforce.stanford.edu>, 6;; Bill Wohler <wohler@newt.com> 7;; Maintainer: Bill Wohler <wohler@newt.com> 8;; Keywords: mail, spam 9 10;; This file is part of GNU Emacs. 11 12;; GNU Emacs is free software; you can redistribute it and/or modify 13;; it under the terms of the GNU General Public License as published by 14;; the Free Software Foundation; either version 2, or (at your option) 15;; any later version. 16 17;; GNU Emacs is distributed in the hope that it will be useful, 18;; but WITHOUT ANY WARRANTY; without even the implied warranty of 19;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20;; GNU General Public License for more details. 21 22;; You should have received a copy of the GNU General Public License 23;; along with GNU Emacs; see the file COPYING. If not, write to the 24;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, 25;; Boston, MA 02110-1301, USA. 26 27;;; Commentary: 28 29;; Spam handling in MH-E. 30 31;;; Change Log: 32 33;;; Code: 34 35(require 'mh-e) 36(require 'mh-scan) 37(mh-require-cl) 38 39;;;###mh-autoload 40(defun mh-junk-blacklist (range) 41 "Blacklist RANGE as spam. 42 43This command trains the spam program in use (see the option 44`mh-junk-program') with the content of RANGE and then handles the 45message(s) as specified by the option `mh-junk-disposition'. 46 47Check the documentation of `mh-interactive-range' to see how RANGE is 48read in interactive use. 49 50For more information about using your particular spam fighting 51program, see: 52 53 - `mh-spamassassin-blacklist' 54 - `mh-bogofilter-blacklist' 55 - `mh-spamprobe-blacklist'" 56 (interactive (list (mh-interactive-range "Blacklist"))) 57 (let ((blacklist-func (nth 1 (assoc mh-junk-choice mh-junk-function-alist)))) 58 (unless blacklist-func 59 (error "Customize `mh-junk-program' appropriately")) 60 (let ((dest (cond ((null mh-junk-disposition) nil) 61 ((equal mh-junk-disposition "") "+") 62 ((eq (aref mh-junk-disposition 0) ?+) 63 mh-junk-disposition) 64 ((eq (aref mh-junk-disposition 0) ?@) 65 (concat mh-current-folder "/" 66 (substring mh-junk-disposition 1))) 67 (t (concat "+" mh-junk-disposition))))) 68 (mh-iterate-on-range msg range 69 (message "Blacklisting message %d..." msg) 70 (funcall (symbol-function blacklist-func) msg) 71 (message "Blacklisting message %d...done" msg) 72 (if (not (memq msg mh-seen-list)) 73 (setq mh-seen-list (cons msg mh-seen-list))) 74 (if dest 75 (mh-refile-a-msg nil (intern dest)) 76 (mh-delete-a-msg nil))) 77 (mh-next-msg)))) 78 79;;;###mh-autoload 80(defun mh-junk-whitelist (range) 81 "Whitelist RANGE as ham. 82 83This command reclassifies the RANGE as ham if it were incorrectly 84classified as spam (see the option `mh-junk-program'). It then 85refiles the message into the \"+inbox\" folder. 86 87Check the documentation of `mh-interactive-range' to see how 88RANGE is read in interactive use." 89 (interactive (list (mh-interactive-range "Whitelist"))) 90 (let ((whitelist-func (nth 2 (assoc mh-junk-choice mh-junk-function-alist)))) 91 (unless whitelist-func 92 (error "Customize `mh-junk-program' appropriately")) 93 (mh-iterate-on-range msg range 94 (message "Whitelisting message %d..." msg) 95 (funcall (symbol-function whitelist-func) msg) 96 (message "Whitelisting message %d...done" msg) 97 (mh-refile-a-msg nil (intern mh-inbox))) 98 (mh-next-msg))) 99 100 101 102;; Spamassassin Interface 103 104(defvar mh-spamassassin-executable (executable-find "spamassassin")) 105(defvar mh-sa-learn-executable (executable-find "sa-learn")) 106 107;;;###mh-autoload 108(defun mh-spamassassin-blacklist (msg) 109 "Blacklist MSG with SpamAssassin. 110 111SpamAssassin is one of the more popular spam filtering programs. 112Get it from your local distribution or from 113http://spamassassin.org/. 114 115To use SpamAssassin, add the following recipes to 116\".procmailrc\": 117 118 PATH=$PATH:/usr/bin/mh 119 MAILDIR=$HOME/`mhparam Path` 120 121 # Fight spam with SpamAssassin. 122 :0fw 123 | spamc 124 125 # Anything with a spam level of 10 or more is junked immediately. 126 :0: 127 * ^X-Spam-Level: .......... 128 /dev/null 129 130 :0: 131 * ^X-Spam-Status: Yes 132 spam/. 133 134If you don't use \"spamc\", use \"spamassassin -P -a\". 135 136Note that one of the recipes above throws away messages with a 137score greater than or equal to 10. Here's how you can determine a 138value that works best for you. 139 140First, run \"spamassassin -t\" on every mail message in your 141archive and use Gnumeric to verify that the average plus the 142standard deviation of good mail is under 5, the SpamAssassin 143default for \"spam\". 144 145Using Gnumeric, sort the messages by score and view the messages 146with the highest score. Determine the score which encompasses all 147of your interesting messages and add a couple of points to be 148conservative. Add that many dots to the \"X-Spam-Level:\" header 149field above to send messages with that score down the drain. 150 151In the example above, messages with a score of 5-9 are set aside 152in the \"+spam\" folder for later review. The major weakness of 153rules-based filters is a plethora of false positives so it is 154worthwhile to check. 155 156If SpamAssassin classifies a message incorrectly, or is unsure, 157you can use the MH-E commands \\[mh-junk-blacklist] and 158\\[mh-junk-whitelist]. 159 160The command \\[mh-junk-blacklist] adds a \"blacklist_from\" entry 161to \"~/spamassassin/user_prefs\", deletes the message, and sends 162the message to the Razor, so that others might not see this spam. 163If the \"sa-learn\" command is available, the message is also 164recategorized as spam. 165 166The command \\[mh-junk-whitelist] adds a \"whitelist_from\" rule 167to the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\" 168command is available, the message is also recategorized as ham. 169 170Over time, you'll observe that the same host or domain occurs 171repeatedly in the \"blacklist_from\" entries, so you might think 172that you could avoid future spam by blacklisting all mail from a 173particular domain. The utility function 174`mh-spamassassin-identify-spammers' helps you do precisely that. 175This function displays a frequency count of the hosts and domains 176in the \"blacklist_from\" entries from the last blank line in 177\"~/.spamassassin/user_prefs\" to the end of the file. This 178information can be used so that you can replace multiple 179\"blacklist_from\" entries with a single wildcard entry such as: 180 181 blacklist_from *@*amazingoffersdirect2u.com 182 183In versions of SpamAssassin (2.50 and on) that support a Bayesian 184classifier, \\[mh-junk-blacklist] uses the program \"sa-learn\" 185to recategorize the message as spam. Neither MH-E, nor 186SpamAssassin, rebuilds the database after adding words, so you 187will need to run \"sa-learn --rebuild\" periodically. This can be 188done by adding the following to your crontab: 189 190 0 * * * * sa-learn --rebuild > /dev/null 2>&1" 191 (unless mh-spamassassin-executable 192 (error "Unable to find the spamassassin executable")) 193 (let ((current-folder mh-current-folder) 194 (msg-file (mh-msg-filename msg mh-current-folder)) 195 (sender)) 196 (message "Reporting message %d..." msg) 197 (mh-truncate-log-buffer) 198 ;; Put call-process output in log buffer if we are saving it 199 ;; (this happens if mh-junk-background is t). 200 (with-current-buffer mh-log-buffer 201 (call-process mh-spamassassin-executable msg-file mh-junk-background nil 202 ;;"--report" "--remove-from-whitelist" 203 "-r" "-R") ; spamassassin V2.20 204 (when mh-sa-learn-executable 205 (message "Recategorizing message %d as spam..." msg) 206 (mh-truncate-log-buffer) 207 (call-process mh-sa-learn-executable msg-file mh-junk-background nil 208 "--single" "--spam" "--local" "--no-rebuild"))) 209 (message "Blacklisting sender of message %d..." msg) 210 (with-current-buffer (get-buffer-create mh-temp-buffer) 211 (erase-buffer) 212 (call-process (expand-file-name mh-scan-prog mh-progs) 213 nil t nil 214 (format "%d" msg) current-folder 215 "-format" "%<(mymbox{from})%|%(addr{from})%>") 216 (goto-char (point-min)) 217 (if (search-forward-regexp "^\\(.+\\)$" nil t) 218 (progn 219 (setq sender (match-string 0)) 220 (mh-spamassassin-add-rule "blacklist_from" sender) 221 (message "Blacklisting sender of message %d...done" msg)) 222 (message "Blacklisting sender of message %d...not done (from my address)" msg))))) 223 224;;;###mh-autoload 225(defun mh-spamassassin-whitelist (msg) 226 "Whitelist MSG with SpamAssassin. 227 228The \\[mh-junk-whitelist] command adds a \"whitelist_from\" rule to 229the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\" command 230is available, the message is also recategorized as ham. 231 232See `mh-spamassassin-blacklist' for more information." 233 (unless mh-spamassassin-executable 234 (error "Unable to find the spamassassin executable")) 235 (let ((msg-file (mh-msg-filename msg mh-current-folder)) 236 (show-buffer (get-buffer mh-show-buffer)) 237 from) 238 (with-current-buffer (get-buffer-create mh-temp-buffer) 239 (erase-buffer) 240 (message "Removing spamassassin markup from message %d..." msg) 241 (call-process mh-spamassassin-executable msg-file t nil 242 ;; "--remove-markup" 243 "-d") ; spamassassin V2.20 244 (if show-buffer 245 (kill-buffer show-buffer)) 246 (write-file msg-file) 247 (when mh-sa-learn-executable 248 (message "Recategorizing message %d as ham..." msg) 249 (mh-truncate-log-buffer) 250 ;; Put call-process output in log buffer if we are saving it 251 ;; (this happens if mh-junk-background is t). 252 (with-current-buffer mh-log-buffer 253 (call-process mh-sa-learn-executable msg-file mh-junk-background nil 254 "--single" "--ham" "--local" "--no-rebuild"))) 255 (message "Whitelisting sender of message %d..." msg) 256 (setq from 257 (car (mh-funcall-if-exists 258 ietf-drums-parse-address (mh-get-header-field "From:")))) 259 (kill-buffer nil) 260 (unless (or (null from) (equal from "")) 261 (mh-spamassassin-add-rule "whitelist_from" from)) 262 (message "Whitelisting sender of message %d...done" msg)))) 263 264(defun mh-spamassassin-add-rule (rule body) 265 "Add a new rule to \"~/.spamassassin/user_prefs\". 266The name of the rule is RULE and its body is BODY." 267 (save-window-excursion 268 (let* ((line (format "%s\t%s\n" rule body)) 269 (case-fold-search t) 270 (file (expand-file-name "~/.spamassassin/user_prefs")) 271 (buffer-exists (find-buffer-visiting file))) 272 (find-file file) 273 (if (not (search-forward (format "\n%s" line) nil t)) 274 (progn 275 (goto-char (point-max)) 276 (insert (if (bolp) "" "\n") line) 277 (save-buffer))) 278 (if (not buffer-exists) 279 (kill-buffer nil))))) 280 281;;;###mh-autoload 282(defun mh-spamassassin-identify-spammers () 283 "Identify spammers who are repeat offenders. 284 285This function displays a frequency count of the hosts and domains 286in the \"blacklist_from\" entries from the last blank line in 287\"~/.spamassassin/user_prefs\" to the end of the file. This 288information can be used so that you can replace multiple 289\"blacklist_from\" entries with a single wildcard entry such as: 290 291 blacklist_from *@*amazingoffersdirect2u.com" 292 (interactive) 293 (let* ((file (expand-file-name "~/.spamassassin/user_prefs")) 294 (domains (make-hash-table :test 'equal))) 295 (find-file file) 296 ;; Only consider entries between last blank line and end of file. 297 (goto-char (1- (point-max))) 298 (search-backward-regexp "^$") 299 ;; Perform frequency count. 300 (save-excursion 301 (while (search-forward-regexp "^blacklist_from\\s-*\\(.*\\)@\\(.*\\)$" 302 nil t) 303 (let ((host (match-string 2)) 304 value) 305 ;; Remove top-level-domain from hostname. 306 (setq host (cdr (reverse (split-string host "\\.")))) 307 ;; Add counts for each host and domain part. 308 (while host 309 (setq value (gethash (car host) domains)) 310 (setf (gethash (car host) domains) (1+ (if (not value) 0 value))) 311 (setq host (cdr host)))))) 312 313 ;; Output 314 (delete-other-windows) 315 (pop-to-buffer (get-buffer-create "*MH-E Spammer Frequencies*")) 316 (erase-buffer) 317 (maphash '(lambda (key value) "" 318 (if (> value 2) 319 (insert (format "%s %s\n" key value)))) 320 domains) 321 (sort-numeric-fields 2 (point-min) (point-max)) 322 (reverse-region (point-min) (point-max)) 323 (goto-char (point-min)))) 324 325 326 327;; Bogofilter Interface 328 329(defvar mh-bogofilter-executable (executable-find "bogofilter")) 330 331;;;###mh-autoload 332(defun mh-bogofilter-blacklist (msg) 333 "Blacklist MSG with bogofilter. 334 335Bogofilter is a Bayesian spam filtering program. Get it from your 336local distribution or from http://bogofilter.sourceforge.net/. 337 338Bogofilter is taught by running: 339 340 bogofilter -n < good-message 341 342on every good message, and 343 344 bogofilter -s < spam-message 345 346on every spam message. This is called a full training; three other 347training methods are described in the FAQ that is distributed with 348bogofilter. Note that most Bayesian filters need 1000 to 5000 of each 349type of message to start doing a good job. 350 351To use bogofilter, add the following recipes to \".procmailrc\": 352 353 PATH=$PATH:/usr/bin/mh 354 MAILDIR=$HOME/`mhparam Path` 355 356 # Fight spam with bogofilter. 357 :0fw 358 | bogofilter -3 -e -p 359 360 :0: 361 * ^X-Bogosity: Yes, tests=bogofilter 362 spam/. 363 364 :0: 365 * ^X-Bogosity: Unsure, tests=bogofilter 366 spam/unsure/. 367 368If bogofilter classifies a message incorrectly, or is unsure, you can 369use the MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist] 370to update bogofilter's training. 371 372The \"Bogofilter FAQ\" suggests that you run the following 373occasionally to shrink the database: 374 375 bogoutil -d wordlist.db | bogoutil -l wordlist.db.new 376 mv wordlist.db wordlist.db.prv 377 mv wordlist.db.new wordlist.db 378 379The \"Bogofilter tuning HOWTO\" describes how you can fine-tune Bogofilter." 380 (unless mh-bogofilter-executable 381 (error "Unable to find the bogofilter executable")) 382 (let ((msg-file (mh-msg-filename msg mh-current-folder))) 383 (mh-truncate-log-buffer) 384 ;; Put call-process output in log buffer if we are saving it 385 ;; (this happens if mh-junk-background is t). 386 (with-current-buffer mh-log-buffer 387 (call-process mh-bogofilter-executable msg-file mh-junk-background 388 nil "-s")))) 389 390;;;###mh-autoload 391(defun mh-bogofilter-whitelist (msg) 392 "Whitelist MSG with bogofilter. 393 394See `mh-bogofilter-blacklist' for more information." 395 (unless mh-bogofilter-executable 396 (error "Unable to find the bogofilter executable")) 397 (let ((msg-file (mh-msg-filename msg mh-current-folder))) 398 (mh-truncate-log-buffer) 399 ;; Put call-process output in log buffer if we are saving it 400 ;; (this happens if mh-junk-background is t). 401 (with-current-buffer mh-log-buffer 402 (call-process mh-bogofilter-executable msg-file mh-junk-background 403 nil "-n")))) 404 405 406 407;; Spamprobe Interface 408 409(defvar mh-spamprobe-executable (executable-find "spamprobe")) 410 411;;;###mh-autoload 412(defun mh-spamprobe-blacklist (msg) 413 "Blacklist MSG with SpamProbe. 414 415SpamProbe is a Bayesian spam filtering program. Get it from your local 416distribution or from http://spamprobe.sourceforge.net. 417 418To use SpamProbe, add the following recipes to \".procmailrc\": 419 420 PATH=$PATH:/usr/bin/mh 421 MAILDIR=$HOME/`mhparam Path` 422 423 # Fight spam with SpamProbe. 424 :0 425 SCORE=| spamprobe receive 426 427 :0 wf 428 | formail -I \"X-SpamProbe: $SCORE\" 429 430 :0: 431 *^X-SpamProbe: SPAM 432 spam/. 433 434If SpamProbe classifies a message incorrectly, you can use the 435MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist] to 436update SpamProbe's training." 437 (unless mh-spamprobe-executable 438 (error "Unable to find the spamprobe executable")) 439 (let ((msg-file (mh-msg-filename msg mh-current-folder))) 440 (mh-truncate-log-buffer) 441 ;; Put call-process output in log buffer if we are saving it 442 ;; (this happens if mh-junk-background is t). 443 (with-current-buffer mh-log-buffer 444 (call-process mh-spamprobe-executable msg-file mh-junk-background 445 nil "spam")))) 446 447;;;###mh-autoload 448(defun mh-spamprobe-whitelist (msg) 449 "Whitelist MSG with SpamProbe. 450 451See `mh-spamprobe-blacklist' for more information." 452 (unless mh-spamprobe-executable 453 (error "Unable to find the spamprobe executable")) 454 (let ((msg-file (mh-msg-filename msg mh-current-folder))) 455 (mh-truncate-log-buffer) 456 ;; Put call-process output in log buffer if we are saving it 457 ;; (this happens if mh-junk-background is t). 458 (with-current-buffer mh-log-buffer 459 (call-process mh-spamprobe-executable msg-file mh-junk-background 460 nil "good")))) 461 462(provide 'mh-junk) 463 464;; Local Variables: 465;; indent-tabs-mode: nil 466;; sentence-end-double-space: nil 467;; End: 468 469;; arch-tag: 603335f1-77ff-4306-8828-5d3dad51abe1 470;;; mh-junk.el ends here 471