1;;; spam-stat.el --- detecting spam based on statistics 2 3;; Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc. 4 5;; Author: Alex Schroeder <alex@gnu.org> 6;; Keywords: network 7;; URL: http://www.emacswiki.org/cgi-bin/wiki.pl?SpamStat 8 9;; This file is part of GNU Emacs. 10 11;; This is free software; you can redistribute it and/or modify it 12;; under the terms of the GNU General Public License as published by 13;; the Free Software Foundation; either version 2, or (at your option) 14;; any later version. 15 16;; This is distributed in the hope that it will be useful, but WITHOUT 17;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 18;; or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public 19;; License for more details. 20 21;; You should have received a copy of the GNU General Public License 22;; along with GNU Emacs; see the file COPYING. If not, write to the 23;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, 24;; Boston, MA 02110-1301, USA. 25 26;;; Commentary: 27 28;; This implements spam analysis according to Paul Graham in "A Plan 29;; for Spam". The basis for all this is a statistical distribution of 30;; words for your spam and non-spam mails. We need this information 31;; in a hash-table so that the analysis can use the information when 32;; looking at your mails. Therefore, before you begin, you need tons 33;; of mails (Graham uses 4000 non-spam and 4000 spam mails for his 34;; experiments). 35;; 36;; The main interface to using spam-stat, are the following functions: 37;; 38;; `spam-stat-buffer-is-spam' -- called in a buffer, that buffer is 39;; considered to be a new spam mail; use this for new mail that has 40;; not been processed before 41;; 42;; `spam-stat-buffer-is-non-spam' -- called in a buffer, that buffer 43;; is considered to be a new non-spam mail; use this for new mail that 44;; has not been processed before 45;; 46;; `spam-stat-buffer-change-to-spam' -- called in a buffer, that 47;; buffer is no longer considered to be normal mail but spam; use this 48;; to change the status of a mail that has already been processed as 49;; non-spam 50;; 51;; `spam-stat-buffer-change-to-non-spam' -- called in a buffer, that 52;; buffer is no longer considered to be spam but normal mail; use this 53;; to change the status of a mail that has already been processed as 54;; spam 55;; 56;; `spam-stat-save' -- save the hash table to the file; the filename 57;; used is stored in the variable `spam-stat-file' 58;; 59;; `spam-stat-load' -- load the hash table from a file; the filename 60;; used is stored in the variable `spam-stat-file' 61;; 62;; `spam-stat-score-word' -- return the spam score for a word 63;; 64;; `spam-stat-score-buffer' -- return the spam score for a buffer 65;; 66;; `spam-stat-split-fancy' -- for fancy mail splitting; add 67;; the rule (: spam-stat-split-fancy) to `nnmail-split-fancy' 68;; 69;; This requires the following in your ~/.gnus file: 70;; 71;; (require 'spam-stat) 72;; (spam-stat-load) 73 74;;; Testing: 75 76;; Typical test will involve calls to the following functions: 77;; 78;; Reset: (spam-stat-reset) 79;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam") 80;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc") 81;; Save table: (spam-stat-save) 82;; File size: (nth 7 (file-attributes spam-stat-file)) 83;; Number of words: (hash-table-count spam-stat) 84;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam") 85;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc") 86;; Reduce table size: (spam-stat-reduce-size) 87;; Save table: (spam-stat-save) 88;; File size: (nth 7 (file-attributes spam-stat-file)) 89;; Number of words: (hash-table-count spam-stat) 90;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam") 91;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc") 92 93;;; Dictionary Creation: 94 95;; Typically, you will filter away mailing lists etc. using specific 96;; rules in `nnmail-split-fancy'. Somewhere among these rules, you 97;; will filter spam. Here is how you would create your dictionary: 98 99;; Reset: (spam-stat-reset) 100;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam") 101;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc") 102;; Repeat for any other non-spam group you need... 103;; Reduce table size: (spam-stat-reduce-size) 104;; Save table: (spam-stat-save) 105 106;;; Todo: 107 108;; Speed it up. Integrate with Gnus such that it uses spam and expiry 109;; marks to call the appropriate functions when leaving the summary 110;; buffer and saves the hash table when leaving Gnus. More testing: 111;; More mails, disabling SpamAssassin, double checking algorithm, find 112;; improved algorithm. 113 114;;; Thanks: 115 116;; Ted Zlatanov <tzz@lifelogs.com> 117;; Jesper Harder <harder@myrealbox.com> 118;; Dan Schmidt <dfan@dfan.org> 119;; Lasse Rasinen <lrasinen@iki.fi> 120;; Milan Zamazal <pdm@zamazal.org> 121 122 123 124;;; Code: 125 126(defvar gnus-original-article-buffer) 127 128(defgroup spam-stat nil 129 "Statistical spam detection for Emacs. 130Use the functions to build a dictionary of words and their statistical 131distribution in spam and non-spam mails. Then use a function to determine 132whether a buffer contains spam or not." 133 :version "22.1" 134 :group 'gnus) 135 136(defcustom spam-stat-file "~/.spam-stat.el" 137 "File used to save and load the dictionary. 138See `spam-stat-to-hash-table' for the format of the file." 139 :type 'file 140 :group 'spam-stat) 141 142(defcustom spam-stat-install-hooks t 143 "Whether spam-stat should install its hooks in Gnus. 144This is set to nil if you use spam-stat through spam.el." 145 :type 'boolean 146 :group 'spam-stat) 147 148(defcustom spam-stat-unknown-word-score 0.2 149 "The score to use for unknown words. 150Also used for words that don't appear often enough." 151 :type 'number 152 :group 'spam-stat) 153 154(defcustom spam-stat-max-word-length 15 155 "Only words shorter than this will be considered." 156 :type 'integer 157 :group 'spam-stat) 158 159(defcustom spam-stat-max-buffer-length 10240 160 "Only the beginning of buffers will be analyzed. 161This variable says how many characters this will be." 162 :type 'integer 163 :group 'spam-stat) 164 165(defcustom spam-stat-split-fancy-spam-group "mail.spam" 166 "Name of the group where spam should be stored, if 167`spam-stat-split-fancy' is used in fancy splitting rules. Has no 168effect when spam-stat is invoked through spam.el." 169 :type 'string 170 :group 'spam-stat) 171 172(defcustom spam-stat-split-fancy-spam-threshhold 0.9 173 "Spam score threshhold in spam-stat-split-fancy." 174 :type 'number 175 :group 'spam-stat) 176 177(defvar spam-stat-syntax-table 178 (let ((table (copy-syntax-table text-mode-syntax-table))) 179 (modify-syntax-entry ?- "w" table) 180 (modify-syntax-entry ?_ "w" table) 181 (modify-syntax-entry ?. "w" table) 182 (modify-syntax-entry ?! "w" table) 183 (modify-syntax-entry ?? "w" table) 184 (modify-syntax-entry ?+ "w" table) 185 table) 186 "Syntax table used when processing mails for statistical analysis. 187The important part is which characters are word constituents.") 188 189(defvar spam-stat-dirty nil 190 "Whether the spam-stat database needs saving.") 191 192(defvar spam-stat-buffer nil 193 "Buffer to use for scoring while splitting. 194This is set by hooking into Gnus.") 195 196(defvar spam-stat-buffer-name " *spam stat buffer*" 197 "Name of the `spam-stat-buffer'.") 198 199;; Functions missing in Emacs 20 200 201(when (memq nil (mapcar 'fboundp 202 '(gethash hash-table-count make-hash-table 203 mapc puthash))) 204 (require 'cl) 205 (unless (fboundp 'puthash) 206 ;; alias puthash is missing from Emacs 20 cl-extra.el 207 (defalias 'puthash 'cl-puthash))) 208 209(eval-when-compile 210 (unless (fboundp 'with-syntax-table) 211 ;; Imported from Emacs 21.2 212 (defmacro with-syntax-table (table &rest body) "\ 213Evaluate BODY with syntax table of current buffer set to a copy of TABLE. 214The syntax table of the current buffer is saved, BODY is evaluated, and the 215saved table is restored, even in case of an abnormal exit. 216Value is what BODY returns." 217 (let ((old-table (make-symbol "table")) 218 (old-buffer (make-symbol "buffer"))) 219 `(let ((,old-table (syntax-table)) 220 (,old-buffer (current-buffer))) 221 (unwind-protect 222 (progn 223 (set-syntax-table (copy-syntax-table ,table)) 224 ,@body) 225 (save-current-buffer 226 (set-buffer ,old-buffer) 227 (set-syntax-table ,old-table)))))))) 228 229;; Hooking into Gnus 230 231(defun spam-stat-store-current-buffer () 232 "Store a copy of the current buffer in `spam-stat-buffer'." 233 (save-excursion 234 (let ((str (buffer-string))) 235 (set-buffer (get-buffer-create spam-stat-buffer-name)) 236 (erase-buffer) 237 (insert str) 238 (setq spam-stat-buffer (current-buffer))))) 239 240(defun spam-stat-store-gnus-article-buffer () 241 "Store a copy of the current article in `spam-stat-buffer'. 242This uses `gnus-article-buffer'." 243 (save-excursion 244 (set-buffer gnus-original-article-buffer) 245 (spam-stat-store-current-buffer))) 246 247;; Data -- not using defstruct in order to save space and time 248 249(defvar spam-stat (make-hash-table :test 'equal) 250 "Hash table used to store the statistics. 251Use `spam-stat-load' to load the file. 252Every word is used as a key in this table. The value is a vector. 253Use `spam-stat-ngood', `spam-stat-nbad', `spam-stat-good', 254`spam-stat-bad', and `spam-stat-score' to access this vector.") 255 256(defvar spam-stat-ngood 0 257 "The number of good mails in the dictionary.") 258 259(defvar spam-stat-nbad 0 260 "The number of bad mails in the dictionary.") 261 262(defsubst spam-stat-good (entry) 263 "Return the number of times this word belongs to good mails." 264 (aref entry 0)) 265 266(defsubst spam-stat-bad (entry) 267 "Return the number of times this word belongs to bad mails." 268 (aref entry 1)) 269 270(defsubst spam-stat-score (entry) 271 "Set the score of this word." 272 (if entry 273 (aref entry 2) 274 spam-stat-unknown-word-score)) 275 276(defsubst spam-stat-set-good (entry value) 277 "Set the number of times this word belongs to good mails." 278 (aset entry 0 value)) 279 280(defsubst spam-stat-set-bad (entry value) 281 "Set the number of times this word belongs to bad mails." 282 (aset entry 1 value)) 283 284(defsubst spam-stat-set-score (entry value) 285 "Set the score of this word." 286 (aset entry 2 value)) 287 288(defsubst spam-stat-make-entry (good bad) 289 "Return a vector with the given properties." 290 (let ((entry (vector good bad nil))) 291 (spam-stat-set-score entry (spam-stat-compute-score entry)) 292 entry)) 293 294;; Computing 295 296(defun spam-stat-compute-score (entry) 297 "Compute the score of this word. 1.0 means spam." 298 ;; promote all numbers to floats for the divisions 299 (let* ((g (* 2.0 (spam-stat-good entry))) 300 (b (float (spam-stat-bad entry)))) 301 (cond ((< (+ g b) 5) 302 .2) 303 ((= 0 spam-stat-ngood) 304 .99) 305 ((= 0 spam-stat-nbad) 306 .01) 307 (t 308 (max .01 309 (min .99 (/ (/ b spam-stat-nbad) 310 (+ (/ g spam-stat-ngood) 311 (/ b spam-stat-nbad))))))))) 312 313;; Parsing 314 315(defmacro with-spam-stat-max-buffer-size (&rest body) 316 "Narrows the buffer down to the first 4k characters, then evaluates BODY." 317 `(save-restriction 318 (when (> (- (point-max) 319 (point-min)) 320 spam-stat-max-buffer-length) 321 (narrow-to-region (point-min) 322 (+ (point-min) spam-stat-max-buffer-length))) 323 ,@body)) 324 325(defun spam-stat-buffer-words () 326 "Return a hash table of words and number of occurrences in the buffer." 327 (with-spam-stat-max-buffer-size 328 (with-syntax-table spam-stat-syntax-table 329 (goto-char (point-min)) 330 (let ((result (make-hash-table :test 'equal)) 331 word count) 332 (while (re-search-forward "\\w+" nil t) 333 (setq word (match-string-no-properties 0) 334 count (1+ (gethash word result 0))) 335 (when (< (length word) spam-stat-max-word-length) 336 (puthash word count result))) 337 result)))) 338 339(defun spam-stat-buffer-is-spam () 340 "Consider current buffer to be a new spam mail." 341 (setq spam-stat-nbad (1+ spam-stat-nbad)) 342 (maphash 343 (lambda (word count) 344 (let ((entry (gethash word spam-stat))) 345 (if entry 346 (spam-stat-set-bad entry (+ count (spam-stat-bad entry))) 347 (setq entry (spam-stat-make-entry 0 count))) 348 (spam-stat-set-score entry (spam-stat-compute-score entry)) 349 (puthash word entry spam-stat))) 350 (spam-stat-buffer-words)) 351 (setq spam-stat-dirty t)) 352 353(defun spam-stat-buffer-is-non-spam () 354 "Consider current buffer to be a new non-spam mail." 355 (setq spam-stat-ngood (1+ spam-stat-ngood)) 356 (maphash 357 (lambda (word count) 358 (let ((entry (gethash word spam-stat))) 359 (if entry 360 (spam-stat-set-good entry (+ count (spam-stat-good entry))) 361 (setq entry (spam-stat-make-entry count 0))) 362 (spam-stat-set-score entry (spam-stat-compute-score entry)) 363 (puthash word entry spam-stat))) 364 (spam-stat-buffer-words)) 365 (setq spam-stat-dirty t)) 366 367(defun spam-stat-buffer-change-to-spam () 368 "Consider current buffer no longer normal mail but spam." 369 (setq spam-stat-nbad (1+ spam-stat-nbad) 370 spam-stat-ngood (1- spam-stat-ngood)) 371 (maphash 372 (lambda (word count) 373 (let ((entry (gethash word spam-stat))) 374 (if (not entry) 375 (error "This buffer has unknown words in it") 376 (spam-stat-set-good entry (- (spam-stat-good entry) count)) 377 (spam-stat-set-bad entry (+ (spam-stat-bad entry) count)) 378 (spam-stat-set-score entry (spam-stat-compute-score entry)) 379 (puthash word entry spam-stat)))) 380 (spam-stat-buffer-words)) 381 (setq spam-stat-dirty t)) 382 383(defun spam-stat-buffer-change-to-non-spam () 384 "Consider current buffer no longer spam but normal mail." 385 (setq spam-stat-nbad (1- spam-stat-nbad) 386 spam-stat-ngood (1+ spam-stat-ngood)) 387 (maphash 388 (lambda (word count) 389 (let ((entry (gethash word spam-stat))) 390 (if (not entry) 391 (error "This buffer has unknown words in it") 392 (spam-stat-set-good entry (+ (spam-stat-good entry) count)) 393 (spam-stat-set-bad entry (- (spam-stat-bad entry) count)) 394 (spam-stat-set-score entry (spam-stat-compute-score entry)) 395 (puthash word entry spam-stat)))) 396 (spam-stat-buffer-words)) 397 (setq spam-stat-dirty t)) 398 399;; Saving and Loading 400 401(defun spam-stat-save (&optional force) 402 "Save the `spam-stat' hash table as lisp file. 403With a prefix argument save unconditionally." 404 (interactive "P") 405 (when (or force spam-stat-dirty) 406 (with-temp-buffer 407 (let ((standard-output (current-buffer)) 408 (font-lock-maximum-size 0)) 409 (insert "(setq spam-stat-ngood " 410 (number-to-string spam-stat-ngood) 411 " spam-stat-nbad " 412 (number-to-string spam-stat-nbad) 413 " spam-stat (spam-stat-to-hash-table '(") 414 (maphash (lambda (word entry) 415 (prin1 (list word 416 (spam-stat-good entry) 417 (spam-stat-bad entry)))) 418 spam-stat) 419 (insert ")))") 420 (write-file spam-stat-file))) 421 (setq spam-stat-dirty nil))) 422 423(defun spam-stat-load () 424 "Read the `spam-stat' hash table from disk." 425 ;; TODO: maybe we should warn the user if spam-stat-dirty is t? 426 (load-file spam-stat-file) 427 (setq spam-stat-dirty nil)) 428 429(defun spam-stat-to-hash-table (entries) 430 "Turn list ENTRIES into a hash table and store as `spam-stat'. 431Every element in ENTRIES has the form \(WORD GOOD BAD) where WORD is 432the word string, NGOOD is the number of good mails it has appeared in, 433NBAD is the number of bad mails it has appeared in, GOOD is the number 434of times it appeared in good mails, and BAD is the number of times it 435has appeared in bad mails." 436 (let ((table (make-hash-table :test 'equal))) 437 (mapc (lambda (l) 438 (puthash (car l) 439 (spam-stat-make-entry (nth 1 l) (nth 2 l)) 440 table)) 441 entries) 442 table)) 443 444(defun spam-stat-reset () 445 "Reset `spam-stat' to an empty hash-table. 446This deletes all the statistics." 447 (interactive) 448 (setq spam-stat (make-hash-table :test 'equal) 449 spam-stat-ngood 0 450 spam-stat-nbad 0) 451 (setq spam-stat-dirty t)) 452 453;; Scoring buffers 454 455(defvar spam-stat-score-data nil 456 "Raw data used in the last run of `spam-stat-score-buffer'.") 457 458(defsubst spam-stat-score-word (word) 459 "Return score for WORD. 460The default score for unknown words is stored in 461`spam-stat-unknown-word-score'." 462 (spam-stat-score (gethash word spam-stat))) 463 464(defun spam-stat-buffer-words-with-scores () 465 "Process current buffer, return the 15 most conspicuous words. 466These are the words whose spam-stat differs the most from 0.5. 467The list returned contains elements of the form \(WORD SCORE DIFF), 468where DIFF is the difference between SCORE and 0.5." 469 (with-spam-stat-max-buffer-size 470 (with-syntax-table spam-stat-syntax-table 471 (let (result word score) 472 (maphash (lambda (word ignore) 473 (setq score (spam-stat-score-word word) 474 result (cons (list word score (abs (- score 0.5))) 475 result))) 476 (spam-stat-buffer-words)) 477 (setq result (sort result (lambda (a b) (< (nth 2 b) (nth 2 a))))) 478 (setcdr (nthcdr 14 result) nil) 479 result)))) 480 481(defun spam-stat-score-buffer () 482 "Return a score describing the spam-probability for this buffer." 483 (setq spam-stat-score-data (spam-stat-buffer-words-with-scores)) 484 (let* ((probs (mapcar (lambda (e) (cadr e)) spam-stat-score-data)) 485 (prod (apply #'* probs))) 486 (/ prod (+ prod (apply #'* (mapcar #'(lambda (x) (- 1 x)) 487 probs)))))) 488 489(defun spam-stat-split-fancy () 490 "Return the name of the spam group if the current mail is spam. 491Use this function on `nnmail-split-fancy'. If you are interested in 492the raw data used for the last run of `spam-stat-score-buffer', 493check the variable `spam-stat-score-data'." 494 (condition-case var 495 (progn 496 (set-buffer spam-stat-buffer) 497 (goto-char (point-min)) 498 (when (> (spam-stat-score-buffer) spam-stat-split-fancy-spam-threshhold) 499 (when (boundp 'nnmail-split-trace) 500 (mapc (lambda (entry) 501 (push entry nnmail-split-trace)) 502 spam-stat-score-data)) 503 spam-stat-split-fancy-spam-group)) 504 (error (message "Error in spam-stat-split-fancy: %S" var) 505 nil))) 506 507;; Testing 508 509(defun spam-stat-process-directory (dir func) 510 "Process all the regular files in directory DIR using function FUNC." 511 (let* ((files (directory-files dir t "^[^.]")) 512 (max (/ (length files) 100.0)) 513 (count 0)) 514 (with-temp-buffer 515 (dolist (f files) 516 (when (and (file-readable-p f) 517 (file-regular-p f) 518 (> (nth 7 (file-attributes f)) 0)) 519 (setq count (1+ count)) 520 (message "Reading %s: %.2f%%" dir (/ count max)) 521 (insert-file-contents f) 522 (funcall func) 523 (erase-buffer)))))) 524 525(defun spam-stat-process-spam-directory (dir) 526 "Process all the regular files in directory DIR as spam." 527 (interactive "D") 528 (spam-stat-process-directory dir 'spam-stat-buffer-is-spam)) 529 530(defun spam-stat-process-non-spam-directory (dir) 531 "Process all the regular files in directory DIR as non-spam." 532 (interactive "D") 533 (spam-stat-process-directory dir 'spam-stat-buffer-is-non-spam)) 534 535(defun spam-stat-count () 536 "Return size of `spam-stat'." 537 (interactive) 538 (hash-table-count spam-stat)) 539 540(defun spam-stat-test-directory (dir) 541 "Test all the regular files in directory DIR for spam. 542If the result is 1.0, then all files are considered spam. 543If the result is 0.0, non of the files is considered spam. 544You can use this to determine error rates." 545 (interactive "D") 546 (let* ((files (directory-files dir t "^[^.]")) 547 (total (length files)) 548 (score 0.0); float 549 (max (/ total 100.0)); float 550 (count 0)) 551 (with-temp-buffer 552 (dolist (f files) 553 (when (and (file-readable-p f) 554 (file-regular-p f) 555 (> (nth 7 (file-attributes f)) 0)) 556 (setq count (1+ count)) 557 (message "Reading %.2f%%, score %.2f%%" 558 (/ count max) (/ score count)) 559 (insert-file-contents f) 560 (when (> (spam-stat-score-buffer) 0.9) 561 (setq score (1+ score))) 562 (erase-buffer)))) 563 (message "Final score: %d / %d = %f" score total (/ score total)))) 564 565;; Shrinking the dictionary 566 567(defun spam-stat-reduce-size (&optional count) 568 "Reduce the size of `spam-stat'. 569This removes all words that occur less than COUNT from the dictionary. 570COUNT defaults to 5" 571 (interactive) 572 (setq count (or count 5)) 573 (maphash (lambda (key entry) 574 (when (< (+ (spam-stat-good entry) 575 (spam-stat-bad entry)) 576 count) 577 (remhash key spam-stat))) 578 spam-stat) 579 (setq spam-stat-dirty t)) 580 581(defun spam-stat-install-hooks-function () 582 "Install the spam-stat function hooks" 583 (interactive) 584 (add-hook 'nnmail-prepare-incoming-message-hook 585 'spam-stat-store-current-buffer) 586 (add-hook 'gnus-select-article-hook 587 'spam-stat-store-gnus-article-buffer)) 588 589(when spam-stat-install-hooks 590 (spam-stat-install-hooks-function)) 591 592(defun spam-stat-unload-hook () 593 "Uninstall the spam-stat function hooks" 594 (interactive) 595 (remove-hook 'nnmail-prepare-incoming-message-hook 596 'spam-stat-store-current-buffer) 597 (remove-hook 'gnus-select-article-hook 598 'spam-stat-store-gnus-article-buffer)) 599 600(add-hook 'spam-stat-unload-hook 'spam-stat-unload-hook) 601 602(provide 'spam-stat) 603 604;;; arch-tag: ff1d2200-8ddb-42fb-bb7b-1b5e20448554 605;;; spam-stat.el ends here 606