1/* vi:set ts=8 sts=4 sw=4: 2 * 3 * VIM - Vi IMproved by Bram Moolenaar 4 * 5 * Do ":help uganda" in Vim to read copying and usage conditions. 6 * Do ":help credits" in Vim to see a list of people who contributed. 7 * See README.txt for an overview of the Vim source code. 8 */ 9 10/* 11 * spell.c: code for spell checking 12 * 13 * The spell checking mechanism uses a tree (aka trie). Each node in the tree 14 * has a list of bytes that can appear (siblings). For each byte there is a 15 * pointer to the node with the byte that follows in the word (child). 16 * 17 * A NUL byte is used where the word may end. The bytes are sorted, so that 18 * binary searching can be used and the NUL bytes are at the start. The 19 * number of possible bytes is stored before the list of bytes. 20 * 21 * The tree uses two arrays: "byts" stores the characters, "idxs" stores 22 * either the next index or flags. The tree starts at index 0. For example, 23 * to lookup "vi" this sequence is followed: 24 * i = 0 25 * len = byts[i] 26 * n = where "v" appears in byts[i + 1] to byts[i + len] 27 * i = idxs[n] 28 * len = byts[i] 29 * n = where "i" appears in byts[i + 1] to byts[i + len] 30 * i = idxs[n] 31 * len = byts[i] 32 * find that byts[i + 1] is 0, idxs[i + 1] has flags for "vi". 33 * 34 * There are two word trees: one with case-folded words and one with words in 35 * original case. The second one is only used for keep-case words and is 36 * usually small. 37 * 38 * There is one additional tree for when not all prefixes are applied when 39 * generating the .spl file. This tree stores all the possible prefixes, as 40 * if they were words. At each word (prefix) end the prefix nr is stored, the 41 * following word must support this prefix nr. And the condition nr is 42 * stored, used to lookup the condition that the word must match with. 43 * 44 * Thanks to Olaf Seibert for providing an example implementation of this tree 45 * and the compression mechanism. 46 * LZ trie ideas: 47 * http://www.irb.hr/hr/home/ristov/papers/RistovLZtrieRevision1.pdf 48 * More papers: http://www-igm.univ-mlv.fr/~laporte/publi_en.html 49 * 50 * Matching involves checking the caps type: Onecap ALLCAP KeepCap. 51 * 52 * Why doesn't Vim use aspell/ispell/myspell/etc.? 53 * See ":help develop-spell". 54 */ 55 56/* Use SPELL_PRINTTREE for debugging: dump the word tree after adding a word. 57 * Only use it for small word lists! */ 58#if 0 59# define SPELL_PRINTTREE 60#endif 61 62/* Use DEBUG_TRIEWALK to print the changes made in suggest_trie_walk() for a 63 * specific word. */ 64#if 0 65# define DEBUG_TRIEWALK 66#endif 67 68/* 69 * Use this to adjust the score after finding suggestions, based on the 70 * suggested word sounding like the bad word. This is much faster than doing 71 * it for every possible suggestion. 72 * Disadvantage: When "the" is typed as "hte" it sounds quite different ("@" 73 * vs "ht") and goes down in the list. 74 * Used when 'spellsuggest' is set to "best". 75 */ 76#define RESCORE(word_score, sound_score) ((3 * word_score + sound_score) / 4) 77 78/* 79 * Do the opposite: based on a maximum end score and a known sound score, 80 * compute the maximum word score that can be used. 81 */ 82#define MAXSCORE(word_score, sound_score) ((4 * word_score - sound_score) / 3) 83 84/* 85 * Vim spell file format: <HEADER> 86 * <SECTIONS> 87 * <LWORDTREE> 88 * <KWORDTREE> 89 * <PREFIXTREE> 90 * 91 * <HEADER>: <fileID> <versionnr> 92 * 93 * <fileID> 8 bytes "VIMspell" 94 * <versionnr> 1 byte VIMSPELLVERSION 95 * 96 * 97 * Sections make it possible to add information to the .spl file without 98 * making it incompatible with previous versions. There are two kinds of 99 * sections: 100 * 1. Not essential for correct spell checking. E.g. for making suggestions. 101 * These are skipped when not supported. 102 * 2. Optional information, but essential for spell checking when present. 103 * E.g. conditions for affixes. When this section is present but not 104 * supported an error message is given. 105 * 106 * <SECTIONS>: <section> ... <sectionend> 107 * 108 * <section>: <sectionID> <sectionflags> <sectionlen> (section contents) 109 * 110 * <sectionID> 1 byte number from 0 to 254 identifying the section 111 * 112 * <sectionflags> 1 byte SNF_REQUIRED: this section is required for correct 113 * spell checking 114 * 115 * <sectionlen> 4 bytes length of section contents, MSB first 116 * 117 * <sectionend> 1 byte SN_END 118 * 119 * 120 * sectionID == SN_INFO: <infotext> 121 * <infotext> N bytes free format text with spell file info (version, 122 * website, etc) 123 * 124 * sectionID == SN_REGION: <regionname> ... 125 * <regionname> 2 bytes Up to 8 region names: ca, au, etc. Lower case. 126 * First <regionname> is region 1. 127 * 128 * sectionID == SN_CHARFLAGS: <charflagslen> <charflags> 129 * <folcharslen> <folchars> 130 * <charflagslen> 1 byte Number of bytes in <charflags> (should be 128). 131 * <charflags> N bytes List of flags (first one is for character 128): 132 * 0x01 word character CF_WORD 133 * 0x02 upper-case character CF_UPPER 134 * <folcharslen> 2 bytes Number of bytes in <folchars>. 135 * <folchars> N bytes Folded characters, first one is for character 128. 136 * 137 * sectionID == SN_MIDWORD: <midword> 138 * <midword> N bytes Characters that are word characters only when used 139 * in the middle of a word. 140 * 141 * sectionID == SN_PREFCOND: <prefcondcnt> <prefcond> ... 142 * <prefcondcnt> 2 bytes Number of <prefcond> items following. 143 * <prefcond> : <condlen> <condstr> 144 * <condlen> 1 byte Length of <condstr>. 145 * <condstr> N bytes Condition for the prefix. 146 * 147 * sectionID == SN_REP: <repcount> <rep> ... 148 * <repcount> 2 bytes number of <rep> items, MSB first. 149 * <rep> : <repfromlen> <repfrom> <reptolen> <repto> 150 * <repfromlen> 1 byte length of <repfrom> 151 * <repfrom> N bytes "from" part of replacement 152 * <reptolen> 1 byte length of <repto> 153 * <repto> N bytes "to" part of replacement 154 * 155 * sectionID == SN_REPSAL: <repcount> <rep> ... 156 * just like SN_REP but for soundfolded words 157 * 158 * sectionID == SN_SAL: <salflags> <salcount> <sal> ... 159 * <salflags> 1 byte flags for soundsalike conversion: 160 * SAL_F0LLOWUP 161 * SAL_COLLAPSE 162 * SAL_REM_ACCENTS 163 * <salcount> 2 bytes number of <sal> items following 164 * <sal> : <salfromlen> <salfrom> <saltolen> <salto> 165 * <salfromlen> 1 byte length of <salfrom> 166 * <salfrom> N bytes "from" part of soundsalike 167 * <saltolen> 1 byte length of <salto> 168 * <salto> N bytes "to" part of soundsalike 169 * 170 * sectionID == SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto> 171 * <sofofromlen> 2 bytes length of <sofofrom> 172 * <sofofrom> N bytes "from" part of soundfold 173 * <sofotolen> 2 bytes length of <sofoto> 174 * <sofoto> N bytes "to" part of soundfold 175 * 176 * sectionID == SN_SUGFILE: <timestamp> 177 * <timestamp> 8 bytes time in seconds that must match with .sug file 178 * 179 * sectionID == SN_NOSPLITSUGS: nothing 180 * 181 * sectionID == SN_WORDS: <word> ... 182 * <word> N bytes NUL terminated common word 183 * 184 * sectionID == SN_MAP: <mapstr> 185 * <mapstr> N bytes String with sequences of similar characters, 186 * separated by slashes. 187 * 188 * sectionID == SN_COMPOUND: <compmax> <compminlen> <compsylmax> <compoptions> 189 * <comppatcount> <comppattern> ... <compflags> 190 * <compmax> 1 byte Maximum nr of words in compound word. 191 * <compminlen> 1 byte Minimal word length for compounding. 192 * <compsylmax> 1 byte Maximum nr of syllables in compound word. 193 * <compoptions> 2 bytes COMP_ flags. 194 * <comppatcount> 2 bytes number of <comppattern> following 195 * <compflags> N bytes Flags from COMPOUNDRULE items, separated by 196 * slashes. 197 * 198 * <comppattern>: <comppatlen> <comppattext> 199 * <comppatlen> 1 byte length of <comppattext> 200 * <comppattext> N bytes end or begin chars from CHECKCOMPOUNDPATTERN 201 * 202 * sectionID == SN_NOBREAK: (empty, its presence is what matters) 203 * 204 * sectionID == SN_SYLLABLE: <syllable> 205 * <syllable> N bytes String from SYLLABLE item. 206 * 207 * <LWORDTREE>: <wordtree> 208 * 209 * <KWORDTREE>: <wordtree> 210 * 211 * <PREFIXTREE>: <wordtree> 212 * 213 * 214 * <wordtree>: <nodecount> <nodedata> ... 215 * 216 * <nodecount> 4 bytes Number of nodes following. MSB first. 217 * 218 * <nodedata>: <siblingcount> <sibling> ... 219 * 220 * <siblingcount> 1 byte Number of siblings in this node. The siblings 221 * follow in sorted order. 222 * 223 * <sibling>: <byte> [ <nodeidx> <xbyte> 224 * | <flags> [<flags2>] [<region>] [<affixID>] 225 * | [<pflags>] <affixID> <prefcondnr> ] 226 * 227 * <byte> 1 byte Byte value of the sibling. Special cases: 228 * BY_NOFLAGS: End of word without flags and for all 229 * regions. 230 * For PREFIXTREE <affixID> and 231 * <prefcondnr> follow. 232 * BY_FLAGS: End of word, <flags> follow. 233 * For PREFIXTREE <pflags>, <affixID> 234 * and <prefcondnr> follow. 235 * BY_FLAGS2: End of word, <flags> and <flags2> 236 * follow. Not used in PREFIXTREE. 237 * BY_INDEX: Child of sibling is shared, <nodeidx> 238 * and <xbyte> follow. 239 * 240 * <nodeidx> 3 bytes Index of child for this sibling, MSB first. 241 * 242 * <xbyte> 1 byte byte value of the sibling. 243 * 244 * <flags> 1 byte bitmask of: 245 * WF_ALLCAP word must have only capitals 246 * WF_ONECAP first char of word must be capital 247 * WF_KEEPCAP keep-case word 248 * WF_FIXCAP keep-case word, all caps not allowed 249 * WF_RARE rare word 250 * WF_BANNED bad word 251 * WF_REGION <region> follows 252 * WF_AFX <affixID> follows 253 * 254 * <flags2> 1 byte Bitmask of: 255 * WF_HAS_AFF >> 8 word includes affix 256 * WF_NEEDCOMP >> 8 word only valid in compound 257 * WF_NOSUGGEST >> 8 word not used for suggestions 258 * WF_COMPROOT >> 8 word already a compound 259 * WF_NOCOMPBEF >> 8 no compounding before this word 260 * WF_NOCOMPAFT >> 8 no compounding after this word 261 * 262 * <pflags> 1 byte bitmask of: 263 * WFP_RARE rare prefix 264 * WFP_NC non-combining prefix 265 * WFP_UP letter after prefix made upper case 266 * 267 * <region> 1 byte Bitmask for regions in which word is valid. When 268 * omitted it's valid in all regions. 269 * Lowest bit is for region 1. 270 * 271 * <affixID> 1 byte ID of affix that can be used with this word. In 272 * PREFIXTREE used for the required prefix ID. 273 * 274 * <prefcondnr> 2 bytes Prefix condition number, index in <prefcond> list 275 * from HEADER. 276 * 277 * All text characters are in 'encoding', but stored as single bytes. 278 */ 279 280/* 281 * Vim .sug file format: <SUGHEADER> 282 * <SUGWORDTREE> 283 * <SUGTABLE> 284 * 285 * <SUGHEADER>: <fileID> <versionnr> <timestamp> 286 * 287 * <fileID> 6 bytes "VIMsug" 288 * <versionnr> 1 byte VIMSUGVERSION 289 * <timestamp> 8 bytes timestamp that must match with .spl file 290 * 291 * 292 * <SUGWORDTREE>: <wordtree> (see above, no flags or region used) 293 * 294 * 295 * <SUGTABLE>: <sugwcount> <sugline> ... 296 * 297 * <sugwcount> 4 bytes number of <sugline> following 298 * 299 * <sugline>: <sugnr> ... NUL 300 * 301 * <sugnr>: X bytes word number that results in this soundfolded word, 302 * stored as an offset to the previous number in as 303 * few bytes as possible, see offset2bytes()) 304 */ 305 306#if defined(MSDOS) || defined(WIN16) || defined(WIN32) || defined(_WIN64) 307# include "vimio.h" /* for lseek(), must be before vim.h */ 308#endif 309 310#include "vim.h" 311 312#if defined(FEAT_SPELL) || defined(PROTO) 313 314#ifndef UNIX /* it's in os_unix.h for Unix */ 315# include <time.h> /* for time_t */ 316#endif 317 318#define MAXWLEN 250 /* Assume max. word len is this many bytes. 319 Some places assume a word length fits in a 320 byte, thus it can't be above 255. */ 321 322/* Type used for indexes in the word tree need to be at least 4 bytes. If int 323 * is 8 bytes we could use something smaller, but what? */ 324#if SIZEOF_INT > 3 325typedef int idx_T; 326#else 327typedef long idx_T; 328#endif 329 330/* Flags used for a word. Only the lowest byte can be used, the region byte 331 * comes above it. */ 332#define WF_REGION 0x01 /* region byte follows */ 333#define WF_ONECAP 0x02 /* word with one capital (or all capitals) */ 334#define WF_ALLCAP 0x04 /* word must be all capitals */ 335#define WF_RARE 0x08 /* rare word */ 336#define WF_BANNED 0x10 /* bad word */ 337#define WF_AFX 0x20 /* affix ID follows */ 338#define WF_FIXCAP 0x40 /* keep-case word, allcap not allowed */ 339#define WF_KEEPCAP 0x80 /* keep-case word */ 340 341/* for <flags2>, shifted up one byte to be used in wn_flags */ 342#define WF_HAS_AFF 0x0100 /* word includes affix */ 343#define WF_NEEDCOMP 0x0200 /* word only valid in compound */ 344#define WF_NOSUGGEST 0x0400 /* word not to be suggested */ 345#define WF_COMPROOT 0x0800 /* already compounded word, COMPOUNDROOT */ 346#define WF_NOCOMPBEF 0x1000 /* no compounding before this word */ 347#define WF_NOCOMPAFT 0x2000 /* no compounding after this word */ 348 349/* only used for su_badflags */ 350#define WF_MIXCAP 0x20 /* mix of upper and lower case: macaRONI */ 351 352#define WF_CAPMASK (WF_ONECAP | WF_ALLCAP | WF_KEEPCAP | WF_FIXCAP) 353 354/* flags for <pflags> */ 355#define WFP_RARE 0x01 /* rare prefix */ 356#define WFP_NC 0x02 /* prefix is not combining */ 357#define WFP_UP 0x04 /* to-upper prefix */ 358#define WFP_COMPPERMIT 0x08 /* prefix with COMPOUNDPERMITFLAG */ 359#define WFP_COMPFORBID 0x10 /* prefix with COMPOUNDFORBIDFLAG */ 360 361/* Flags for postponed prefixes in "sl_pidxs". Must be above affixID (one 362 * byte) and prefcondnr (two bytes). */ 363#define WF_RAREPFX (WFP_RARE << 24) /* rare postponed prefix */ 364#define WF_PFX_NC (WFP_NC << 24) /* non-combining postponed prefix */ 365#define WF_PFX_UP (WFP_UP << 24) /* to-upper postponed prefix */ 366#define WF_PFX_COMPPERMIT (WFP_COMPPERMIT << 24) /* postponed prefix with 367 * COMPOUNDPERMITFLAG */ 368#define WF_PFX_COMPFORBID (WFP_COMPFORBID << 24) /* postponed prefix with 369 * COMPOUNDFORBIDFLAG */ 370 371 372/* flags for <compoptions> */ 373#define COMP_CHECKDUP 1 /* CHECKCOMPOUNDDUP */ 374#define COMP_CHECKREP 2 /* CHECKCOMPOUNDREP */ 375#define COMP_CHECKCASE 4 /* CHECKCOMPOUNDCASE */ 376#define COMP_CHECKTRIPLE 8 /* CHECKCOMPOUNDTRIPLE */ 377 378/* Special byte values for <byte>. Some are only used in the tree for 379 * postponed prefixes, some only in the other trees. This is a bit messy... */ 380#define BY_NOFLAGS 0 /* end of word without flags or region; for 381 * postponed prefix: no <pflags> */ 382#define BY_INDEX 1 /* child is shared, index follows */ 383#define BY_FLAGS 2 /* end of word, <flags> byte follows; for 384 * postponed prefix: <pflags> follows */ 385#define BY_FLAGS2 3 /* end of word, <flags> and <flags2> bytes 386 * follow; never used in prefix tree */ 387#define BY_SPECIAL BY_FLAGS2 /* highest special byte value */ 388 389/* Info from "REP", "REPSAL" and "SAL" entries in ".aff" file used in si_rep, 390 * si_repsal, sl_rep, and si_sal. Not for sl_sal! 391 * One replacement: from "ft_from" to "ft_to". */ 392typedef struct fromto_S 393{ 394 char_u *ft_from; 395 char_u *ft_to; 396} fromto_T; 397 398/* Info from "SAL" entries in ".aff" file used in sl_sal. 399 * The info is split for quick processing by spell_soundfold(). 400 * Note that "sm_oneof" and "sm_rules" point into sm_lead. */ 401typedef struct salitem_S 402{ 403 char_u *sm_lead; /* leading letters */ 404 int sm_leadlen; /* length of "sm_lead" */ 405 char_u *sm_oneof; /* letters from () or NULL */ 406 char_u *sm_rules; /* rules like ^, $, priority */ 407 char_u *sm_to; /* replacement. */ 408#ifdef FEAT_MBYTE 409 int *sm_lead_w; /* wide character copy of "sm_lead" */ 410 int *sm_oneof_w; /* wide character copy of "sm_oneof" */ 411 int *sm_to_w; /* wide character copy of "sm_to" */ 412#endif 413} salitem_T; 414 415#ifdef FEAT_MBYTE 416typedef int salfirst_T; 417#else 418typedef short salfirst_T; 419#endif 420 421/* Values for SP_*ERROR are negative, positive values are used by 422 * read_cnt_string(). */ 423#define SP_TRUNCERROR -1 /* spell file truncated error */ 424#define SP_FORMERROR -2 /* format error in spell file */ 425#define SP_OTHERERROR -3 /* other error while reading spell file */ 426 427/* 428 * Structure used to store words and other info for one language, loaded from 429 * a .spl file. 430 * The main access is through the tree in "sl_fbyts/sl_fidxs", storing the 431 * case-folded words. "sl_kbyts/sl_kidxs" is for keep-case words. 432 * 433 * The "byts" array stores the possible bytes in each tree node, preceded by 434 * the number of possible bytes, sorted on byte value: 435 * <len> <byte1> <byte2> ... 436 * The "idxs" array stores the index of the child node corresponding to the 437 * byte in "byts". 438 * Exception: when the byte is zero, the word may end here and "idxs" holds 439 * the flags, region mask and affixID for the word. There may be several 440 * zeros in sequence for alternative flag/region/affixID combinations. 441 */ 442typedef struct slang_S slang_T; 443struct slang_S 444{ 445 slang_T *sl_next; /* next language */ 446 char_u *sl_name; /* language name "en", "en.rare", "nl", etc. */ 447 char_u *sl_fname; /* name of .spl file */ 448 int sl_add; /* TRUE if it's a .add file. */ 449 450 char_u *sl_fbyts; /* case-folded word bytes */ 451 idx_T *sl_fidxs; /* case-folded word indexes */ 452 char_u *sl_kbyts; /* keep-case word bytes */ 453 idx_T *sl_kidxs; /* keep-case word indexes */ 454 char_u *sl_pbyts; /* prefix tree word bytes */ 455 idx_T *sl_pidxs; /* prefix tree word indexes */ 456 457 char_u *sl_info; /* infotext string or NULL */ 458 459 char_u sl_regions[17]; /* table with up to 8 region names plus NUL */ 460 461 char_u *sl_midword; /* MIDWORD string or NULL */ 462 463 hashtab_T sl_wordcount; /* hashtable with word count, wordcount_T */ 464 465 int sl_compmax; /* COMPOUNDWORDMAX (default: MAXWLEN) */ 466 int sl_compminlen; /* COMPOUNDMIN (default: 0) */ 467 int sl_compsylmax; /* COMPOUNDSYLMAX (default: MAXWLEN) */ 468 int sl_compoptions; /* COMP_* flags */ 469 garray_T sl_comppat; /* CHECKCOMPOUNDPATTERN items */ 470 regprog_T *sl_compprog; /* COMPOUNDRULE turned into a regexp progrm 471 * (NULL when no compounding) */ 472 char_u *sl_comprules; /* all COMPOUNDRULE concatenated (or NULL) */ 473 char_u *sl_compstartflags; /* flags for first compound word */ 474 char_u *sl_compallflags; /* all flags for compound words */ 475 char_u sl_nobreak; /* When TRUE: no spaces between words */ 476 char_u *sl_syllable; /* SYLLABLE repeatable chars or NULL */ 477 garray_T sl_syl_items; /* syllable items */ 478 479 int sl_prefixcnt; /* number of items in "sl_prefprog" */ 480 regprog_T **sl_prefprog; /* table with regprogs for prefixes */ 481 482 garray_T sl_rep; /* list of fromto_T entries from REP lines */ 483 short sl_rep_first[256]; /* indexes where byte first appears, -1 if 484 there is none */ 485 garray_T sl_sal; /* list of salitem_T entries from SAL lines */ 486 salfirst_T sl_sal_first[256]; /* indexes where byte first appears, -1 if 487 there is none */ 488 int sl_followup; /* SAL followup */ 489 int sl_collapse; /* SAL collapse_result */ 490 int sl_rem_accents; /* SAL remove_accents */ 491 int sl_sofo; /* SOFOFROM and SOFOTO instead of SAL items: 492 * "sl_sal_first" maps chars, when has_mbyte 493 * "sl_sal" is a list of wide char lists. */ 494 garray_T sl_repsal; /* list of fromto_T entries from REPSAL lines */ 495 short sl_repsal_first[256]; /* sl_rep_first for REPSAL lines */ 496 int sl_nosplitsugs; /* don't suggest splitting a word */ 497 498 /* Info from the .sug file. Loaded on demand. */ 499 time_t sl_sugtime; /* timestamp for .sug file */ 500 char_u *sl_sbyts; /* soundfolded word bytes */ 501 idx_T *sl_sidxs; /* soundfolded word indexes */ 502 buf_T *sl_sugbuf; /* buffer with word number table */ 503 int sl_sugloaded; /* TRUE when .sug file was loaded or failed to 504 load */ 505 506 int sl_has_map; /* TRUE if there is a MAP line */ 507#ifdef FEAT_MBYTE 508 hashtab_T sl_map_hash; /* MAP for multi-byte chars */ 509 int sl_map_array[256]; /* MAP for first 256 chars */ 510#else 511 char_u sl_map_array[256]; /* MAP for first 256 chars */ 512#endif 513 hashtab_T sl_sounddone; /* table with soundfolded words that have 514 handled, see add_sound_suggest() */ 515}; 516 517/* First language that is loaded, start of the linked list of loaded 518 * languages. */ 519static slang_T *first_lang = NULL; 520 521/* Flags used in .spl file for soundsalike flags. */ 522#define SAL_F0LLOWUP 1 523#define SAL_COLLAPSE 2 524#define SAL_REM_ACCENTS 4 525 526/* 527 * Structure used in "b_langp", filled from 'spelllang'. 528 */ 529typedef struct langp_S 530{ 531 slang_T *lp_slang; /* info for this language */ 532 slang_T *lp_sallang; /* language used for sound folding or NULL */ 533 slang_T *lp_replang; /* language used for REP items or NULL */ 534 int lp_region; /* bitmask for region or REGION_ALL */ 535} langp_T; 536 537#define LANGP_ENTRY(ga, i) (((langp_T *)(ga).ga_data) + (i)) 538 539#define REGION_ALL 0xff /* word valid in all regions */ 540 541#define VIMSPELLMAGIC "VIMspell" /* string at start of Vim spell file */ 542#define VIMSPELLMAGICL 8 543#define VIMSPELLVERSION 50 544 545#define VIMSUGMAGIC "VIMsug" /* string at start of Vim .sug file */ 546#define VIMSUGMAGICL 6 547#define VIMSUGVERSION 1 548 549/* Section IDs. Only renumber them when VIMSPELLVERSION changes! */ 550#define SN_REGION 0 /* <regionname> section */ 551#define SN_CHARFLAGS 1 /* charflags section */ 552#define SN_MIDWORD 2 /* <midword> section */ 553#define SN_PREFCOND 3 /* <prefcond> section */ 554#define SN_REP 4 /* REP items section */ 555#define SN_SAL 5 /* SAL items section */ 556#define SN_SOFO 6 /* soundfolding section */ 557#define SN_MAP 7 /* MAP items section */ 558#define SN_COMPOUND 8 /* compound words section */ 559#define SN_SYLLABLE 9 /* syllable section */ 560#define SN_NOBREAK 10 /* NOBREAK section */ 561#define SN_SUGFILE 11 /* timestamp for .sug file */ 562#define SN_REPSAL 12 /* REPSAL items section */ 563#define SN_WORDS 13 /* common words */ 564#define SN_NOSPLITSUGS 14 /* don't split word for suggestions */ 565#define SN_INFO 15 /* info section */ 566#define SN_END 255 /* end of sections */ 567 568#define SNF_REQUIRED 1 /* <sectionflags>: required section */ 569 570/* Result values. Lower number is accepted over higher one. */ 571#define SP_BANNED -1 572#define SP_OK 0 573#define SP_RARE 1 574#define SP_LOCAL 2 575#define SP_BAD 3 576 577/* file used for "zG" and "zW" */ 578static char_u *int_wordlist = NULL; 579 580typedef struct wordcount_S 581{ 582 short_u wc_count; /* nr of times word was seen */ 583 char_u wc_word[1]; /* word, actually longer */ 584} wordcount_T; 585 586static wordcount_T dumwc; 587#define WC_KEY_OFF (unsigned)(dumwc.wc_word - (char_u *)&dumwc) 588#define HI2WC(hi) ((wordcount_T *)((hi)->hi_key - WC_KEY_OFF)) 589#define MAXWORDCOUNT 0xffff 590 591/* 592 * Information used when looking for suggestions. 593 */ 594typedef struct suginfo_S 595{ 596 garray_T su_ga; /* suggestions, contains "suggest_T" */ 597 int su_maxcount; /* max. number of suggestions displayed */ 598 int su_maxscore; /* maximum score for adding to su_ga */ 599 int su_sfmaxscore; /* idem, for when doing soundfold words */ 600 garray_T su_sga; /* like su_ga, sound-folded scoring */ 601 char_u *su_badptr; /* start of bad word in line */ 602 int su_badlen; /* length of detected bad word in line */ 603 int su_badflags; /* caps flags for bad word */ 604 char_u su_badword[MAXWLEN]; /* bad word truncated at su_badlen */ 605 char_u su_fbadword[MAXWLEN]; /* su_badword case-folded */ 606 char_u su_sal_badword[MAXWLEN]; /* su_badword soundfolded */ 607 hashtab_T su_banned; /* table with banned words */ 608 slang_T *su_sallang; /* default language for sound folding */ 609} suginfo_T; 610 611/* One word suggestion. Used in "si_ga". */ 612typedef struct suggest_S 613{ 614 char_u *st_word; /* suggested word, allocated string */ 615 int st_wordlen; /* STRLEN(st_word) */ 616 int st_orglen; /* length of replaced text */ 617 int st_score; /* lower is better */ 618 int st_altscore; /* used when st_score compares equal */ 619 int st_salscore; /* st_score is for soundalike */ 620 int st_had_bonus; /* bonus already included in score */ 621 slang_T *st_slang; /* language used for sound folding */ 622} suggest_T; 623 624#define SUG(ga, i) (((suggest_T *)(ga).ga_data)[i]) 625 626/* TRUE if a word appears in the list of banned words. */ 627#define WAS_BANNED(su, word) (!HASHITEM_EMPTY(hash_find(&su->su_banned, word))) 628 629/* Number of suggestions kept when cleaning up. We need to keep more than 630 * what is displayed, because when rescore_suggestions() is called the score 631 * may change and wrong suggestions may be removed later. */ 632#define SUG_CLEAN_COUNT(su) ((su)->su_maxcount < 130 ? 150 : (su)->su_maxcount + 20) 633 634/* Threshold for sorting and cleaning up suggestions. Don't want to keep lots 635 * of suggestions that are not going to be displayed. */ 636#define SUG_MAX_COUNT(su) (SUG_CLEAN_COUNT(su) + 50) 637 638/* score for various changes */ 639#define SCORE_SPLIT 149 /* split bad word */ 640#define SCORE_SPLIT_NO 249 /* split bad word with NOSPLITSUGS */ 641#define SCORE_ICASE 52 /* slightly different case */ 642#define SCORE_REGION 200 /* word is for different region */ 643#define SCORE_RARE 180 /* rare word */ 644#define SCORE_SWAP 75 /* swap two characters */ 645#define SCORE_SWAP3 110 /* swap two characters in three */ 646#define SCORE_REP 65 /* REP replacement */ 647#define SCORE_SUBST 93 /* substitute a character */ 648#define SCORE_SIMILAR 33 /* substitute a similar character */ 649#define SCORE_SUBCOMP 33 /* substitute a composing character */ 650#define SCORE_DEL 94 /* delete a character */ 651#define SCORE_DELDUP 66 /* delete a duplicated character */ 652#define SCORE_DELCOMP 28 /* delete a composing character */ 653#define SCORE_INS 96 /* insert a character */ 654#define SCORE_INSDUP 67 /* insert a duplicate character */ 655#define SCORE_INSCOMP 30 /* insert a composing character */ 656#define SCORE_NONWORD 103 /* change non-word to word char */ 657 658#define SCORE_FILE 30 /* suggestion from a file */ 659#define SCORE_MAXINIT 350 /* Initial maximum score: higher == slower. 660 * 350 allows for about three changes. */ 661 662#define SCORE_COMMON1 30 /* subtracted for words seen before */ 663#define SCORE_COMMON2 40 /* subtracted for words often seen */ 664#define SCORE_COMMON3 50 /* subtracted for words very often seen */ 665#define SCORE_THRES2 10 /* word count threshold for COMMON2 */ 666#define SCORE_THRES3 100 /* word count threshold for COMMON3 */ 667 668/* When trying changed soundfold words it becomes slow when trying more than 669 * two changes. With less then two changes it's slightly faster but we miss a 670 * few good suggestions. In rare cases we need to try three of four changes. 671 */ 672#define SCORE_SFMAX1 200 /* maximum score for first try */ 673#define SCORE_SFMAX2 300 /* maximum score for second try */ 674#define SCORE_SFMAX3 400 /* maximum score for third try */ 675 676#define SCORE_BIG SCORE_INS * 3 /* big difference */ 677#define SCORE_MAXMAX 999999 /* accept any score */ 678#define SCORE_LIMITMAX 350 /* for spell_edit_score_limit() */ 679 680/* for spell_edit_score_limit() we need to know the minimum value of 681 * SCORE_ICASE, SCORE_SWAP, SCORE_DEL, SCORE_SIMILAR and SCORE_INS */ 682#define SCORE_EDIT_MIN SCORE_SIMILAR 683 684/* 685 * Structure to store info for word matching. 686 */ 687typedef struct matchinf_S 688{ 689 langp_T *mi_lp; /* info for language and region */ 690 691 /* pointers to original text to be checked */ 692 char_u *mi_word; /* start of word being checked */ 693 char_u *mi_end; /* end of matching word so far */ 694 char_u *mi_fend; /* next char to be added to mi_fword */ 695 char_u *mi_cend; /* char after what was used for 696 mi_capflags */ 697 698 /* case-folded text */ 699 char_u mi_fword[MAXWLEN + 1]; /* mi_word case-folded */ 700 int mi_fwordlen; /* nr of valid bytes in mi_fword */ 701 702 /* for when checking word after a prefix */ 703 int mi_prefarridx; /* index in sl_pidxs with list of 704 affixID/condition */ 705 int mi_prefcnt; /* number of entries at mi_prefarridx */ 706 int mi_prefixlen; /* byte length of prefix */ 707#ifdef FEAT_MBYTE 708 int mi_cprefixlen; /* byte length of prefix in original 709 case */ 710#else 711# define mi_cprefixlen mi_prefixlen /* it's the same value */ 712#endif 713 714 /* for when checking a compound word */ 715 int mi_compoff; /* start of following word offset */ 716 char_u mi_compflags[MAXWLEN]; /* flags for compound words used */ 717 int mi_complen; /* nr of compound words used */ 718 int mi_compextra; /* nr of COMPOUNDROOT words */ 719 720 /* others */ 721 int mi_result; /* result so far: SP_BAD, SP_OK, etc. */ 722 int mi_capflags; /* WF_ONECAP WF_ALLCAP WF_KEEPCAP */ 723 win_T *mi_win; /* buffer being checked */ 724 725 /* for NOBREAK */ 726 int mi_result2; /* "mi_resul" without following word */ 727 char_u *mi_end2; /* "mi_end" without following word */ 728} matchinf_T; 729 730/* 731 * The tables used for recognizing word characters according to spelling. 732 * These are only used for the first 256 characters of 'encoding'. 733 */ 734typedef struct spelltab_S 735{ 736 char_u st_isw[256]; /* flags: is word char */ 737 char_u st_isu[256]; /* flags: is uppercase char */ 738 char_u st_fold[256]; /* chars: folded case */ 739 char_u st_upper[256]; /* chars: upper case */ 740} spelltab_T; 741 742static spelltab_T spelltab; 743static int did_set_spelltab; 744 745#define CF_WORD 0x01 746#define CF_UPPER 0x02 747 748static void clear_spell_chartab __ARGS((spelltab_T *sp)); 749static int set_spell_finish __ARGS((spelltab_T *new_st)); 750static int spell_iswordp __ARGS((char_u *p, win_T *wp)); 751static int spell_iswordp_nmw __ARGS((char_u *p)); 752#ifdef FEAT_MBYTE 753static int spell_mb_isword_class __ARGS((int cl)); 754static int spell_iswordp_w __ARGS((int *p, win_T *wp)); 755#endif 756static int write_spell_prefcond __ARGS((FILE *fd, garray_T *gap)); 757 758/* 759 * For finding suggestions: At each node in the tree these states are tried: 760 */ 761typedef enum 762{ 763 STATE_START = 0, /* At start of node check for NUL bytes (goodword 764 * ends); if badword ends there is a match, otherwise 765 * try splitting word. */ 766 STATE_NOPREFIX, /* try without prefix */ 767 STATE_SPLITUNDO, /* Undo splitting. */ 768 STATE_ENDNUL, /* Past NUL bytes at start of the node. */ 769 STATE_PLAIN, /* Use each byte of the node. */ 770 STATE_DEL, /* Delete a byte from the bad word. */ 771 STATE_INS_PREP, /* Prepare for inserting bytes. */ 772 STATE_INS, /* Insert a byte in the bad word. */ 773 STATE_SWAP, /* Swap two bytes. */ 774 STATE_UNSWAP, /* Undo swap two characters. */ 775 STATE_SWAP3, /* Swap two characters over three. */ 776 STATE_UNSWAP3, /* Undo Swap two characters over three. */ 777 STATE_UNROT3L, /* Undo rotate three characters left */ 778 STATE_UNROT3R, /* Undo rotate three characters right */ 779 STATE_REP_INI, /* Prepare for using REP items. */ 780 STATE_REP, /* Use matching REP items from the .aff file. */ 781 STATE_REP_UNDO, /* Undo a REP item replacement. */ 782 STATE_FINAL /* End of this node. */ 783} state_T; 784 785/* 786 * Struct to keep the state at each level in suggest_try_change(). 787 */ 788typedef struct trystate_S 789{ 790 state_T ts_state; /* state at this level, STATE_ */ 791 int ts_score; /* score */ 792 idx_T ts_arridx; /* index in tree array, start of node */ 793 short ts_curi; /* index in list of child nodes */ 794 char_u ts_fidx; /* index in fword[], case-folded bad word */ 795 char_u ts_fidxtry; /* ts_fidx at which bytes may be changed */ 796 char_u ts_twordlen; /* valid length of tword[] */ 797 char_u ts_prefixdepth; /* stack depth for end of prefix or 798 * PFD_PREFIXTREE or PFD_NOPREFIX */ 799 char_u ts_flags; /* TSF_ flags */ 800#ifdef FEAT_MBYTE 801 char_u ts_tcharlen; /* number of bytes in tword character */ 802 char_u ts_tcharidx; /* current byte index in tword character */ 803 char_u ts_isdiff; /* DIFF_ values */ 804 char_u ts_fcharstart; /* index in fword where badword char started */ 805#endif 806 char_u ts_prewordlen; /* length of word in "preword[]" */ 807 char_u ts_splitoff; /* index in "tword" after last split */ 808 char_u ts_splitfidx; /* "ts_fidx" at word split */ 809 char_u ts_complen; /* nr of compound words used */ 810 char_u ts_compsplit; /* index for "compflags" where word was spit */ 811 char_u ts_save_badflags; /* su_badflags saved here */ 812 char_u ts_delidx; /* index in fword for char that was deleted, 813 valid when "ts_flags" has TSF_DIDDEL */ 814} trystate_T; 815 816/* values for ts_isdiff */ 817#define DIFF_NONE 0 /* no different byte (yet) */ 818#define DIFF_YES 1 /* different byte found */ 819#define DIFF_INSERT 2 /* inserting character */ 820 821/* values for ts_flags */ 822#define TSF_PREFIXOK 1 /* already checked that prefix is OK */ 823#define TSF_DIDSPLIT 2 /* tried split at this point */ 824#define TSF_DIDDEL 4 /* did a delete, "ts_delidx" has index */ 825 826/* special values ts_prefixdepth */ 827#define PFD_NOPREFIX 0xff /* not using prefixes */ 828#define PFD_PREFIXTREE 0xfe /* walking through the prefix tree */ 829#define PFD_NOTSPECIAL 0xfd /* highest value that's not special */ 830 831/* mode values for find_word */ 832#define FIND_FOLDWORD 0 /* find word case-folded */ 833#define FIND_KEEPWORD 1 /* find keep-case word */ 834#define FIND_PREFIX 2 /* find word after prefix */ 835#define FIND_COMPOUND 3 /* find case-folded compound word */ 836#define FIND_KEEPCOMPOUND 4 /* find keep-case compound word */ 837 838static slang_T *slang_alloc __ARGS((char_u *lang)); 839static void slang_free __ARGS((slang_T *lp)); 840static void slang_clear __ARGS((slang_T *lp)); 841static void slang_clear_sug __ARGS((slang_T *lp)); 842static void find_word __ARGS((matchinf_T *mip, int mode)); 843static int match_checkcompoundpattern __ARGS((char_u *ptr, int wlen, garray_T *gap)); 844static int can_compound __ARGS((slang_T *slang, char_u *word, char_u *flags)); 845static int can_be_compound __ARGS((trystate_T *sp, slang_T *slang, char_u *compflags, int flag)); 846static int match_compoundrule __ARGS((slang_T *slang, char_u *compflags)); 847static int valid_word_prefix __ARGS((int totprefcnt, int arridx, int flags, char_u *word, slang_T *slang, int cond_req)); 848static void find_prefix __ARGS((matchinf_T *mip, int mode)); 849static int fold_more __ARGS((matchinf_T *mip)); 850static int spell_valid_case __ARGS((int wordflags, int treeflags)); 851static int no_spell_checking __ARGS((win_T *wp)); 852static void spell_load_lang __ARGS((char_u *lang)); 853static char_u *spell_enc __ARGS((void)); 854static void int_wordlist_spl __ARGS((char_u *fname)); 855static void spell_load_cb __ARGS((char_u *fname, void *cookie)); 856static slang_T *spell_load_file __ARGS((char_u *fname, char_u *lang, slang_T *old_lp, int silent)); 857static char_u *read_cnt_string __ARGS((FILE *fd, int cnt_bytes, int *lenp)); 858static int read_region_section __ARGS((FILE *fd, slang_T *slang, int len)); 859static int read_charflags_section __ARGS((FILE *fd)); 860static int read_prefcond_section __ARGS((FILE *fd, slang_T *lp)); 861static int read_rep_section __ARGS((FILE *fd, garray_T *gap, short *first)); 862static int read_sal_section __ARGS((FILE *fd, slang_T *slang)); 863static int read_words_section __ARGS((FILE *fd, slang_T *lp, int len)); 864static void count_common_word __ARGS((slang_T *lp, char_u *word, int len, int count)); 865static int score_wordcount_adj __ARGS((slang_T *slang, int score, char_u *word, int split)); 866static int read_sofo_section __ARGS((FILE *fd, slang_T *slang)); 867static int read_compound __ARGS((FILE *fd, slang_T *slang, int len)); 868static int byte_in_str __ARGS((char_u *str, int byte)); 869static int init_syl_tab __ARGS((slang_T *slang)); 870static int count_syllables __ARGS((slang_T *slang, char_u *word)); 871static int set_sofo __ARGS((slang_T *lp, char_u *from, char_u *to)); 872static void set_sal_first __ARGS((slang_T *lp)); 873#ifdef FEAT_MBYTE 874static int *mb_str2wide __ARGS((char_u *s)); 875#endif 876static int spell_read_tree __ARGS((FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt)); 877static idx_T read_tree_node __ARGS((FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr)); 878static void clear_midword __ARGS((win_T *buf)); 879static void use_midword __ARGS((slang_T *lp, win_T *buf)); 880static int find_region __ARGS((char_u *rp, char_u *region)); 881static int captype __ARGS((char_u *word, char_u *end)); 882static int badword_captype __ARGS((char_u *word, char_u *end)); 883static void spell_reload_one __ARGS((char_u *fname, int added_word)); 884static void set_spell_charflags __ARGS((char_u *flags, int cnt, char_u *upp)); 885static int set_spell_chartab __ARGS((char_u *fol, char_u *low, char_u *upp)); 886static int spell_casefold __ARGS((char_u *p, int len, char_u *buf, int buflen)); 887static int check_need_cap __ARGS((linenr_T lnum, colnr_T col)); 888static void spell_find_suggest __ARGS((char_u *badptr, int badlen, suginfo_T *su, int maxcount, int banbadword, int need_cap, int interactive)); 889#ifdef FEAT_EVAL 890static void spell_suggest_expr __ARGS((suginfo_T *su, char_u *expr)); 891#endif 892static void spell_suggest_file __ARGS((suginfo_T *su, char_u *fname)); 893static void spell_suggest_intern __ARGS((suginfo_T *su, int interactive)); 894static void suggest_load_files __ARGS((void)); 895static void tree_count_words __ARGS((char_u *byts, idx_T *idxs)); 896static void spell_find_cleanup __ARGS((suginfo_T *su)); 897static void onecap_copy __ARGS((char_u *word, char_u *wcopy, int upper)); 898static void allcap_copy __ARGS((char_u *word, char_u *wcopy)); 899static void suggest_try_special __ARGS((suginfo_T *su)); 900static void suggest_try_change __ARGS((suginfo_T *su)); 901static void suggest_trie_walk __ARGS((suginfo_T *su, langp_T *lp, char_u *fword, int soundfold)); 902static void go_deeper __ARGS((trystate_T *stack, int depth, int score_add)); 903#ifdef FEAT_MBYTE 904static int nofold_len __ARGS((char_u *fword, int flen, char_u *word)); 905#endif 906static void find_keepcap_word __ARGS((slang_T *slang, char_u *fword, char_u *kword)); 907static void score_comp_sal __ARGS((suginfo_T *su)); 908static void score_combine __ARGS((suginfo_T *su)); 909static int stp_sal_score __ARGS((suggest_T *stp, suginfo_T *su, slang_T *slang, char_u *badsound)); 910static void suggest_try_soundalike_prep __ARGS((void)); 911static void suggest_try_soundalike __ARGS((suginfo_T *su)); 912static void suggest_try_soundalike_finish __ARGS((void)); 913static void add_sound_suggest __ARGS((suginfo_T *su, char_u *goodword, int score, langp_T *lp)); 914static int soundfold_find __ARGS((slang_T *slang, char_u *word)); 915static void make_case_word __ARGS((char_u *fword, char_u *cword, int flags)); 916static void set_map_str __ARGS((slang_T *lp, char_u *map)); 917static int similar_chars __ARGS((slang_T *slang, int c1, int c2)); 918static void add_suggestion __ARGS((suginfo_T *su, garray_T *gap, char_u *goodword, int badlen, int score, int altscore, int had_bonus, slang_T *slang, int maxsf)); 919static void check_suggestions __ARGS((suginfo_T *su, garray_T *gap)); 920static void add_banned __ARGS((suginfo_T *su, char_u *word)); 921static void rescore_suggestions __ARGS((suginfo_T *su)); 922static void rescore_one __ARGS((suginfo_T *su, suggest_T *stp)); 923static int cleanup_suggestions __ARGS((garray_T *gap, int maxscore, int keep)); 924static void spell_soundfold __ARGS((slang_T *slang, char_u *inword, int folded, char_u *res)); 925static void spell_soundfold_sofo __ARGS((slang_T *slang, char_u *inword, char_u *res)); 926static void spell_soundfold_sal __ARGS((slang_T *slang, char_u *inword, char_u *res)); 927#ifdef FEAT_MBYTE 928static void spell_soundfold_wsal __ARGS((slang_T *slang, char_u *inword, char_u *res)); 929#endif 930static int soundalike_score __ARGS((char_u *goodsound, char_u *badsound)); 931static int spell_edit_score __ARGS((slang_T *slang, char_u *badword, char_u *goodword)); 932static int spell_edit_score_limit __ARGS((slang_T *slang, char_u *badword, char_u *goodword, int limit)); 933#ifdef FEAT_MBYTE 934static int spell_edit_score_limit_w __ARGS((slang_T *slang, char_u *badword, char_u *goodword, int limit)); 935#endif 936static void dump_word __ARGS((slang_T *slang, char_u *word, char_u *pat, int *dir, int round, int flags, linenr_T lnum)); 937static linenr_T dump_prefixes __ARGS((slang_T *slang, char_u *word, char_u *pat, int *dir, int round, int flags, linenr_T startlnum)); 938static buf_T *open_spellbuf __ARGS((void)); 939static void close_spellbuf __ARGS((buf_T *buf)); 940 941/* 942 * Use our own character-case definitions, because the current locale may 943 * differ from what the .spl file uses. 944 * These must not be called with negative number! 945 */ 946#ifndef FEAT_MBYTE 947/* Non-multi-byte implementation. */ 948# define SPELL_TOFOLD(c) ((c) < 256 ? (int)spelltab.st_fold[c] : (c)) 949# define SPELL_TOUPPER(c) ((c) < 256 ? (int)spelltab.st_upper[c] : (c)) 950# define SPELL_ISUPPER(c) ((c) < 256 ? spelltab.st_isu[c] : FALSE) 951#else 952# if defined(HAVE_WCHAR_H) 953# include <wchar.h> /* for towupper() and towlower() */ 954# endif 955/* Multi-byte implementation. For Unicode we can call utf_*(), but don't do 956 * that for ASCII, because we don't want to use 'casemap' here. Otherwise use 957 * the "w" library function for characters above 255 if available. */ 958# ifdef HAVE_TOWLOWER 959# define SPELL_TOFOLD(c) (enc_utf8 && (c) >= 128 ? utf_fold(c) \ 960 : (c) < 256 ? (int)spelltab.st_fold[c] : (int)towlower(c)) 961# else 962# define SPELL_TOFOLD(c) (enc_utf8 && (c) >= 128 ? utf_fold(c) \ 963 : (c) < 256 ? (int)spelltab.st_fold[c] : (c)) 964# endif 965 966# ifdef HAVE_TOWUPPER 967# define SPELL_TOUPPER(c) (enc_utf8 && (c) >= 128 ? utf_toupper(c) \ 968 : (c) < 256 ? (int)spelltab.st_upper[c] : (int)towupper(c)) 969# else 970# define SPELL_TOUPPER(c) (enc_utf8 && (c) >= 128 ? utf_toupper(c) \ 971 : (c) < 256 ? (int)spelltab.st_upper[c] : (c)) 972# endif 973 974# ifdef HAVE_ISWUPPER 975# define SPELL_ISUPPER(c) (enc_utf8 && (c) >= 128 ? utf_isupper(c) \ 976 : (c) < 256 ? spelltab.st_isu[c] : iswupper(c)) 977# else 978# define SPELL_ISUPPER(c) (enc_utf8 && (c) >= 128 ? utf_isupper(c) \ 979 : (c) < 256 ? spelltab.st_isu[c] : (FALSE)) 980# endif 981#endif 982 983 984static char *e_format = N_("E759: Format error in spell file"); 985static char *e_spell_trunc = N_("E758: Truncated spell file"); 986static char *e_afftrailing = N_("Trailing text in %s line %d: %s"); 987static char *e_affname = N_("Affix name too long in %s line %d: %s"); 988static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP"); 989static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range"); 990static char *msg_compressing = N_("Compressing word tree..."); 991 992/* Remember what "z?" replaced. */ 993static char_u *repl_from = NULL; 994static char_u *repl_to = NULL; 995 996/* 997 * Main spell-checking function. 998 * "ptr" points to a character that could be the start of a word. 999 * "*attrp" is set to the highlight index for a badly spelled word. For a 1000 * non-word or when it's OK it remains unchanged. 1001 * This must only be called when 'spelllang' is not empty. 1002 * 1003 * "capcol" is used to check for a Capitalised word after the end of a 1004 * sentence. If it's zero then perform the check. Return the column where to 1005 * check next, or -1 when no sentence end was found. If it's NULL then don't 1006 * worry. 1007 * 1008 * Returns the length of the word in bytes, also when it's OK, so that the 1009 * caller can skip over the word. 1010 */ 1011 int 1012spell_check(wp, ptr, attrp, capcol, docount) 1013 win_T *wp; /* current window */ 1014 char_u *ptr; 1015 hlf_T *attrp; 1016 int *capcol; /* column to check for Capital */ 1017 int docount; /* count good words */ 1018{ 1019 matchinf_T mi; /* Most things are put in "mi" so that it can 1020 be passed to functions quickly. */ 1021 int nrlen = 0; /* found a number first */ 1022 int c; 1023 int wrongcaplen = 0; 1024 int lpi; 1025 int count_word = docount; 1026 1027 /* A word never starts at a space or a control character. Return quickly 1028 * then, skipping over the character. */ 1029 if (*ptr <= ' ') 1030 return 1; 1031 1032 /* Return here when loading language files failed. */ 1033 if (wp->w_s->b_langp.ga_len == 0) 1034 return 1; 1035 1036 vim_memset(&mi, 0, sizeof(matchinf_T)); 1037 1038 /* A number is always OK. Also skip hexadecimal numbers 0xFF99 and 1039 * 0X99FF. But always do check spelling to find "3GPP" and "11 1040 * julifeest". */ 1041 if (*ptr >= '0' && *ptr <= '9') 1042 { 1043 if (*ptr == '0' && (ptr[1] == 'x' || ptr[1] == 'X')) 1044 mi.mi_end = skiphex(ptr + 2); 1045 else 1046 mi.mi_end = skipdigits(ptr); 1047 nrlen = (int)(mi.mi_end - ptr); 1048 } 1049 1050 /* Find the normal end of the word (until the next non-word character). */ 1051 mi.mi_word = ptr; 1052 mi.mi_fend = ptr; 1053 if (spell_iswordp(mi.mi_fend, wp)) 1054 { 1055 do 1056 { 1057 mb_ptr_adv(mi.mi_fend); 1058 } while (*mi.mi_fend != NUL && spell_iswordp(mi.mi_fend, wp)); 1059 1060 if (capcol != NULL && *capcol == 0 && wp->w_s->b_cap_prog != NULL) 1061 { 1062 /* Check word starting with capital letter. */ 1063 c = PTR2CHAR(ptr); 1064 if (!SPELL_ISUPPER(c)) 1065 wrongcaplen = (int)(mi.mi_fend - ptr); 1066 } 1067 } 1068 if (capcol != NULL) 1069 *capcol = -1; 1070 1071 /* We always use the characters up to the next non-word character, 1072 * also for bad words. */ 1073 mi.mi_end = mi.mi_fend; 1074 1075 /* Check caps type later. */ 1076 mi.mi_capflags = 0; 1077 mi.mi_cend = NULL; 1078 mi.mi_win = wp; 1079 1080 /* case-fold the word with one non-word character, so that we can check 1081 * for the word end. */ 1082 if (*mi.mi_fend != NUL) 1083 mb_ptr_adv(mi.mi_fend); 1084 1085 (void)spell_casefold(ptr, (int)(mi.mi_fend - ptr), mi.mi_fword, 1086 MAXWLEN + 1); 1087 mi.mi_fwordlen = (int)STRLEN(mi.mi_fword); 1088 1089 /* The word is bad unless we recognize it. */ 1090 mi.mi_result = SP_BAD; 1091 mi.mi_result2 = SP_BAD; 1092 1093 /* 1094 * Loop over the languages specified in 'spelllang'. 1095 * We check them all, because a word may be matched longer in another 1096 * language. 1097 */ 1098 for (lpi = 0; lpi < wp->w_s->b_langp.ga_len; ++lpi) 1099 { 1100 mi.mi_lp = LANGP_ENTRY(wp->w_s->b_langp, lpi); 1101 1102 /* If reloading fails the language is still in the list but everything 1103 * has been cleared. */ 1104 if (mi.mi_lp->lp_slang->sl_fidxs == NULL) 1105 continue; 1106 1107 /* Check for a matching word in case-folded words. */ 1108 find_word(&mi, FIND_FOLDWORD); 1109 1110 /* Check for a matching word in keep-case words. */ 1111 find_word(&mi, FIND_KEEPWORD); 1112 1113 /* Check for matching prefixes. */ 1114 find_prefix(&mi, FIND_FOLDWORD); 1115 1116 /* For a NOBREAK language, may want to use a word without a following 1117 * word as a backup. */ 1118 if (mi.mi_lp->lp_slang->sl_nobreak && mi.mi_result == SP_BAD 1119 && mi.mi_result2 != SP_BAD) 1120 { 1121 mi.mi_result = mi.mi_result2; 1122 mi.mi_end = mi.mi_end2; 1123 } 1124 1125 /* Count the word in the first language where it's found to be OK. */ 1126 if (count_word && mi.mi_result == SP_OK) 1127 { 1128 count_common_word(mi.mi_lp->lp_slang, ptr, 1129 (int)(mi.mi_end - ptr), 1); 1130 count_word = FALSE; 1131 } 1132 } 1133 1134 if (mi.mi_result != SP_OK) 1135 { 1136 /* If we found a number skip over it. Allows for "42nd". Do flag 1137 * rare and local words, e.g., "3GPP". */ 1138 if (nrlen > 0) 1139 { 1140 if (mi.mi_result == SP_BAD || mi.mi_result == SP_BANNED) 1141 return nrlen; 1142 } 1143 1144 /* When we are at a non-word character there is no error, just 1145 * skip over the character (try looking for a word after it). */ 1146 else if (!spell_iswordp_nmw(ptr)) 1147 { 1148 if (capcol != NULL && wp->w_s->b_cap_prog != NULL) 1149 { 1150 regmatch_T regmatch; 1151 1152 /* Check for end of sentence. */ 1153 regmatch.regprog = wp->w_s->b_cap_prog; 1154 regmatch.rm_ic = FALSE; 1155 if (vim_regexec(®match, ptr, 0)) 1156 *capcol = (int)(regmatch.endp[0] - ptr); 1157 } 1158 1159#ifdef FEAT_MBYTE 1160 if (has_mbyte) 1161 return (*mb_ptr2len)(ptr); 1162#endif 1163 return 1; 1164 } 1165 else if (mi.mi_end == ptr) 1166 /* Always include at least one character. Required for when there 1167 * is a mixup in "midword". */ 1168 mb_ptr_adv(mi.mi_end); 1169 else if (mi.mi_result == SP_BAD 1170 && LANGP_ENTRY(wp->w_s->b_langp, 0)->lp_slang->sl_nobreak) 1171 { 1172 char_u *p, *fp; 1173 int save_result = mi.mi_result; 1174 1175 /* First language in 'spelllang' is NOBREAK. Find first position 1176 * at which any word would be valid. */ 1177 mi.mi_lp = LANGP_ENTRY(wp->w_s->b_langp, 0); 1178 if (mi.mi_lp->lp_slang->sl_fidxs != NULL) 1179 { 1180 p = mi.mi_word; 1181 fp = mi.mi_fword; 1182 for (;;) 1183 { 1184 mb_ptr_adv(p); 1185 mb_ptr_adv(fp); 1186 if (p >= mi.mi_end) 1187 break; 1188 mi.mi_compoff = (int)(fp - mi.mi_fword); 1189 find_word(&mi, FIND_COMPOUND); 1190 if (mi.mi_result != SP_BAD) 1191 { 1192 mi.mi_end = p; 1193 break; 1194 } 1195 } 1196 mi.mi_result = save_result; 1197 } 1198 } 1199 1200 if (mi.mi_result == SP_BAD || mi.mi_result == SP_BANNED) 1201 *attrp = HLF_SPB; 1202 else if (mi.mi_result == SP_RARE) 1203 *attrp = HLF_SPR; 1204 else 1205 *attrp = HLF_SPL; 1206 } 1207 1208 if (wrongcaplen > 0 && (mi.mi_result == SP_OK || mi.mi_result == SP_RARE)) 1209 { 1210 /* Report SpellCap only when the word isn't badly spelled. */ 1211 *attrp = HLF_SPC; 1212 return wrongcaplen; 1213 } 1214 1215 return (int)(mi.mi_end - ptr); 1216} 1217 1218/* 1219 * Check if the word at "mip->mi_word" is in the tree. 1220 * When "mode" is FIND_FOLDWORD check in fold-case word tree. 1221 * When "mode" is FIND_KEEPWORD check in keep-case word tree. 1222 * When "mode" is FIND_PREFIX check for word after prefix in fold-case word 1223 * tree. 1224 * 1225 * For a match mip->mi_result is updated. 1226 */ 1227 static void 1228find_word(mip, mode) 1229 matchinf_T *mip; 1230 int mode; 1231{ 1232 idx_T arridx = 0; 1233 int endlen[MAXWLEN]; /* length at possible word endings */ 1234 idx_T endidx[MAXWLEN]; /* possible word endings */ 1235 int endidxcnt = 0; 1236 int len; 1237 int wlen = 0; 1238 int flen; 1239 int c; 1240 char_u *ptr; 1241 idx_T lo, hi, m; 1242#ifdef FEAT_MBYTE 1243 char_u *s; 1244#endif 1245 char_u *p; 1246 int res = SP_BAD; 1247 slang_T *slang = mip->mi_lp->lp_slang; 1248 unsigned flags; 1249 char_u *byts; 1250 idx_T *idxs; 1251 int word_ends; 1252 int prefix_found; 1253 int nobreak_result; 1254 1255 if (mode == FIND_KEEPWORD || mode == FIND_KEEPCOMPOUND) 1256 { 1257 /* Check for word with matching case in keep-case tree. */ 1258 ptr = mip->mi_word; 1259 flen = 9999; /* no case folding, always enough bytes */ 1260 byts = slang->sl_kbyts; 1261 idxs = slang->sl_kidxs; 1262 1263 if (mode == FIND_KEEPCOMPOUND) 1264 /* Skip over the previously found word(s). */ 1265 wlen += mip->mi_compoff; 1266 } 1267 else 1268 { 1269 /* Check for case-folded in case-folded tree. */ 1270 ptr = mip->mi_fword; 1271 flen = mip->mi_fwordlen; /* available case-folded bytes */ 1272 byts = slang->sl_fbyts; 1273 idxs = slang->sl_fidxs; 1274 1275 if (mode == FIND_PREFIX) 1276 { 1277 /* Skip over the prefix. */ 1278 wlen = mip->mi_prefixlen; 1279 flen -= mip->mi_prefixlen; 1280 } 1281 else if (mode == FIND_COMPOUND) 1282 { 1283 /* Skip over the previously found word(s). */ 1284 wlen = mip->mi_compoff; 1285 flen -= mip->mi_compoff; 1286 } 1287 1288 } 1289 1290 if (byts == NULL) 1291 return; /* array is empty */ 1292 1293 /* 1294 * Repeat advancing in the tree until: 1295 * - there is a byte that doesn't match, 1296 * - we reach the end of the tree, 1297 * - or we reach the end of the line. 1298 */ 1299 for (;;) 1300 { 1301 if (flen <= 0 && *mip->mi_fend != NUL) 1302 flen = fold_more(mip); 1303 1304 len = byts[arridx++]; 1305 1306 /* If the first possible byte is a zero the word could end here. 1307 * Remember this index, we first check for the longest word. */ 1308 if (byts[arridx] == 0) 1309 { 1310 if (endidxcnt == MAXWLEN) 1311 { 1312 /* Must be a corrupted spell file. */ 1313 EMSG(_(e_format)); 1314 return; 1315 } 1316 endlen[endidxcnt] = wlen; 1317 endidx[endidxcnt++] = arridx++; 1318 --len; 1319 1320 /* Skip over the zeros, there can be several flag/region 1321 * combinations. */ 1322 while (len > 0 && byts[arridx] == 0) 1323 { 1324 ++arridx; 1325 --len; 1326 } 1327 if (len == 0) 1328 break; /* no children, word must end here */ 1329 } 1330 1331 /* Stop looking at end of the line. */ 1332 if (ptr[wlen] == NUL) 1333 break; 1334 1335 /* Perform a binary search in the list of accepted bytes. */ 1336 c = ptr[wlen]; 1337 if (c == TAB) /* <Tab> is handled like <Space> */ 1338 c = ' '; 1339 lo = arridx; 1340 hi = arridx + len - 1; 1341 while (lo < hi) 1342 { 1343 m = (lo + hi) / 2; 1344 if (byts[m] > c) 1345 hi = m - 1; 1346 else if (byts[m] < c) 1347 lo = m + 1; 1348 else 1349 { 1350 lo = hi = m; 1351 break; 1352 } 1353 } 1354 1355 /* Stop if there is no matching byte. */ 1356 if (hi < lo || byts[lo] != c) 1357 break; 1358 1359 /* Continue at the child (if there is one). */ 1360 arridx = idxs[lo]; 1361 ++wlen; 1362 --flen; 1363 1364 /* One space in the good word may stand for several spaces in the 1365 * checked word. */ 1366 if (c == ' ') 1367 { 1368 for (;;) 1369 { 1370 if (flen <= 0 && *mip->mi_fend != NUL) 1371 flen = fold_more(mip); 1372 if (ptr[wlen] != ' ' && ptr[wlen] != TAB) 1373 break; 1374 ++wlen; 1375 --flen; 1376 } 1377 } 1378 } 1379 1380 /* 1381 * Verify that one of the possible endings is valid. Try the longest 1382 * first. 1383 */ 1384 while (endidxcnt > 0) 1385 { 1386 --endidxcnt; 1387 arridx = endidx[endidxcnt]; 1388 wlen = endlen[endidxcnt]; 1389 1390#ifdef FEAT_MBYTE 1391 if ((*mb_head_off)(ptr, ptr + wlen) > 0) 1392 continue; /* not at first byte of character */ 1393#endif 1394 if (spell_iswordp(ptr + wlen, mip->mi_win)) 1395 { 1396 if (slang->sl_compprog == NULL && !slang->sl_nobreak) 1397 continue; /* next char is a word character */ 1398 word_ends = FALSE; 1399 } 1400 else 1401 word_ends = TRUE; 1402 /* The prefix flag is before compound flags. Once a valid prefix flag 1403 * has been found we try compound flags. */ 1404 prefix_found = FALSE; 1405 1406#ifdef FEAT_MBYTE 1407 if (mode != FIND_KEEPWORD && has_mbyte) 1408 { 1409 /* Compute byte length in original word, length may change 1410 * when folding case. This can be slow, take a shortcut when the 1411 * case-folded word is equal to the keep-case word. */ 1412 p = mip->mi_word; 1413 if (STRNCMP(ptr, p, wlen) != 0) 1414 { 1415 for (s = ptr; s < ptr + wlen; mb_ptr_adv(s)) 1416 mb_ptr_adv(p); 1417 wlen = (int)(p - mip->mi_word); 1418 } 1419 } 1420#endif 1421 1422 /* Check flags and region. For FIND_PREFIX check the condition and 1423 * prefix ID. 1424 * Repeat this if there are more flags/region alternatives until there 1425 * is a match. */ 1426 res = SP_BAD; 1427 for (len = byts[arridx - 1]; len > 0 && byts[arridx] == 0; 1428 --len, ++arridx) 1429 { 1430 flags = idxs[arridx]; 1431 1432 /* For the fold-case tree check that the case of the checked word 1433 * matches with what the word in the tree requires. 1434 * For keep-case tree the case is always right. For prefixes we 1435 * don't bother to check. */ 1436 if (mode == FIND_FOLDWORD) 1437 { 1438 if (mip->mi_cend != mip->mi_word + wlen) 1439 { 1440 /* mi_capflags was set for a different word length, need 1441 * to do it again. */ 1442 mip->mi_cend = mip->mi_word + wlen; 1443 mip->mi_capflags = captype(mip->mi_word, mip->mi_cend); 1444 } 1445 1446 if (mip->mi_capflags == WF_KEEPCAP 1447 || !spell_valid_case(mip->mi_capflags, flags)) 1448 continue; 1449 } 1450 1451 /* When mode is FIND_PREFIX the word must support the prefix: 1452 * check the prefix ID and the condition. Do that for the list at 1453 * mip->mi_prefarridx that find_prefix() filled. */ 1454 else if (mode == FIND_PREFIX && !prefix_found) 1455 { 1456 c = valid_word_prefix(mip->mi_prefcnt, mip->mi_prefarridx, 1457 flags, 1458 mip->mi_word + mip->mi_cprefixlen, slang, 1459 FALSE); 1460 if (c == 0) 1461 continue; 1462 1463 /* Use the WF_RARE flag for a rare prefix. */ 1464 if (c & WF_RAREPFX) 1465 flags |= WF_RARE; 1466 prefix_found = TRUE; 1467 } 1468 1469 if (slang->sl_nobreak) 1470 { 1471 if ((mode == FIND_COMPOUND || mode == FIND_KEEPCOMPOUND) 1472 && (flags & WF_BANNED) == 0) 1473 { 1474 /* NOBREAK: found a valid following word. That's all we 1475 * need to know, so return. */ 1476 mip->mi_result = SP_OK; 1477 break; 1478 } 1479 } 1480 1481 else if ((mode == FIND_COMPOUND || mode == FIND_KEEPCOMPOUND 1482 || !word_ends)) 1483 { 1484 /* If there is no compound flag or the word is shorter than 1485 * COMPOUNDMIN reject it quickly. 1486 * Makes you wonder why someone puts a compound flag on a word 1487 * that's too short... Myspell compatibility requires this 1488 * anyway. */ 1489 if (((unsigned)flags >> 24) == 0 1490 || wlen - mip->mi_compoff < slang->sl_compminlen) 1491 continue; 1492#ifdef FEAT_MBYTE 1493 /* For multi-byte chars check character length against 1494 * COMPOUNDMIN. */ 1495 if (has_mbyte 1496 && slang->sl_compminlen > 0 1497 && mb_charlen_len(mip->mi_word + mip->mi_compoff, 1498 wlen - mip->mi_compoff) < slang->sl_compminlen) 1499 continue; 1500#endif 1501 1502 /* Limit the number of compound words to COMPOUNDWORDMAX if no 1503 * maximum for syllables is specified. */ 1504 if (!word_ends && mip->mi_complen + mip->mi_compextra + 2 1505 > slang->sl_compmax 1506 && slang->sl_compsylmax == MAXWLEN) 1507 continue; 1508 1509 /* Don't allow compounding on a side where an affix was added, 1510 * unless COMPOUNDPERMITFLAG was used. */ 1511 if (mip->mi_complen > 0 && (flags & WF_NOCOMPBEF)) 1512 continue; 1513 if (!word_ends && (flags & WF_NOCOMPAFT)) 1514 continue; 1515 1516 /* Quickly check if compounding is possible with this flag. */ 1517 if (!byte_in_str(mip->mi_complen == 0 1518 ? slang->sl_compstartflags 1519 : slang->sl_compallflags, 1520 ((unsigned)flags >> 24))) 1521 continue; 1522 1523 /* If there is a match with a CHECKCOMPOUNDPATTERN rule 1524 * discard the compound word. */ 1525 if (match_checkcompoundpattern(ptr, wlen, &slang->sl_comppat)) 1526 continue; 1527 1528 if (mode == FIND_COMPOUND) 1529 { 1530 int capflags; 1531 1532 /* Need to check the caps type of the appended compound 1533 * word. */ 1534#ifdef FEAT_MBYTE 1535 if (has_mbyte && STRNCMP(ptr, mip->mi_word, 1536 mip->mi_compoff) != 0) 1537 { 1538 /* case folding may have changed the length */ 1539 p = mip->mi_word; 1540 for (s = ptr; s < ptr + mip->mi_compoff; mb_ptr_adv(s)) 1541 mb_ptr_adv(p); 1542 } 1543 else 1544#endif 1545 p = mip->mi_word + mip->mi_compoff; 1546 capflags = captype(p, mip->mi_word + wlen); 1547 if (capflags == WF_KEEPCAP || (capflags == WF_ALLCAP 1548 && (flags & WF_FIXCAP) != 0)) 1549 continue; 1550 1551 if (capflags != WF_ALLCAP) 1552 { 1553 /* When the character before the word is a word 1554 * character we do not accept a Onecap word. We do 1555 * accept a no-caps word, even when the dictionary 1556 * word specifies ONECAP. */ 1557 mb_ptr_back(mip->mi_word, p); 1558 if (spell_iswordp_nmw(p) 1559 ? capflags == WF_ONECAP 1560 : (flags & WF_ONECAP) != 0 1561 && capflags != WF_ONECAP) 1562 continue; 1563 } 1564 } 1565 1566 /* If the word ends the sequence of compound flags of the 1567 * words must match with one of the COMPOUNDRULE items and 1568 * the number of syllables must not be too large. */ 1569 mip->mi_compflags[mip->mi_complen] = ((unsigned)flags >> 24); 1570 mip->mi_compflags[mip->mi_complen + 1] = NUL; 1571 if (word_ends) 1572 { 1573 char_u fword[MAXWLEN]; 1574 1575 if (slang->sl_compsylmax < MAXWLEN) 1576 { 1577 /* "fword" is only needed for checking syllables. */ 1578 if (ptr == mip->mi_word) 1579 (void)spell_casefold(ptr, wlen, fword, MAXWLEN); 1580 else 1581 vim_strncpy(fword, ptr, endlen[endidxcnt]); 1582 } 1583 if (!can_compound(slang, fword, mip->mi_compflags)) 1584 continue; 1585 } 1586 else if (slang->sl_comprules != NULL 1587 && !match_compoundrule(slang, mip->mi_compflags)) 1588 /* The compound flags collected so far do not match any 1589 * COMPOUNDRULE, discard the compounded word. */ 1590 continue; 1591 } 1592 1593 /* Check NEEDCOMPOUND: can't use word without compounding. */ 1594 else if (flags & WF_NEEDCOMP) 1595 continue; 1596 1597 nobreak_result = SP_OK; 1598 1599 if (!word_ends) 1600 { 1601 int save_result = mip->mi_result; 1602 char_u *save_end = mip->mi_end; 1603 langp_T *save_lp = mip->mi_lp; 1604 int lpi; 1605 1606 /* Check that a valid word follows. If there is one and we 1607 * are compounding, it will set "mi_result", thus we are 1608 * always finished here. For NOBREAK we only check that a 1609 * valid word follows. 1610 * Recursive! */ 1611 if (slang->sl_nobreak) 1612 mip->mi_result = SP_BAD; 1613 1614 /* Find following word in case-folded tree. */ 1615 mip->mi_compoff = endlen[endidxcnt]; 1616#ifdef FEAT_MBYTE 1617 if (has_mbyte && mode == FIND_KEEPWORD) 1618 { 1619 /* Compute byte length in case-folded word from "wlen": 1620 * byte length in keep-case word. Length may change when 1621 * folding case. This can be slow, take a shortcut when 1622 * the case-folded word is equal to the keep-case word. */ 1623 p = mip->mi_fword; 1624 if (STRNCMP(ptr, p, wlen) != 0) 1625 { 1626 for (s = ptr; s < ptr + wlen; mb_ptr_adv(s)) 1627 mb_ptr_adv(p); 1628 mip->mi_compoff = (int)(p - mip->mi_fword); 1629 } 1630 } 1631#endif 1632 c = mip->mi_compoff; 1633 ++mip->mi_complen; 1634 if (flags & WF_COMPROOT) 1635 ++mip->mi_compextra; 1636 1637 /* For NOBREAK we need to try all NOBREAK languages, at least 1638 * to find the ".add" file(s). */ 1639 for (lpi = 0; lpi < mip->mi_win->w_s->b_langp.ga_len; ++lpi) 1640 { 1641 if (slang->sl_nobreak) 1642 { 1643 mip->mi_lp = LANGP_ENTRY(mip->mi_win->w_s->b_langp, lpi); 1644 if (mip->mi_lp->lp_slang->sl_fidxs == NULL 1645 || !mip->mi_lp->lp_slang->sl_nobreak) 1646 continue; 1647 } 1648 1649 find_word(mip, FIND_COMPOUND); 1650 1651 /* When NOBREAK any word that matches is OK. Otherwise we 1652 * need to find the longest match, thus try with keep-case 1653 * and prefix too. */ 1654 if (!slang->sl_nobreak || mip->mi_result == SP_BAD) 1655 { 1656 /* Find following word in keep-case tree. */ 1657 mip->mi_compoff = wlen; 1658 find_word(mip, FIND_KEEPCOMPOUND); 1659 1660#if 0 /* Disabled, a prefix must not appear halfway a compound word, 1661 unless the COMPOUNDPERMITFLAG is used and then it can't be a 1662 postponed prefix. */ 1663 if (!slang->sl_nobreak || mip->mi_result == SP_BAD) 1664 { 1665 /* Check for following word with prefix. */ 1666 mip->mi_compoff = c; 1667 find_prefix(mip, FIND_COMPOUND); 1668 } 1669#endif 1670 } 1671 1672 if (!slang->sl_nobreak) 1673 break; 1674 } 1675 --mip->mi_complen; 1676 if (flags & WF_COMPROOT) 1677 --mip->mi_compextra; 1678 mip->mi_lp = save_lp; 1679 1680 if (slang->sl_nobreak) 1681 { 1682 nobreak_result = mip->mi_result; 1683 mip->mi_result = save_result; 1684 mip->mi_end = save_end; 1685 } 1686 else 1687 { 1688 if (mip->mi_result == SP_OK) 1689 break; 1690 continue; 1691 } 1692 } 1693 1694 if (flags & WF_BANNED) 1695 res = SP_BANNED; 1696 else if (flags & WF_REGION) 1697 { 1698 /* Check region. */ 1699 if ((mip->mi_lp->lp_region & (flags >> 16)) != 0) 1700 res = SP_OK; 1701 else 1702 res = SP_LOCAL; 1703 } 1704 else if (flags & WF_RARE) 1705 res = SP_RARE; 1706 else 1707 res = SP_OK; 1708 1709 /* Always use the longest match and the best result. For NOBREAK 1710 * we separately keep the longest match without a following good 1711 * word as a fall-back. */ 1712 if (nobreak_result == SP_BAD) 1713 { 1714 if (mip->mi_result2 > res) 1715 { 1716 mip->mi_result2 = res; 1717 mip->mi_end2 = mip->mi_word + wlen; 1718 } 1719 else if (mip->mi_result2 == res 1720 && mip->mi_end2 < mip->mi_word + wlen) 1721 mip->mi_end2 = mip->mi_word + wlen; 1722 } 1723 else if (mip->mi_result > res) 1724 { 1725 mip->mi_result = res; 1726 mip->mi_end = mip->mi_word + wlen; 1727 } 1728 else if (mip->mi_result == res && mip->mi_end < mip->mi_word + wlen) 1729 mip->mi_end = mip->mi_word + wlen; 1730 1731 if (mip->mi_result == SP_OK) 1732 break; 1733 } 1734 1735 if (mip->mi_result == SP_OK) 1736 break; 1737 } 1738} 1739 1740/* 1741 * Return TRUE if there is a match between the word ptr[wlen] and 1742 * CHECKCOMPOUNDPATTERN rules, assuming that we will concatenate with another 1743 * word. 1744 * A match means that the first part of CHECKCOMPOUNDPATTERN matches at the 1745 * end of ptr[wlen] and the second part matches after it. 1746 */ 1747 static int 1748match_checkcompoundpattern(ptr, wlen, gap) 1749 char_u *ptr; 1750 int wlen; 1751 garray_T *gap; /* &sl_comppat */ 1752{ 1753 int i; 1754 char_u *p; 1755 int len; 1756 1757 for (i = 0; i + 1 < gap->ga_len; i += 2) 1758 { 1759 p = ((char_u **)gap->ga_data)[i + 1]; 1760 if (STRNCMP(ptr + wlen, p, STRLEN(p)) == 0) 1761 { 1762 /* Second part matches at start of following compound word, now 1763 * check if first part matches at end of previous word. */ 1764 p = ((char_u **)gap->ga_data)[i]; 1765 len = (int)STRLEN(p); 1766 if (len <= wlen && STRNCMP(ptr + wlen - len, p, len) == 0) 1767 return TRUE; 1768 } 1769 } 1770 return FALSE; 1771} 1772 1773/* 1774 * Return TRUE if "flags" is a valid sequence of compound flags and "word" 1775 * does not have too many syllables. 1776 */ 1777 static int 1778can_compound(slang, word, flags) 1779 slang_T *slang; 1780 char_u *word; 1781 char_u *flags; 1782{ 1783 regmatch_T regmatch; 1784#ifdef FEAT_MBYTE 1785 char_u uflags[MAXWLEN * 2]; 1786 int i; 1787#endif 1788 char_u *p; 1789 1790 if (slang->sl_compprog == NULL) 1791 return FALSE; 1792#ifdef FEAT_MBYTE 1793 if (enc_utf8) 1794 { 1795 /* Need to convert the single byte flags to utf8 characters. */ 1796 p = uflags; 1797 for (i = 0; flags[i] != NUL; ++i) 1798 p += mb_char2bytes(flags[i], p); 1799 *p = NUL; 1800 p = uflags; 1801 } 1802 else 1803#endif 1804 p = flags; 1805 regmatch.regprog = slang->sl_compprog; 1806 regmatch.rm_ic = FALSE; 1807 if (!vim_regexec(®match, p, 0)) 1808 return FALSE; 1809 1810 /* Count the number of syllables. This may be slow, do it last. If there 1811 * are too many syllables AND the number of compound words is above 1812 * COMPOUNDWORDMAX then compounding is not allowed. */ 1813 if (slang->sl_compsylmax < MAXWLEN 1814 && count_syllables(slang, word) > slang->sl_compsylmax) 1815 return (int)STRLEN(flags) < slang->sl_compmax; 1816 return TRUE; 1817} 1818 1819/* 1820 * Return TRUE when the sequence of flags in "compflags" plus "flag" can 1821 * possibly form a valid compounded word. This also checks the COMPOUNDRULE 1822 * lines if they don't contain wildcards. 1823 */ 1824 static int 1825can_be_compound(sp, slang, compflags, flag) 1826 trystate_T *sp; 1827 slang_T *slang; 1828 char_u *compflags; 1829 int flag; 1830{ 1831 /* If the flag doesn't appear in sl_compstartflags or sl_compallflags 1832 * then it can't possibly compound. */ 1833 if (!byte_in_str(sp->ts_complen == sp->ts_compsplit 1834 ? slang->sl_compstartflags : slang->sl_compallflags, flag)) 1835 return FALSE; 1836 1837 /* If there are no wildcards, we can check if the flags collected so far 1838 * possibly can form a match with COMPOUNDRULE patterns. This only 1839 * makes sense when we have two or more words. */ 1840 if (slang->sl_comprules != NULL && sp->ts_complen > sp->ts_compsplit) 1841 { 1842 int v; 1843 1844 compflags[sp->ts_complen] = flag; 1845 compflags[sp->ts_complen + 1] = NUL; 1846 v = match_compoundrule(slang, compflags + sp->ts_compsplit); 1847 compflags[sp->ts_complen] = NUL; 1848 return v; 1849 } 1850 1851 return TRUE; 1852} 1853 1854 1855/* 1856 * Return TRUE if the compound flags in compflags[] match the start of any 1857 * compound rule. This is used to stop trying a compound if the flags 1858 * collected so far can't possibly match any compound rule. 1859 * Caller must check that slang->sl_comprules is not NULL. 1860 */ 1861 static int 1862match_compoundrule(slang, compflags) 1863 slang_T *slang; 1864 char_u *compflags; 1865{ 1866 char_u *p; 1867 int i; 1868 int c; 1869 1870 /* loop over all the COMPOUNDRULE entries */ 1871 for (p = slang->sl_comprules; *p != NUL; ++p) 1872 { 1873 /* loop over the flags in the compound word we have made, match 1874 * them against the current rule entry */ 1875 for (i = 0; ; ++i) 1876 { 1877 c = compflags[i]; 1878 if (c == NUL) 1879 /* found a rule that matches for the flags we have so far */ 1880 return TRUE; 1881 if (*p == '/' || *p == NUL) 1882 break; /* end of rule, it's too short */ 1883 if (*p == '[') 1884 { 1885 int match = FALSE; 1886 1887 /* compare against all the flags in [] */ 1888 ++p; 1889 while (*p != ']' && *p != NUL) 1890 if (*p++ == c) 1891 match = TRUE; 1892 if (!match) 1893 break; /* none matches */ 1894 } 1895 else if (*p != c) 1896 break; /* flag of word doesn't match flag in pattern */ 1897 ++p; 1898 } 1899 1900 /* Skip to the next "/", where the next pattern starts. */ 1901 p = vim_strchr(p, '/'); 1902 if (p == NULL) 1903 break; 1904 } 1905 1906 /* Checked all the rules and none of them match the flags, so there 1907 * can't possibly be a compound starting with these flags. */ 1908 return FALSE; 1909} 1910 1911/* 1912 * Return non-zero if the prefix indicated by "arridx" matches with the prefix 1913 * ID in "flags" for the word "word". 1914 * The WF_RAREPFX flag is included in the return value for a rare prefix. 1915 */ 1916 static int 1917valid_word_prefix(totprefcnt, arridx, flags, word, slang, cond_req) 1918 int totprefcnt; /* nr of prefix IDs */ 1919 int arridx; /* idx in sl_pidxs[] */ 1920 int flags; 1921 char_u *word; 1922 slang_T *slang; 1923 int cond_req; /* only use prefixes with a condition */ 1924{ 1925 int prefcnt; 1926 int pidx; 1927 regprog_T *rp; 1928 regmatch_T regmatch; 1929 int prefid; 1930 1931 prefid = (unsigned)flags >> 24; 1932 for (prefcnt = totprefcnt - 1; prefcnt >= 0; --prefcnt) 1933 { 1934 pidx = slang->sl_pidxs[arridx + prefcnt]; 1935 1936 /* Check the prefix ID. */ 1937 if (prefid != (pidx & 0xff)) 1938 continue; 1939 1940 /* Check if the prefix doesn't combine and the word already has a 1941 * suffix. */ 1942 if ((flags & WF_HAS_AFF) && (pidx & WF_PFX_NC)) 1943 continue; 1944 1945 /* Check the condition, if there is one. The condition index is 1946 * stored in the two bytes above the prefix ID byte. */ 1947 rp = slang->sl_prefprog[((unsigned)pidx >> 8) & 0xffff]; 1948 if (rp != NULL) 1949 { 1950 regmatch.regprog = rp; 1951 regmatch.rm_ic = FALSE; 1952 if (!vim_regexec(®match, word, 0)) 1953 continue; 1954 } 1955 else if (cond_req) 1956 continue; 1957 1958 /* It's a match! Return the WF_ flags. */ 1959 return pidx; 1960 } 1961 return 0; 1962} 1963 1964/* 1965 * Check if the word at "mip->mi_word" has a matching prefix. 1966 * If it does, then check the following word. 1967 * 1968 * If "mode" is "FIND_COMPOUND" then do the same after another word, find a 1969 * prefix in a compound word. 1970 * 1971 * For a match mip->mi_result is updated. 1972 */ 1973 static void 1974find_prefix(mip, mode) 1975 matchinf_T *mip; 1976 int mode; 1977{ 1978 idx_T arridx = 0; 1979 int len; 1980 int wlen = 0; 1981 int flen; 1982 int c; 1983 char_u *ptr; 1984 idx_T lo, hi, m; 1985 slang_T *slang = mip->mi_lp->lp_slang; 1986 char_u *byts; 1987 idx_T *idxs; 1988 1989 byts = slang->sl_pbyts; 1990 if (byts == NULL) 1991 return; /* array is empty */ 1992 1993 /* We use the case-folded word here, since prefixes are always 1994 * case-folded. */ 1995 ptr = mip->mi_fword; 1996 flen = mip->mi_fwordlen; /* available case-folded bytes */ 1997 if (mode == FIND_COMPOUND) 1998 { 1999 /* Skip over the previously found word(s). */ 2000 ptr += mip->mi_compoff; 2001 flen -= mip->mi_compoff; 2002 } 2003 idxs = slang->sl_pidxs; 2004 2005 /* 2006 * Repeat advancing in the tree until: 2007 * - there is a byte that doesn't match, 2008 * - we reach the end of the tree, 2009 * - or we reach the end of the line. 2010 */ 2011 for (;;) 2012 { 2013 if (flen == 0 && *mip->mi_fend != NUL) 2014 flen = fold_more(mip); 2015 2016 len = byts[arridx++]; 2017 2018 /* If the first possible byte is a zero the prefix could end here. 2019 * Check if the following word matches and supports the prefix. */ 2020 if (byts[arridx] == 0) 2021 { 2022 /* There can be several prefixes with different conditions. We 2023 * try them all, since we don't know which one will give the 2024 * longest match. The word is the same each time, pass the list 2025 * of possible prefixes to find_word(). */ 2026 mip->mi_prefarridx = arridx; 2027 mip->mi_prefcnt = len; 2028 while (len > 0 && byts[arridx] == 0) 2029 { 2030 ++arridx; 2031 --len; 2032 } 2033 mip->mi_prefcnt -= len; 2034 2035 /* Find the word that comes after the prefix. */ 2036 mip->mi_prefixlen = wlen; 2037 if (mode == FIND_COMPOUND) 2038 /* Skip over the previously found word(s). */ 2039 mip->mi_prefixlen += mip->mi_compoff; 2040 2041#ifdef FEAT_MBYTE 2042 if (has_mbyte) 2043 { 2044 /* Case-folded length may differ from original length. */ 2045 mip->mi_cprefixlen = nofold_len(mip->mi_fword, 2046 mip->mi_prefixlen, mip->mi_word); 2047 } 2048 else 2049 mip->mi_cprefixlen = mip->mi_prefixlen; 2050#endif 2051 find_word(mip, FIND_PREFIX); 2052 2053 2054 if (len == 0) 2055 break; /* no children, word must end here */ 2056 } 2057 2058 /* Stop looking at end of the line. */ 2059 if (ptr[wlen] == NUL) 2060 break; 2061 2062 /* Perform a binary search in the list of accepted bytes. */ 2063 c = ptr[wlen]; 2064 lo = arridx; 2065 hi = arridx + len - 1; 2066 while (lo < hi) 2067 { 2068 m = (lo + hi) / 2; 2069 if (byts[m] > c) 2070 hi = m - 1; 2071 else if (byts[m] < c) 2072 lo = m + 1; 2073 else 2074 { 2075 lo = hi = m; 2076 break; 2077 } 2078 } 2079 2080 /* Stop if there is no matching byte. */ 2081 if (hi < lo || byts[lo] != c) 2082 break; 2083 2084 /* Continue at the child (if there is one). */ 2085 arridx = idxs[lo]; 2086 ++wlen; 2087 --flen; 2088 } 2089} 2090 2091/* 2092 * Need to fold at least one more character. Do until next non-word character 2093 * for efficiency. Include the non-word character too. 2094 * Return the length of the folded chars in bytes. 2095 */ 2096 static int 2097fold_more(mip) 2098 matchinf_T *mip; 2099{ 2100 int flen; 2101 char_u *p; 2102 2103 p = mip->mi_fend; 2104 do 2105 { 2106 mb_ptr_adv(mip->mi_fend); 2107 } while (*mip->mi_fend != NUL && spell_iswordp(mip->mi_fend, mip->mi_win)); 2108 2109 /* Include the non-word character so that we can check for the word end. */ 2110 if (*mip->mi_fend != NUL) 2111 mb_ptr_adv(mip->mi_fend); 2112 2113 (void)spell_casefold(p, (int)(mip->mi_fend - p), 2114 mip->mi_fword + mip->mi_fwordlen, 2115 MAXWLEN - mip->mi_fwordlen); 2116 flen = (int)STRLEN(mip->mi_fword + mip->mi_fwordlen); 2117 mip->mi_fwordlen += flen; 2118 return flen; 2119} 2120 2121/* 2122 * Check case flags for a word. Return TRUE if the word has the requested 2123 * case. 2124 */ 2125 static int 2126spell_valid_case(wordflags, treeflags) 2127 int wordflags; /* flags for the checked word. */ 2128 int treeflags; /* flags for the word in the spell tree */ 2129{ 2130 return ((wordflags == WF_ALLCAP && (treeflags & WF_FIXCAP) == 0) 2131 || ((treeflags & (WF_ALLCAP | WF_KEEPCAP)) == 0 2132 && ((treeflags & WF_ONECAP) == 0 2133 || (wordflags & WF_ONECAP) != 0))); 2134} 2135 2136/* 2137 * Return TRUE if spell checking is not enabled. 2138 */ 2139 static int 2140no_spell_checking(wp) 2141 win_T *wp; 2142{ 2143 if (!wp->w_p_spell || *wp->w_s->b_p_spl == NUL 2144 || wp->w_s->b_langp.ga_len == 0) 2145 { 2146 EMSG(_("E756: Spell checking is not enabled")); 2147 return TRUE; 2148 } 2149 return FALSE; 2150} 2151 2152/* 2153 * Move to next spell error. 2154 * "curline" is FALSE for "[s", "]s", "[S" and "]S". 2155 * "curline" is TRUE to find word under/after cursor in the same line. 2156 * For Insert mode completion "dir" is BACKWARD and "curline" is TRUE: move 2157 * to after badly spelled word before the cursor. 2158 * Return 0 if not found, length of the badly spelled word otherwise. 2159 */ 2160 int 2161spell_move_to(wp, dir, allwords, curline, attrp) 2162 win_T *wp; 2163 int dir; /* FORWARD or BACKWARD */ 2164 int allwords; /* TRUE for "[s"/"]s", FALSE for "[S"/"]S" */ 2165 int curline; 2166 hlf_T *attrp; /* return: attributes of bad word or NULL 2167 (only when "dir" is FORWARD) */ 2168{ 2169 linenr_T lnum; 2170 pos_T found_pos; 2171 int found_len = 0; 2172 char_u *line; 2173 char_u *p; 2174 char_u *endp; 2175 hlf_T attr; 2176 int len; 2177# ifdef FEAT_SYN_HL 2178 int has_syntax = syntax_present(wp); 2179# endif 2180 int col; 2181 int can_spell; 2182 char_u *buf = NULL; 2183 int buflen = 0; 2184 int skip = 0; 2185 int capcol = -1; 2186 int found_one = FALSE; 2187 int wrapped = FALSE; 2188 2189 if (no_spell_checking(wp)) 2190 return 0; 2191 2192 /* 2193 * Start looking for bad word at the start of the line, because we can't 2194 * start halfway a word, we don't know where it starts or ends. 2195 * 2196 * When searching backwards, we continue in the line to find the last 2197 * bad word (in the cursor line: before the cursor). 2198 * 2199 * We concatenate the start of the next line, so that wrapped words work 2200 * (e.g. "et<line-break>cetera"). Doesn't work when searching backwards 2201 * though... 2202 */ 2203 lnum = wp->w_cursor.lnum; 2204 clearpos(&found_pos); 2205 2206 while (!got_int) 2207 { 2208 line = ml_get_buf(wp->w_buffer, lnum, FALSE); 2209 2210 len = (int)STRLEN(line); 2211 if (buflen < len + MAXWLEN + 2) 2212 { 2213 vim_free(buf); 2214 buflen = len + MAXWLEN + 2; 2215 buf = alloc(buflen); 2216 if (buf == NULL) 2217 break; 2218 } 2219 2220 /* In first line check first word for Capital. */ 2221 if (lnum == 1) 2222 capcol = 0; 2223 2224 /* For checking first word with a capital skip white space. */ 2225 if (capcol == 0) 2226 capcol = (int)(skipwhite(line) - line); 2227 else if (curline && wp == curwin) 2228 { 2229 /* For spellbadword(): check if first word needs a capital. */ 2230 col = (int)(skipwhite(line) - line); 2231 if (check_need_cap(lnum, col)) 2232 capcol = col; 2233 2234 /* Need to get the line again, may have looked at the previous 2235 * one. */ 2236 line = ml_get_buf(wp->w_buffer, lnum, FALSE); 2237 } 2238 2239 /* Copy the line into "buf" and append the start of the next line if 2240 * possible. */ 2241 STRCPY(buf, line); 2242 if (lnum < wp->w_buffer->b_ml.ml_line_count) 2243 spell_cat_line(buf + STRLEN(buf), 2244 ml_get_buf(wp->w_buffer, lnum + 1, FALSE), MAXWLEN); 2245 2246 p = buf + skip; 2247 endp = buf + len; 2248 while (p < endp) 2249 { 2250 /* When searching backward don't search after the cursor. Unless 2251 * we wrapped around the end of the buffer. */ 2252 if (dir == BACKWARD 2253 && lnum == wp->w_cursor.lnum 2254 && !wrapped 2255 && (colnr_T)(p - buf) >= wp->w_cursor.col) 2256 break; 2257 2258 /* start of word */ 2259 attr = HLF_COUNT; 2260 len = spell_check(wp, p, &attr, &capcol, FALSE); 2261 2262 if (attr != HLF_COUNT) 2263 { 2264 /* We found a bad word. Check the attribute. */ 2265 if (allwords || attr == HLF_SPB) 2266 { 2267 /* When searching forward only accept a bad word after 2268 * the cursor. */ 2269 if (dir == BACKWARD 2270 || lnum != wp->w_cursor.lnum 2271 || (lnum == wp->w_cursor.lnum 2272 && (wrapped 2273 || (colnr_T)(curline ? p - buf + len 2274 : p - buf) 2275 > wp->w_cursor.col))) 2276 { 2277# ifdef FEAT_SYN_HL 2278 if (has_syntax) 2279 { 2280 col = (int)(p - buf); 2281 (void)syn_get_id(wp, lnum, (colnr_T)col, 2282 FALSE, &can_spell, FALSE); 2283 if (!can_spell) 2284 attr = HLF_COUNT; 2285 } 2286 else 2287#endif 2288 can_spell = TRUE; 2289 2290 if (can_spell) 2291 { 2292 found_one = TRUE; 2293 found_pos.lnum = lnum; 2294 found_pos.col = (int)(p - buf); 2295#ifdef FEAT_VIRTUALEDIT 2296 found_pos.coladd = 0; 2297#endif 2298 if (dir == FORWARD) 2299 { 2300 /* No need to search further. */ 2301 wp->w_cursor = found_pos; 2302 vim_free(buf); 2303 if (attrp != NULL) 2304 *attrp = attr; 2305 return len; 2306 } 2307 else if (curline) 2308 /* Insert mode completion: put cursor after 2309 * the bad word. */ 2310 found_pos.col += len; 2311 found_len = len; 2312 } 2313 } 2314 else 2315 found_one = TRUE; 2316 } 2317 } 2318 2319 /* advance to character after the word */ 2320 p += len; 2321 capcol -= len; 2322 } 2323 2324 if (dir == BACKWARD && found_pos.lnum != 0) 2325 { 2326 /* Use the last match in the line (before the cursor). */ 2327 wp->w_cursor = found_pos; 2328 vim_free(buf); 2329 return found_len; 2330 } 2331 2332 if (curline) 2333 break; /* only check cursor line */ 2334 2335 /* Advance to next line. */ 2336 if (dir == BACKWARD) 2337 { 2338 /* If we are back at the starting line and searched it again there 2339 * is no match, give up. */ 2340 if (lnum == wp->w_cursor.lnum && wrapped) 2341 break; 2342 2343 if (lnum > 1) 2344 --lnum; 2345 else if (!p_ws) 2346 break; /* at first line and 'nowrapscan' */ 2347 else 2348 { 2349 /* Wrap around to the end of the buffer. May search the 2350 * starting line again and accept the last match. */ 2351 lnum = wp->w_buffer->b_ml.ml_line_count; 2352 wrapped = TRUE; 2353 if (!shortmess(SHM_SEARCH)) 2354 give_warning((char_u *)_(top_bot_msg), TRUE); 2355 } 2356 capcol = -1; 2357 } 2358 else 2359 { 2360 if (lnum < wp->w_buffer->b_ml.ml_line_count) 2361 ++lnum; 2362 else if (!p_ws) 2363 break; /* at first line and 'nowrapscan' */ 2364 else 2365 { 2366 /* Wrap around to the start of the buffer. May search the 2367 * starting line again and accept the first match. */ 2368 lnum = 1; 2369 wrapped = TRUE; 2370 if (!shortmess(SHM_SEARCH)) 2371 give_warning((char_u *)_(bot_top_msg), TRUE); 2372 } 2373 2374 /* If we are back at the starting line and there is no match then 2375 * give up. */ 2376 if (lnum == wp->w_cursor.lnum && (!found_one || wrapped)) 2377 break; 2378 2379 /* Skip the characters at the start of the next line that were 2380 * included in a match crossing line boundaries. */ 2381 if (attr == HLF_COUNT) 2382 skip = (int)(p - endp); 2383 else 2384 skip = 0; 2385 2386 /* Capcol skips over the inserted space. */ 2387 --capcol; 2388 2389 /* But after empty line check first word in next line */ 2390 if (*skipwhite(line) == NUL) 2391 capcol = 0; 2392 } 2393 2394 line_breakcheck(); 2395 } 2396 2397 vim_free(buf); 2398 return 0; 2399} 2400 2401/* 2402 * For spell checking: concatenate the start of the following line "line" into 2403 * "buf", blanking-out special characters. Copy less then "maxlen" bytes. 2404 * Keep the blanks at the start of the next line, this is used in win_line() 2405 * to skip those bytes if the word was OK. 2406 */ 2407 void 2408spell_cat_line(buf, line, maxlen) 2409 char_u *buf; 2410 char_u *line; 2411 int maxlen; 2412{ 2413 char_u *p; 2414 int n; 2415 2416 p = skipwhite(line); 2417 while (vim_strchr((char_u *)"*#/\"\t", *p) != NULL) 2418 p = skipwhite(p + 1); 2419 2420 if (*p != NUL) 2421 { 2422 /* Only worth concatenating if there is something else than spaces to 2423 * concatenate. */ 2424 n = (int)(p - line) + 1; 2425 if (n < maxlen - 1) 2426 { 2427 vim_memset(buf, ' ', n); 2428 vim_strncpy(buf + n, p, maxlen - 1 - n); 2429 } 2430 } 2431} 2432 2433/* 2434 * Structure used for the cookie argument of do_in_runtimepath(). 2435 */ 2436typedef struct spelload_S 2437{ 2438 char_u sl_lang[MAXWLEN + 1]; /* language name */ 2439 slang_T *sl_slang; /* resulting slang_T struct */ 2440 int sl_nobreak; /* NOBREAK language found */ 2441} spelload_T; 2442 2443/* 2444 * Load word list(s) for "lang" from Vim spell file(s). 2445 * "lang" must be the language without the region: e.g., "en". 2446 */ 2447 static void 2448spell_load_lang(lang) 2449 char_u *lang; 2450{ 2451 char_u fname_enc[85]; 2452 int r; 2453 spelload_T sl; 2454#ifdef FEAT_AUTOCMD 2455 int round; 2456#endif 2457 2458 /* Copy the language name to pass it to spell_load_cb() as a cookie. 2459 * It's truncated when an error is detected. */ 2460 STRCPY(sl.sl_lang, lang); 2461 sl.sl_slang = NULL; 2462 sl.sl_nobreak = FALSE; 2463 2464#ifdef FEAT_AUTOCMD 2465 /* We may retry when no spell file is found for the language, an 2466 * autocommand may load it then. */ 2467 for (round = 1; round <= 2; ++round) 2468#endif 2469 { 2470 /* 2471 * Find the first spell file for "lang" in 'runtimepath' and load it. 2472 */ 2473 vim_snprintf((char *)fname_enc, sizeof(fname_enc) - 5, 2474 "spell/%s.%s.spl", lang, spell_enc()); 2475 r = do_in_runtimepath(fname_enc, FALSE, spell_load_cb, &sl); 2476 2477 if (r == FAIL && *sl.sl_lang != NUL) 2478 { 2479 /* Try loading the ASCII version. */ 2480 vim_snprintf((char *)fname_enc, sizeof(fname_enc) - 5, 2481 "spell/%s.ascii.spl", lang); 2482 r = do_in_runtimepath(fname_enc, FALSE, spell_load_cb, &sl); 2483 2484#ifdef FEAT_AUTOCMD 2485 if (r == FAIL && *sl.sl_lang != NUL && round == 1 2486 && apply_autocmds(EVENT_SPELLFILEMISSING, lang, 2487 curbuf->b_fname, FALSE, curbuf)) 2488 continue; 2489 break; 2490#endif 2491 } 2492#ifdef FEAT_AUTOCMD 2493 break; 2494#endif 2495 } 2496 2497 if (r == FAIL) 2498 { 2499 smsg((char_u *)_("Warning: Cannot find word list \"%s.%s.spl\" or \"%s.ascii.spl\""), 2500 lang, spell_enc(), lang); 2501 } 2502 else if (sl.sl_slang != NULL) 2503 { 2504 /* At least one file was loaded, now load ALL the additions. */ 2505 STRCPY(fname_enc + STRLEN(fname_enc) - 3, "add.spl"); 2506 do_in_runtimepath(fname_enc, TRUE, spell_load_cb, &sl); 2507 } 2508} 2509 2510/* 2511 * Return the encoding used for spell checking: Use 'encoding', except that we 2512 * use "latin1" for "latin9". And limit to 60 characters (just in case). 2513 */ 2514 static char_u * 2515spell_enc() 2516{ 2517 2518#ifdef FEAT_MBYTE 2519 if (STRLEN(p_enc) < 60 && STRCMP(p_enc, "iso-8859-15") != 0) 2520 return p_enc; 2521#endif 2522 return (char_u *)"latin1"; 2523} 2524 2525/* 2526 * Get the name of the .spl file for the internal wordlist into 2527 * "fname[MAXPATHL]". 2528 */ 2529 static void 2530int_wordlist_spl(fname) 2531 char_u *fname; 2532{ 2533 vim_snprintf((char *)fname, MAXPATHL, "%s.%s.spl", 2534 int_wordlist, spell_enc()); 2535} 2536 2537/* 2538 * Allocate a new slang_T for language "lang". "lang" can be NULL. 2539 * Caller must fill "sl_next". 2540 */ 2541 static slang_T * 2542slang_alloc(lang) 2543 char_u *lang; 2544{ 2545 slang_T *lp; 2546 2547 lp = (slang_T *)alloc_clear(sizeof(slang_T)); 2548 if (lp != NULL) 2549 { 2550 if (lang != NULL) 2551 lp->sl_name = vim_strsave(lang); 2552 ga_init2(&lp->sl_rep, sizeof(fromto_T), 10); 2553 ga_init2(&lp->sl_repsal, sizeof(fromto_T), 10); 2554 lp->sl_compmax = MAXWLEN; 2555 lp->sl_compsylmax = MAXWLEN; 2556 hash_init(&lp->sl_wordcount); 2557 } 2558 2559 return lp; 2560} 2561 2562/* 2563 * Free the contents of an slang_T and the structure itself. 2564 */ 2565 static void 2566slang_free(lp) 2567 slang_T *lp; 2568{ 2569 vim_free(lp->sl_name); 2570 vim_free(lp->sl_fname); 2571 slang_clear(lp); 2572 vim_free(lp); 2573} 2574 2575/* 2576 * Clear an slang_T so that the file can be reloaded. 2577 */ 2578 static void 2579slang_clear(lp) 2580 slang_T *lp; 2581{ 2582 garray_T *gap; 2583 fromto_T *ftp; 2584 salitem_T *smp; 2585 int i; 2586 int round; 2587 2588 vim_free(lp->sl_fbyts); 2589 lp->sl_fbyts = NULL; 2590 vim_free(lp->sl_kbyts); 2591 lp->sl_kbyts = NULL; 2592 vim_free(lp->sl_pbyts); 2593 lp->sl_pbyts = NULL; 2594 2595 vim_free(lp->sl_fidxs); 2596 lp->sl_fidxs = NULL; 2597 vim_free(lp->sl_kidxs); 2598 lp->sl_kidxs = NULL; 2599 vim_free(lp->sl_pidxs); 2600 lp->sl_pidxs = NULL; 2601 2602 for (round = 1; round <= 2; ++round) 2603 { 2604 gap = round == 1 ? &lp->sl_rep : &lp->sl_repsal; 2605 while (gap->ga_len > 0) 2606 { 2607 ftp = &((fromto_T *)gap->ga_data)[--gap->ga_len]; 2608 vim_free(ftp->ft_from); 2609 vim_free(ftp->ft_to); 2610 } 2611 ga_clear(gap); 2612 } 2613 2614 gap = &lp->sl_sal; 2615 if (lp->sl_sofo) 2616 { 2617 /* "ga_len" is set to 1 without adding an item for latin1 */ 2618 if (gap->ga_data != NULL) 2619 /* SOFOFROM and SOFOTO items: free lists of wide characters. */ 2620 for (i = 0; i < gap->ga_len; ++i) 2621 vim_free(((int **)gap->ga_data)[i]); 2622 } 2623 else 2624 /* SAL items: free salitem_T items */ 2625 while (gap->ga_len > 0) 2626 { 2627 smp = &((salitem_T *)gap->ga_data)[--gap->ga_len]; 2628 vim_free(smp->sm_lead); 2629 /* Don't free sm_oneof and sm_rules, they point into sm_lead. */ 2630 vim_free(smp->sm_to); 2631#ifdef FEAT_MBYTE 2632 vim_free(smp->sm_lead_w); 2633 vim_free(smp->sm_oneof_w); 2634 vim_free(smp->sm_to_w); 2635#endif 2636 } 2637 ga_clear(gap); 2638 2639 for (i = 0; i < lp->sl_prefixcnt; ++i) 2640 vim_free(lp->sl_prefprog[i]); 2641 lp->sl_prefixcnt = 0; 2642 vim_free(lp->sl_prefprog); 2643 lp->sl_prefprog = NULL; 2644 2645 vim_free(lp->sl_info); 2646 lp->sl_info = NULL; 2647 2648 vim_free(lp->sl_midword); 2649 lp->sl_midword = NULL; 2650 2651 vim_free(lp->sl_compprog); 2652 vim_free(lp->sl_comprules); 2653 vim_free(lp->sl_compstartflags); 2654 vim_free(lp->sl_compallflags); 2655 lp->sl_compprog = NULL; 2656 lp->sl_comprules = NULL; 2657 lp->sl_compstartflags = NULL; 2658 lp->sl_compallflags = NULL; 2659 2660 vim_free(lp->sl_syllable); 2661 lp->sl_syllable = NULL; 2662 ga_clear(&lp->sl_syl_items); 2663 2664 ga_clear_strings(&lp->sl_comppat); 2665 2666 hash_clear_all(&lp->sl_wordcount, WC_KEY_OFF); 2667 hash_init(&lp->sl_wordcount); 2668 2669#ifdef FEAT_MBYTE 2670 hash_clear_all(&lp->sl_map_hash, 0); 2671#endif 2672 2673 /* Clear info from .sug file. */ 2674 slang_clear_sug(lp); 2675 2676 lp->sl_compmax = MAXWLEN; 2677 lp->sl_compminlen = 0; 2678 lp->sl_compsylmax = MAXWLEN; 2679 lp->sl_regions[0] = NUL; 2680} 2681 2682/* 2683 * Clear the info from the .sug file in "lp". 2684 */ 2685 static void 2686slang_clear_sug(lp) 2687 slang_T *lp; 2688{ 2689 vim_free(lp->sl_sbyts); 2690 lp->sl_sbyts = NULL; 2691 vim_free(lp->sl_sidxs); 2692 lp->sl_sidxs = NULL; 2693 close_spellbuf(lp->sl_sugbuf); 2694 lp->sl_sugbuf = NULL; 2695 lp->sl_sugloaded = FALSE; 2696 lp->sl_sugtime = 0; 2697} 2698 2699/* 2700 * Load one spell file and store the info into a slang_T. 2701 * Invoked through do_in_runtimepath(). 2702 */ 2703 static void 2704spell_load_cb(fname, cookie) 2705 char_u *fname; 2706 void *cookie; 2707{ 2708 spelload_T *slp = (spelload_T *)cookie; 2709 slang_T *slang; 2710 2711 slang = spell_load_file(fname, slp->sl_lang, NULL, FALSE); 2712 if (slang != NULL) 2713 { 2714 /* When a previously loaded file has NOBREAK also use it for the 2715 * ".add" files. */ 2716 if (slp->sl_nobreak && slang->sl_add) 2717 slang->sl_nobreak = TRUE; 2718 else if (slang->sl_nobreak) 2719 slp->sl_nobreak = TRUE; 2720 2721 slp->sl_slang = slang; 2722 } 2723} 2724 2725/* 2726 * Load one spell file and store the info into a slang_T. 2727 * 2728 * This is invoked in three ways: 2729 * - From spell_load_cb() to load a spell file for the first time. "lang" is 2730 * the language name, "old_lp" is NULL. Will allocate an slang_T. 2731 * - To reload a spell file that was changed. "lang" is NULL and "old_lp" 2732 * points to the existing slang_T. 2733 * - Just after writing a .spl file; it's read back to produce the .sug file. 2734 * "old_lp" is NULL and "lang" is NULL. Will allocate an slang_T. 2735 * 2736 * Returns the slang_T the spell file was loaded into. NULL for error. 2737 */ 2738 static slang_T * 2739spell_load_file(fname, lang, old_lp, silent) 2740 char_u *fname; 2741 char_u *lang; 2742 slang_T *old_lp; 2743 int silent; /* no error if file doesn't exist */ 2744{ 2745 FILE *fd; 2746 char_u buf[VIMSPELLMAGICL]; 2747 char_u *p; 2748 int i; 2749 int n; 2750 int len; 2751 char_u *save_sourcing_name = sourcing_name; 2752 linenr_T save_sourcing_lnum = sourcing_lnum; 2753 slang_T *lp = NULL; 2754 int c = 0; 2755 int res; 2756 2757 fd = mch_fopen((char *)fname, "r"); 2758 if (fd == NULL) 2759 { 2760 if (!silent) 2761 EMSG2(_(e_notopen), fname); 2762 else if (p_verbose > 2) 2763 { 2764 verbose_enter(); 2765 smsg((char_u *)e_notopen, fname); 2766 verbose_leave(); 2767 } 2768 goto endFAIL; 2769 } 2770 if (p_verbose > 2) 2771 { 2772 verbose_enter(); 2773 smsg((char_u *)_("Reading spell file \"%s\""), fname); 2774 verbose_leave(); 2775 } 2776 2777 if (old_lp == NULL) 2778 { 2779 lp = slang_alloc(lang); 2780 if (lp == NULL) 2781 goto endFAIL; 2782 2783 /* Remember the file name, used to reload the file when it's updated. */ 2784 lp->sl_fname = vim_strsave(fname); 2785 if (lp->sl_fname == NULL) 2786 goto endFAIL; 2787 2788 /* Check for .add.spl. */ 2789 lp->sl_add = strstr((char *)gettail(fname), ".add.") != NULL; 2790 } 2791 else 2792 lp = old_lp; 2793 2794 /* Set sourcing_name, so that error messages mention the file name. */ 2795 sourcing_name = fname; 2796 sourcing_lnum = 0; 2797 2798 /* 2799 * <HEADER>: <fileID> 2800 */ 2801 for (i = 0; i < VIMSPELLMAGICL; ++i) 2802 buf[i] = getc(fd); /* <fileID> */ 2803 if (STRNCMP(buf, VIMSPELLMAGIC, VIMSPELLMAGICL) != 0) 2804 { 2805 EMSG(_("E757: This does not look like a spell file")); 2806 goto endFAIL; 2807 } 2808 c = getc(fd); /* <versionnr> */ 2809 if (c < VIMSPELLVERSION) 2810 { 2811 EMSG(_("E771: Old spell file, needs to be updated")); 2812 goto endFAIL; 2813 } 2814 else if (c > VIMSPELLVERSION) 2815 { 2816 EMSG(_("E772: Spell file is for newer version of Vim")); 2817 goto endFAIL; 2818 } 2819 2820 2821 /* 2822 * <SECTIONS>: <section> ... <sectionend> 2823 * <section>: <sectionID> <sectionflags> <sectionlen> (section contents) 2824 */ 2825 for (;;) 2826 { 2827 n = getc(fd); /* <sectionID> or <sectionend> */ 2828 if (n == SN_END) 2829 break; 2830 c = getc(fd); /* <sectionflags> */ 2831 len = get4c(fd); /* <sectionlen> */ 2832 if (len < 0) 2833 goto truncerr; 2834 2835 res = 0; 2836 switch (n) 2837 { 2838 case SN_INFO: 2839 lp->sl_info = read_string(fd, len); /* <infotext> */ 2840 if (lp->sl_info == NULL) 2841 goto endFAIL; 2842 break; 2843 2844 case SN_REGION: 2845 res = read_region_section(fd, lp, len); 2846 break; 2847 2848 case SN_CHARFLAGS: 2849 res = read_charflags_section(fd); 2850 break; 2851 2852 case SN_MIDWORD: 2853 lp->sl_midword = read_string(fd, len); /* <midword> */ 2854 if (lp->sl_midword == NULL) 2855 goto endFAIL; 2856 break; 2857 2858 case SN_PREFCOND: 2859 res = read_prefcond_section(fd, lp); 2860 break; 2861 2862 case SN_REP: 2863 res = read_rep_section(fd, &lp->sl_rep, lp->sl_rep_first); 2864 break; 2865 2866 case SN_REPSAL: 2867 res = read_rep_section(fd, &lp->sl_repsal, lp->sl_repsal_first); 2868 break; 2869 2870 case SN_SAL: 2871 res = read_sal_section(fd, lp); 2872 break; 2873 2874 case SN_SOFO: 2875 res = read_sofo_section(fd, lp); 2876 break; 2877 2878 case SN_MAP: 2879 p = read_string(fd, len); /* <mapstr> */ 2880 if (p == NULL) 2881 goto endFAIL; 2882 set_map_str(lp, p); 2883 vim_free(p); 2884 break; 2885 2886 case SN_WORDS: 2887 res = read_words_section(fd, lp, len); 2888 break; 2889 2890 case SN_SUGFILE: 2891 lp->sl_sugtime = get8ctime(fd); /* <timestamp> */ 2892 break; 2893 2894 case SN_NOSPLITSUGS: 2895 lp->sl_nosplitsugs = TRUE; /* <timestamp> */ 2896 break; 2897 2898 case SN_COMPOUND: 2899 res = read_compound(fd, lp, len); 2900 break; 2901 2902 case SN_NOBREAK: 2903 lp->sl_nobreak = TRUE; 2904 break; 2905 2906 case SN_SYLLABLE: 2907 lp->sl_syllable = read_string(fd, len); /* <syllable> */ 2908 if (lp->sl_syllable == NULL) 2909 goto endFAIL; 2910 if (init_syl_tab(lp) == FAIL) 2911 goto endFAIL; 2912 break; 2913 2914 default: 2915 /* Unsupported section. When it's required give an error 2916 * message. When it's not required skip the contents. */ 2917 if (c & SNF_REQUIRED) 2918 { 2919 EMSG(_("E770: Unsupported section in spell file")); 2920 goto endFAIL; 2921 } 2922 while (--len >= 0) 2923 if (getc(fd) < 0) 2924 goto truncerr; 2925 break; 2926 } 2927someerror: 2928 if (res == SP_FORMERROR) 2929 { 2930 EMSG(_(e_format)); 2931 goto endFAIL; 2932 } 2933 if (res == SP_TRUNCERROR) 2934 { 2935truncerr: 2936 EMSG(_(e_spell_trunc)); 2937 goto endFAIL; 2938 } 2939 if (res == SP_OTHERERROR) 2940 goto endFAIL; 2941 } 2942 2943 /* <LWORDTREE> */ 2944 res = spell_read_tree(fd, &lp->sl_fbyts, &lp->sl_fidxs, FALSE, 0); 2945 if (res != 0) 2946 goto someerror; 2947 2948 /* <KWORDTREE> */ 2949 res = spell_read_tree(fd, &lp->sl_kbyts, &lp->sl_kidxs, FALSE, 0); 2950 if (res != 0) 2951 goto someerror; 2952 2953 /* <PREFIXTREE> */ 2954 res = spell_read_tree(fd, &lp->sl_pbyts, &lp->sl_pidxs, TRUE, 2955 lp->sl_prefixcnt); 2956 if (res != 0) 2957 goto someerror; 2958 2959 /* For a new file link it in the list of spell files. */ 2960 if (old_lp == NULL && lang != NULL) 2961 { 2962 lp->sl_next = first_lang; 2963 first_lang = lp; 2964 } 2965 2966 goto endOK; 2967 2968endFAIL: 2969 if (lang != NULL) 2970 /* truncating the name signals the error to spell_load_lang() */ 2971 *lang = NUL; 2972 if (lp != NULL && old_lp == NULL) 2973 slang_free(lp); 2974 lp = NULL; 2975 2976endOK: 2977 if (fd != NULL) 2978 fclose(fd); 2979 sourcing_name = save_sourcing_name; 2980 sourcing_lnum = save_sourcing_lnum; 2981 2982 return lp; 2983} 2984 2985/* 2986 * Read a length field from "fd" in "cnt_bytes" bytes. 2987 * Allocate memory, read the string into it and add a NUL at the end. 2988 * Returns NULL when the count is zero. 2989 * Sets "*cntp" to SP_*ERROR when there is an error, length of the result 2990 * otherwise. 2991 */ 2992 static char_u * 2993read_cnt_string(fd, cnt_bytes, cntp) 2994 FILE *fd; 2995 int cnt_bytes; 2996 int *cntp; 2997{ 2998 int cnt = 0; 2999 int i; 3000 char_u *str; 3001 3002 /* read the length bytes, MSB first */ 3003 for (i = 0; i < cnt_bytes; ++i) 3004 cnt = (cnt << 8) + getc(fd); 3005 if (cnt < 0) 3006 { 3007 *cntp = SP_TRUNCERROR; 3008 return NULL; 3009 } 3010 *cntp = cnt; 3011 if (cnt == 0) 3012 return NULL; /* nothing to read, return NULL */ 3013 3014 str = read_string(fd, cnt); 3015 if (str == NULL) 3016 *cntp = SP_OTHERERROR; 3017 return str; 3018} 3019 3020/* 3021 * Read SN_REGION: <regionname> ... 3022 * Return SP_*ERROR flags. 3023 */ 3024 static int 3025read_region_section(fd, lp, len) 3026 FILE *fd; 3027 slang_T *lp; 3028 int len; 3029{ 3030 int i; 3031 3032 if (len > 16) 3033 return SP_FORMERROR; 3034 for (i = 0; i < len; ++i) 3035 lp->sl_regions[i] = getc(fd); /* <regionname> */ 3036 lp->sl_regions[len] = NUL; 3037 return 0; 3038} 3039 3040/* 3041 * Read SN_CHARFLAGS section: <charflagslen> <charflags> 3042 * <folcharslen> <folchars> 3043 * Return SP_*ERROR flags. 3044 */ 3045 static int 3046read_charflags_section(fd) 3047 FILE *fd; 3048{ 3049 char_u *flags; 3050 char_u *fol; 3051 int flagslen, follen; 3052 3053 /* <charflagslen> <charflags> */ 3054 flags = read_cnt_string(fd, 1, &flagslen); 3055 if (flagslen < 0) 3056 return flagslen; 3057 3058 /* <folcharslen> <folchars> */ 3059 fol = read_cnt_string(fd, 2, &follen); 3060 if (follen < 0) 3061 { 3062 vim_free(flags); 3063 return follen; 3064 } 3065 3066 /* Set the word-char flags and fill SPELL_ISUPPER() table. */ 3067 if (flags != NULL && fol != NULL) 3068 set_spell_charflags(flags, flagslen, fol); 3069 3070 vim_free(flags); 3071 vim_free(fol); 3072 3073 /* When <charflagslen> is zero then <fcharlen> must also be zero. */ 3074 if ((flags == NULL) != (fol == NULL)) 3075 return SP_FORMERROR; 3076 return 0; 3077} 3078 3079/* 3080 * Read SN_PREFCOND section. 3081 * Return SP_*ERROR flags. 3082 */ 3083 static int 3084read_prefcond_section(fd, lp) 3085 FILE *fd; 3086 slang_T *lp; 3087{ 3088 int cnt; 3089 int i; 3090 int n; 3091 char_u *p; 3092 char_u buf[MAXWLEN + 1]; 3093 3094 /* <prefcondcnt> <prefcond> ... */ 3095 cnt = get2c(fd); /* <prefcondcnt> */ 3096 if (cnt <= 0) 3097 return SP_FORMERROR; 3098 3099 lp->sl_prefprog = (regprog_T **)alloc_clear( 3100 (unsigned)sizeof(regprog_T *) * cnt); 3101 if (lp->sl_prefprog == NULL) 3102 return SP_OTHERERROR; 3103 lp->sl_prefixcnt = cnt; 3104 3105 for (i = 0; i < cnt; ++i) 3106 { 3107 /* <prefcond> : <condlen> <condstr> */ 3108 n = getc(fd); /* <condlen> */ 3109 if (n < 0 || n >= MAXWLEN) 3110 return SP_FORMERROR; 3111 3112 /* When <condlen> is zero we have an empty condition. Otherwise 3113 * compile the regexp program used to check for the condition. */ 3114 if (n > 0) 3115 { 3116 buf[0] = '^'; /* always match at one position only */ 3117 p = buf + 1; 3118 while (n-- > 0) 3119 *p++ = getc(fd); /* <condstr> */ 3120 *p = NUL; 3121 lp->sl_prefprog[i] = vim_regcomp(buf, RE_MAGIC + RE_STRING); 3122 } 3123 } 3124 return 0; 3125} 3126 3127/* 3128 * Read REP or REPSAL items section from "fd": <repcount> <rep> ... 3129 * Return SP_*ERROR flags. 3130 */ 3131 static int 3132read_rep_section(fd, gap, first) 3133 FILE *fd; 3134 garray_T *gap; 3135 short *first; 3136{ 3137 int cnt; 3138 fromto_T *ftp; 3139 int i; 3140 3141 cnt = get2c(fd); /* <repcount> */ 3142 if (cnt < 0) 3143 return SP_TRUNCERROR; 3144 3145 if (ga_grow(gap, cnt) == FAIL) 3146 return SP_OTHERERROR; 3147 3148 /* <rep> : <repfromlen> <repfrom> <reptolen> <repto> */ 3149 for (; gap->ga_len < cnt; ++gap->ga_len) 3150 { 3151 ftp = &((fromto_T *)gap->ga_data)[gap->ga_len]; 3152 ftp->ft_from = read_cnt_string(fd, 1, &i); 3153 if (i < 0) 3154 return i; 3155 if (i == 0) 3156 return SP_FORMERROR; 3157 ftp->ft_to = read_cnt_string(fd, 1, &i); 3158 if (i <= 0) 3159 { 3160 vim_free(ftp->ft_from); 3161 if (i < 0) 3162 return i; 3163 return SP_FORMERROR; 3164 } 3165 } 3166 3167 /* Fill the first-index table. */ 3168 for (i = 0; i < 256; ++i) 3169 first[i] = -1; 3170 for (i = 0; i < gap->ga_len; ++i) 3171 { 3172 ftp = &((fromto_T *)gap->ga_data)[i]; 3173 if (first[*ftp->ft_from] == -1) 3174 first[*ftp->ft_from] = i; 3175 } 3176 return 0; 3177} 3178 3179/* 3180 * Read SN_SAL section: <salflags> <salcount> <sal> ... 3181 * Return SP_*ERROR flags. 3182 */ 3183 static int 3184read_sal_section(fd, slang) 3185 FILE *fd; 3186 slang_T *slang; 3187{ 3188 int i; 3189 int cnt; 3190 garray_T *gap; 3191 salitem_T *smp; 3192 int ccnt; 3193 char_u *p; 3194 int c = NUL; 3195 3196 slang->sl_sofo = FALSE; 3197 3198 i = getc(fd); /* <salflags> */ 3199 if (i & SAL_F0LLOWUP) 3200 slang->sl_followup = TRUE; 3201 if (i & SAL_COLLAPSE) 3202 slang->sl_collapse = TRUE; 3203 if (i & SAL_REM_ACCENTS) 3204 slang->sl_rem_accents = TRUE; 3205 3206 cnt = get2c(fd); /* <salcount> */ 3207 if (cnt < 0) 3208 return SP_TRUNCERROR; 3209 3210 gap = &slang->sl_sal; 3211 ga_init2(gap, sizeof(salitem_T), 10); 3212 if (ga_grow(gap, cnt + 1) == FAIL) 3213 return SP_OTHERERROR; 3214 3215 /* <sal> : <salfromlen> <salfrom> <saltolen> <salto> */ 3216 for (; gap->ga_len < cnt; ++gap->ga_len) 3217 { 3218 smp = &((salitem_T *)gap->ga_data)[gap->ga_len]; 3219 ccnt = getc(fd); /* <salfromlen> */ 3220 if (ccnt < 0) 3221 return SP_TRUNCERROR; 3222 if ((p = alloc(ccnt + 2)) == NULL) 3223 return SP_OTHERERROR; 3224 smp->sm_lead = p; 3225 3226 /* Read up to the first special char into sm_lead. */ 3227 for (i = 0; i < ccnt; ++i) 3228 { 3229 c = getc(fd); /* <salfrom> */ 3230 if (vim_strchr((char_u *)"0123456789(-<^$", c) != NULL) 3231 break; 3232 *p++ = c; 3233 } 3234 smp->sm_leadlen = (int)(p - smp->sm_lead); 3235 *p++ = NUL; 3236 3237 /* Put (abc) chars in sm_oneof, if any. */ 3238 if (c == '(') 3239 { 3240 smp->sm_oneof = p; 3241 for (++i; i < ccnt; ++i) 3242 { 3243 c = getc(fd); /* <salfrom> */ 3244 if (c == ')') 3245 break; 3246 *p++ = c; 3247 } 3248 *p++ = NUL; 3249 if (++i < ccnt) 3250 c = getc(fd); 3251 } 3252 else 3253 smp->sm_oneof = NULL; 3254 3255 /* Any following chars go in sm_rules. */ 3256 smp->sm_rules = p; 3257 if (i < ccnt) 3258 /* store the char we got while checking for end of sm_lead */ 3259 *p++ = c; 3260 for (++i; i < ccnt; ++i) 3261 *p++ = getc(fd); /* <salfrom> */ 3262 *p++ = NUL; 3263 3264 /* <saltolen> <salto> */ 3265 smp->sm_to = read_cnt_string(fd, 1, &ccnt); 3266 if (ccnt < 0) 3267 { 3268 vim_free(smp->sm_lead); 3269 return ccnt; 3270 } 3271 3272#ifdef FEAT_MBYTE 3273 if (has_mbyte) 3274 { 3275 /* convert the multi-byte strings to wide char strings */ 3276 smp->sm_lead_w = mb_str2wide(smp->sm_lead); 3277 smp->sm_leadlen = mb_charlen(smp->sm_lead); 3278 if (smp->sm_oneof == NULL) 3279 smp->sm_oneof_w = NULL; 3280 else 3281 smp->sm_oneof_w = mb_str2wide(smp->sm_oneof); 3282 if (smp->sm_to == NULL) 3283 smp->sm_to_w = NULL; 3284 else 3285 smp->sm_to_w = mb_str2wide(smp->sm_to); 3286 if (smp->sm_lead_w == NULL 3287 || (smp->sm_oneof_w == NULL && smp->sm_oneof != NULL) 3288 || (smp->sm_to_w == NULL && smp->sm_to != NULL)) 3289 { 3290 vim_free(smp->sm_lead); 3291 vim_free(smp->sm_to); 3292 vim_free(smp->sm_lead_w); 3293 vim_free(smp->sm_oneof_w); 3294 vim_free(smp->sm_to_w); 3295 return SP_OTHERERROR; 3296 } 3297 } 3298#endif 3299 } 3300 3301 if (gap->ga_len > 0) 3302 { 3303 /* Add one extra entry to mark the end with an empty sm_lead. Avoids 3304 * that we need to check the index every time. */ 3305 smp = &((salitem_T *)gap->ga_data)[gap->ga_len]; 3306 if ((p = alloc(1)) == NULL) 3307 return SP_OTHERERROR; 3308 p[0] = NUL; 3309 smp->sm_lead = p; 3310 smp->sm_leadlen = 0; 3311 smp->sm_oneof = NULL; 3312 smp->sm_rules = p; 3313 smp->sm_to = NULL; 3314#ifdef FEAT_MBYTE 3315 if (has_mbyte) 3316 { 3317 smp->sm_lead_w = mb_str2wide(smp->sm_lead); 3318 smp->sm_leadlen = 0; 3319 smp->sm_oneof_w = NULL; 3320 smp->sm_to_w = NULL; 3321 } 3322#endif 3323 ++gap->ga_len; 3324 } 3325 3326 /* Fill the first-index table. */ 3327 set_sal_first(slang); 3328 3329 return 0; 3330} 3331 3332/* 3333 * Read SN_WORDS: <word> ... 3334 * Return SP_*ERROR flags. 3335 */ 3336 static int 3337read_words_section(fd, lp, len) 3338 FILE *fd; 3339 slang_T *lp; 3340 int len; 3341{ 3342 int done = 0; 3343 int i; 3344 int c; 3345 char_u word[MAXWLEN]; 3346 3347 while (done < len) 3348 { 3349 /* Read one word at a time. */ 3350 for (i = 0; ; ++i) 3351 { 3352 c = getc(fd); 3353 if (c == EOF) 3354 return SP_TRUNCERROR; 3355 word[i] = c; 3356 if (word[i] == NUL) 3357 break; 3358 if (i == MAXWLEN - 1) 3359 return SP_FORMERROR; 3360 } 3361 3362 /* Init the count to 10. */ 3363 count_common_word(lp, word, -1, 10); 3364 done += i + 1; 3365 } 3366 return 0; 3367} 3368 3369/* 3370 * Add a word to the hashtable of common words. 3371 * If it's already there then the counter is increased. 3372 */ 3373 static void 3374count_common_word(lp, word, len, count) 3375 slang_T *lp; 3376 char_u *word; 3377 int len; /* word length, -1 for upto NUL */ 3378 int count; /* 1 to count once, 10 to init */ 3379{ 3380 hash_T hash; 3381 hashitem_T *hi; 3382 wordcount_T *wc; 3383 char_u buf[MAXWLEN]; 3384 char_u *p; 3385 3386 if (len == -1) 3387 p = word; 3388 else 3389 { 3390 vim_strncpy(buf, word, len); 3391 p = buf; 3392 } 3393 3394 hash = hash_hash(p); 3395 hi = hash_lookup(&lp->sl_wordcount, p, hash); 3396 if (HASHITEM_EMPTY(hi)) 3397 { 3398 wc = (wordcount_T *)alloc((unsigned)(sizeof(wordcount_T) + STRLEN(p))); 3399 if (wc == NULL) 3400 return; 3401 STRCPY(wc->wc_word, p); 3402 wc->wc_count = count; 3403 hash_add_item(&lp->sl_wordcount, hi, wc->wc_word, hash); 3404 } 3405 else 3406 { 3407 wc = HI2WC(hi); 3408 if ((wc->wc_count += count) < (unsigned)count) /* check for overflow */ 3409 wc->wc_count = MAXWORDCOUNT; 3410 } 3411} 3412 3413/* 3414 * Adjust the score of common words. 3415 */ 3416 static int 3417score_wordcount_adj(slang, score, word, split) 3418 slang_T *slang; 3419 int score; 3420 char_u *word; 3421 int split; /* word was split, less bonus */ 3422{ 3423 hashitem_T *hi; 3424 wordcount_T *wc; 3425 int bonus; 3426 int newscore; 3427 3428 hi = hash_find(&slang->sl_wordcount, word); 3429 if (!HASHITEM_EMPTY(hi)) 3430 { 3431 wc = HI2WC(hi); 3432 if (wc->wc_count < SCORE_THRES2) 3433 bonus = SCORE_COMMON1; 3434 else if (wc->wc_count < SCORE_THRES3) 3435 bonus = SCORE_COMMON2; 3436 else 3437 bonus = SCORE_COMMON3; 3438 if (split) 3439 newscore = score - bonus / 2; 3440 else 3441 newscore = score - bonus; 3442 if (newscore < 0) 3443 return 0; 3444 return newscore; 3445 } 3446 return score; 3447} 3448 3449/* 3450 * SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto> 3451 * Return SP_*ERROR flags. 3452 */ 3453 static int 3454read_sofo_section(fd, slang) 3455 FILE *fd; 3456 slang_T *slang; 3457{ 3458 int cnt; 3459 char_u *from, *to; 3460 int res; 3461 3462 slang->sl_sofo = TRUE; 3463 3464 /* <sofofromlen> <sofofrom> */ 3465 from = read_cnt_string(fd, 2, &cnt); 3466 if (cnt < 0) 3467 return cnt; 3468 3469 /* <sofotolen> <sofoto> */ 3470 to = read_cnt_string(fd, 2, &cnt); 3471 if (cnt < 0) 3472 { 3473 vim_free(from); 3474 return cnt; 3475 } 3476 3477 /* Store the info in slang->sl_sal and/or slang->sl_sal_first. */ 3478 if (from != NULL && to != NULL) 3479 res = set_sofo(slang, from, to); 3480 else if (from != NULL || to != NULL) 3481 res = SP_FORMERROR; /* only one of two strings is an error */ 3482 else 3483 res = 0; 3484 3485 vim_free(from); 3486 vim_free(to); 3487 return res; 3488} 3489 3490/* 3491 * Read the compound section from the .spl file: 3492 * <compmax> <compminlen> <compsylmax> <compoptions> <compflags> 3493 * Returns SP_*ERROR flags. 3494 */ 3495 static int 3496read_compound(fd, slang, len) 3497 FILE *fd; 3498 slang_T *slang; 3499 int len; 3500{ 3501 int todo = len; 3502 int c; 3503 int atstart; 3504 char_u *pat; 3505 char_u *pp; 3506 char_u *cp; 3507 char_u *ap; 3508 char_u *crp; 3509 int cnt; 3510 garray_T *gap; 3511 3512 if (todo < 2) 3513 return SP_FORMERROR; /* need at least two bytes */ 3514 3515 --todo; 3516 c = getc(fd); /* <compmax> */ 3517 if (c < 2) 3518 c = MAXWLEN; 3519 slang->sl_compmax = c; 3520 3521 --todo; 3522 c = getc(fd); /* <compminlen> */ 3523 if (c < 1) 3524 c = 0; 3525 slang->sl_compminlen = c; 3526 3527 --todo; 3528 c = getc(fd); /* <compsylmax> */ 3529 if (c < 1) 3530 c = MAXWLEN; 3531 slang->sl_compsylmax = c; 3532 3533 c = getc(fd); /* <compoptions> */ 3534 if (c != 0) 3535 ungetc(c, fd); /* be backwards compatible with Vim 7.0b */ 3536 else 3537 { 3538 --todo; 3539 c = getc(fd); /* only use the lower byte for now */ 3540 --todo; 3541 slang->sl_compoptions = c; 3542 3543 gap = &slang->sl_comppat; 3544 c = get2c(fd); /* <comppatcount> */ 3545 todo -= 2; 3546 ga_init2(gap, sizeof(char_u *), c); 3547 if (ga_grow(gap, c) == OK) 3548 while (--c >= 0) 3549 { 3550 ((char_u **)(gap->ga_data))[gap->ga_len++] = 3551 read_cnt_string(fd, 1, &cnt); 3552 /* <comppatlen> <comppattext> */ 3553 if (cnt < 0) 3554 return cnt; 3555 todo -= cnt + 1; 3556 } 3557 } 3558 if (todo < 0) 3559 return SP_FORMERROR; 3560 3561 /* Turn the COMPOUNDRULE items into a regexp pattern: 3562 * "a[bc]/a*b+" -> "^\(a[bc]\|a*b\+\)$". 3563 * Inserting backslashes may double the length, "^\(\)$<Nul>" is 7 bytes. 3564 * Conversion to utf-8 may double the size. */ 3565 c = todo * 2 + 7; 3566#ifdef FEAT_MBYTE 3567 if (enc_utf8) 3568 c += todo * 2; 3569#endif 3570 pat = alloc((unsigned)c); 3571 if (pat == NULL) 3572 return SP_OTHERERROR; 3573 3574 /* We also need a list of all flags that can appear at the start and one 3575 * for all flags. */ 3576 cp = alloc(todo + 1); 3577 if (cp == NULL) 3578 { 3579 vim_free(pat); 3580 return SP_OTHERERROR; 3581 } 3582 slang->sl_compstartflags = cp; 3583 *cp = NUL; 3584 3585 ap = alloc(todo + 1); 3586 if (ap == NULL) 3587 { 3588 vim_free(pat); 3589 return SP_OTHERERROR; 3590 } 3591 slang->sl_compallflags = ap; 3592 *ap = NUL; 3593 3594 /* And a list of all patterns in their original form, for checking whether 3595 * compounding may work in match_compoundrule(). This is freed when we 3596 * encounter a wildcard, the check doesn't work then. */ 3597 crp = alloc(todo + 1); 3598 slang->sl_comprules = crp; 3599 3600 pp = pat; 3601 *pp++ = '^'; 3602 *pp++ = '\\'; 3603 *pp++ = '('; 3604 3605 atstart = 1; 3606 while (todo-- > 0) 3607 { 3608 c = getc(fd); /* <compflags> */ 3609 if (c == EOF) 3610 { 3611 vim_free(pat); 3612 return SP_TRUNCERROR; 3613 } 3614 3615 /* Add all flags to "sl_compallflags". */ 3616 if (vim_strchr((char_u *)"+*[]/", c) == NULL 3617 && !byte_in_str(slang->sl_compallflags, c)) 3618 { 3619 *ap++ = c; 3620 *ap = NUL; 3621 } 3622 3623 if (atstart != 0) 3624 { 3625 /* At start of item: copy flags to "sl_compstartflags". For a 3626 * [abc] item set "atstart" to 2 and copy up to the ']'. */ 3627 if (c == '[') 3628 atstart = 2; 3629 else if (c == ']') 3630 atstart = 0; 3631 else 3632 { 3633 if (!byte_in_str(slang->sl_compstartflags, c)) 3634 { 3635 *cp++ = c; 3636 *cp = NUL; 3637 } 3638 if (atstart == 1) 3639 atstart = 0; 3640 } 3641 } 3642 3643 /* Copy flag to "sl_comprules", unless we run into a wildcard. */ 3644 if (crp != NULL) 3645 { 3646 if (c == '+' || c == '*') 3647 { 3648 vim_free(slang->sl_comprules); 3649 slang->sl_comprules = NULL; 3650 crp = NULL; 3651 } 3652 else 3653 *crp++ = c; 3654 } 3655 3656 if (c == '/') /* slash separates two items */ 3657 { 3658 *pp++ = '\\'; 3659 *pp++ = '|'; 3660 atstart = 1; 3661 } 3662 else /* normal char, "[abc]" and '*' are copied as-is */ 3663 { 3664 if (c == '+' || c == '~') 3665 *pp++ = '\\'; /* "a+" becomes "a\+" */ 3666#ifdef FEAT_MBYTE 3667 if (enc_utf8) 3668 pp += mb_char2bytes(c, pp); 3669 else 3670#endif 3671 *pp++ = c; 3672 } 3673 } 3674 3675 *pp++ = '\\'; 3676 *pp++ = ')'; 3677 *pp++ = '$'; 3678 *pp = NUL; 3679 3680 if (crp != NULL) 3681 *crp = NUL; 3682 3683 slang->sl_compprog = vim_regcomp(pat, RE_MAGIC + RE_STRING + RE_STRICT); 3684 vim_free(pat); 3685 if (slang->sl_compprog == NULL) 3686 return SP_FORMERROR; 3687 3688 return 0; 3689} 3690 3691/* 3692 * Return TRUE if byte "n" appears in "str". 3693 * Like strchr() but independent of locale. 3694 */ 3695 static int 3696byte_in_str(str, n) 3697 char_u *str; 3698 int n; 3699{ 3700 char_u *p; 3701 3702 for (p = str; *p != NUL; ++p) 3703 if (*p == n) 3704 return TRUE; 3705 return FALSE; 3706} 3707 3708#define SY_MAXLEN 30 3709typedef struct syl_item_S 3710{ 3711 char_u sy_chars[SY_MAXLEN]; /* the sequence of chars */ 3712 int sy_len; 3713} syl_item_T; 3714 3715/* 3716 * Truncate "slang->sl_syllable" at the first slash and put the following items 3717 * in "slang->sl_syl_items". 3718 */ 3719 static int 3720init_syl_tab(slang) 3721 slang_T *slang; 3722{ 3723 char_u *p; 3724 char_u *s; 3725 int l; 3726 syl_item_T *syl; 3727 3728 ga_init2(&slang->sl_syl_items, sizeof(syl_item_T), 4); 3729 p = vim_strchr(slang->sl_syllable, '/'); 3730 while (p != NULL) 3731 { 3732 *p++ = NUL; 3733 if (*p == NUL) /* trailing slash */ 3734 break; 3735 s = p; 3736 p = vim_strchr(p, '/'); 3737 if (p == NULL) 3738 l = (int)STRLEN(s); 3739 else 3740 l = (int)(p - s); 3741 if (l >= SY_MAXLEN) 3742 return SP_FORMERROR; 3743 if (ga_grow(&slang->sl_syl_items, 1) == FAIL) 3744 return SP_OTHERERROR; 3745 syl = ((syl_item_T *)slang->sl_syl_items.ga_data) 3746 + slang->sl_syl_items.ga_len++; 3747 vim_strncpy(syl->sy_chars, s, l); 3748 syl->sy_len = l; 3749 } 3750 return OK; 3751} 3752 3753/* 3754 * Count the number of syllables in "word". 3755 * When "word" contains spaces the syllables after the last space are counted. 3756 * Returns zero if syllables are not defines. 3757 */ 3758 static int 3759count_syllables(slang, word) 3760 slang_T *slang; 3761 char_u *word; 3762{ 3763 int cnt = 0; 3764 int skip = FALSE; 3765 char_u *p; 3766 int len; 3767 int i; 3768 syl_item_T *syl; 3769 int c; 3770 3771 if (slang->sl_syllable == NULL) 3772 return 0; 3773 3774 for (p = word; *p != NUL; p += len) 3775 { 3776 /* When running into a space reset counter. */ 3777 if (*p == ' ') 3778 { 3779 len = 1; 3780 cnt = 0; 3781 continue; 3782 } 3783 3784 /* Find longest match of syllable items. */ 3785 len = 0; 3786 for (i = 0; i < slang->sl_syl_items.ga_len; ++i) 3787 { 3788 syl = ((syl_item_T *)slang->sl_syl_items.ga_data) + i; 3789 if (syl->sy_len > len 3790 && STRNCMP(p, syl->sy_chars, syl->sy_len) == 0) 3791 len = syl->sy_len; 3792 } 3793 if (len != 0) /* found a match, count syllable */ 3794 { 3795 ++cnt; 3796 skip = FALSE; 3797 } 3798 else 3799 { 3800 /* No recognized syllable item, at least a syllable char then? */ 3801#ifdef FEAT_MBYTE 3802 c = mb_ptr2char(p); 3803 len = (*mb_ptr2len)(p); 3804#else 3805 c = *p; 3806 len = 1; 3807#endif 3808 if (vim_strchr(slang->sl_syllable, c) == NULL) 3809 skip = FALSE; /* No, search for next syllable */ 3810 else if (!skip) 3811 { 3812 ++cnt; /* Yes, count it */ 3813 skip = TRUE; /* don't count following syllable chars */ 3814 } 3815 } 3816 } 3817 return cnt; 3818} 3819 3820/* 3821 * Set the SOFOFROM and SOFOTO items in language "lp". 3822 * Returns SP_*ERROR flags when there is something wrong. 3823 */ 3824 static int 3825set_sofo(lp, from, to) 3826 slang_T *lp; 3827 char_u *from; 3828 char_u *to; 3829{ 3830 int i; 3831 3832#ifdef FEAT_MBYTE 3833 garray_T *gap; 3834 char_u *s; 3835 char_u *p; 3836 int c; 3837 int *inp; 3838 3839 if (has_mbyte) 3840 { 3841 /* Use "sl_sal" as an array with 256 pointers to a list of wide 3842 * characters. The index is the low byte of the character. 3843 * The list contains from-to pairs with a terminating NUL. 3844 * sl_sal_first[] is used for latin1 "from" characters. */ 3845 gap = &lp->sl_sal; 3846 ga_init2(gap, sizeof(int *), 1); 3847 if (ga_grow(gap, 256) == FAIL) 3848 return SP_OTHERERROR; 3849 vim_memset(gap->ga_data, 0, sizeof(int *) * 256); 3850 gap->ga_len = 256; 3851 3852 /* First count the number of items for each list. Temporarily use 3853 * sl_sal_first[] for this. */ 3854 for (p = from, s = to; *p != NUL && *s != NUL; ) 3855 { 3856 c = mb_cptr2char_adv(&p); 3857 mb_cptr_adv(s); 3858 if (c >= 256) 3859 ++lp->sl_sal_first[c & 0xff]; 3860 } 3861 if (*p != NUL || *s != NUL) /* lengths differ */ 3862 return SP_FORMERROR; 3863 3864 /* Allocate the lists. */ 3865 for (i = 0; i < 256; ++i) 3866 if (lp->sl_sal_first[i] > 0) 3867 { 3868 p = alloc(sizeof(int) * (lp->sl_sal_first[i] * 2 + 1)); 3869 if (p == NULL) 3870 return SP_OTHERERROR; 3871 ((int **)gap->ga_data)[i] = (int *)p; 3872 *(int *)p = 0; 3873 } 3874 3875 /* Put the characters up to 255 in sl_sal_first[] the rest in a sl_sal 3876 * list. */ 3877 vim_memset(lp->sl_sal_first, 0, sizeof(salfirst_T) * 256); 3878 for (p = from, s = to; *p != NUL && *s != NUL; ) 3879 { 3880 c = mb_cptr2char_adv(&p); 3881 i = mb_cptr2char_adv(&s); 3882 if (c >= 256) 3883 { 3884 /* Append the from-to chars at the end of the list with 3885 * the low byte. */ 3886 inp = ((int **)gap->ga_data)[c & 0xff]; 3887 while (*inp != 0) 3888 ++inp; 3889 *inp++ = c; /* from char */ 3890 *inp++ = i; /* to char */ 3891 *inp++ = NUL; /* NUL at the end */ 3892 } 3893 else 3894 /* mapping byte to char is done in sl_sal_first[] */ 3895 lp->sl_sal_first[c] = i; 3896 } 3897 } 3898 else 3899#endif 3900 { 3901 /* mapping bytes to bytes is done in sl_sal_first[] */ 3902 if (STRLEN(from) != STRLEN(to)) 3903 return SP_FORMERROR; 3904 3905 for (i = 0; to[i] != NUL; ++i) 3906 lp->sl_sal_first[from[i]] = to[i]; 3907 lp->sl_sal.ga_len = 1; /* indicates we have soundfolding */ 3908 } 3909 3910 return 0; 3911} 3912 3913/* 3914 * Fill the first-index table for "lp". 3915 */ 3916 static void 3917set_sal_first(lp) 3918 slang_T *lp; 3919{ 3920 salfirst_T *sfirst; 3921 int i; 3922 salitem_T *smp; 3923 int c; 3924 garray_T *gap = &lp->sl_sal; 3925 3926 sfirst = lp->sl_sal_first; 3927 for (i = 0; i < 256; ++i) 3928 sfirst[i] = -1; 3929 smp = (salitem_T *)gap->ga_data; 3930 for (i = 0; i < gap->ga_len; ++i) 3931 { 3932#ifdef FEAT_MBYTE 3933 if (has_mbyte) 3934 /* Use the lowest byte of the first character. For latin1 it's 3935 * the character, for other encodings it should differ for most 3936 * characters. */ 3937 c = *smp[i].sm_lead_w & 0xff; 3938 else 3939#endif 3940 c = *smp[i].sm_lead; 3941 if (sfirst[c] == -1) 3942 { 3943 sfirst[c] = i; 3944#ifdef FEAT_MBYTE 3945 if (has_mbyte) 3946 { 3947 int n; 3948 3949 /* Make sure all entries with this byte are following each 3950 * other. Move the ones that are in the wrong position. Do 3951 * keep the same ordering! */ 3952 while (i + 1 < gap->ga_len 3953 && (*smp[i + 1].sm_lead_w & 0xff) == c) 3954 /* Skip over entry with same index byte. */ 3955 ++i; 3956 3957 for (n = 1; i + n < gap->ga_len; ++n) 3958 if ((*smp[i + n].sm_lead_w & 0xff) == c) 3959 { 3960 salitem_T tsal; 3961 3962 /* Move entry with same index byte after the entries 3963 * we already found. */ 3964 ++i; 3965 --n; 3966 tsal = smp[i + n]; 3967 mch_memmove(smp + i + 1, smp + i, 3968 sizeof(salitem_T) * n); 3969 smp[i] = tsal; 3970 } 3971 } 3972#endif 3973 } 3974 } 3975} 3976 3977#ifdef FEAT_MBYTE 3978/* 3979 * Turn a multi-byte string into a wide character string. 3980 * Return it in allocated memory (NULL for out-of-memory) 3981 */ 3982 static int * 3983mb_str2wide(s) 3984 char_u *s; 3985{ 3986 int *res; 3987 char_u *p; 3988 int i = 0; 3989 3990 res = (int *)alloc(sizeof(int) * (mb_charlen(s) + 1)); 3991 if (res != NULL) 3992 { 3993 for (p = s; *p != NUL; ) 3994 res[i++] = mb_ptr2char_adv(&p); 3995 res[i] = NUL; 3996 } 3997 return res; 3998} 3999#endif 4000 4001/* 4002 * Read a tree from the .spl or .sug file. 4003 * Allocates the memory and stores pointers in "bytsp" and "idxsp". 4004 * This is skipped when the tree has zero length. 4005 * Returns zero when OK, SP_ value for an error. 4006 */ 4007 static int 4008spell_read_tree(fd, bytsp, idxsp, prefixtree, prefixcnt) 4009 FILE *fd; 4010 char_u **bytsp; 4011 idx_T **idxsp; 4012 int prefixtree; /* TRUE for the prefix tree */ 4013 int prefixcnt; /* when "prefixtree" is TRUE: prefix count */ 4014{ 4015 int len; 4016 int idx; 4017 char_u *bp; 4018 idx_T *ip; 4019 4020 /* The tree size was computed when writing the file, so that we can 4021 * allocate it as one long block. <nodecount> */ 4022 len = get4c(fd); 4023 if (len < 0) 4024 return SP_TRUNCERROR; 4025 if (len > 0) 4026 { 4027 /* Allocate the byte array. */ 4028 bp = lalloc((long_u)len, TRUE); 4029 if (bp == NULL) 4030 return SP_OTHERERROR; 4031 *bytsp = bp; 4032 4033 /* Allocate the index array. */ 4034 ip = (idx_T *)lalloc_clear((long_u)(len * sizeof(int)), TRUE); 4035 if (ip == NULL) 4036 return SP_OTHERERROR; 4037 *idxsp = ip; 4038 4039 /* Recursively read the tree and store it in the array. */ 4040 idx = read_tree_node(fd, bp, ip, len, 0, prefixtree, prefixcnt); 4041 if (idx < 0) 4042 return idx; 4043 } 4044 return 0; 4045} 4046 4047/* 4048 * Read one row of siblings from the spell file and store it in the byte array 4049 * "byts" and index array "idxs". Recursively read the children. 4050 * 4051 * NOTE: The code here must match put_node()! 4052 * 4053 * Returns the index (>= 0) following the siblings. 4054 * Returns SP_TRUNCERROR if the file is shorter than expected. 4055 * Returns SP_FORMERROR if there is a format error. 4056 */ 4057 static idx_T 4058read_tree_node(fd, byts, idxs, maxidx, startidx, prefixtree, maxprefcondnr) 4059 FILE *fd; 4060 char_u *byts; 4061 idx_T *idxs; 4062 int maxidx; /* size of arrays */ 4063 idx_T startidx; /* current index in "byts" and "idxs" */ 4064 int prefixtree; /* TRUE for reading PREFIXTREE */ 4065 int maxprefcondnr; /* maximum for <prefcondnr> */ 4066{ 4067 int len; 4068 int i; 4069 int n; 4070 idx_T idx = startidx; 4071 int c; 4072 int c2; 4073#define SHARED_MASK 0x8000000 4074 4075 len = getc(fd); /* <siblingcount> */ 4076 if (len <= 0) 4077 return SP_TRUNCERROR; 4078 4079 if (startidx + len >= maxidx) 4080 return SP_FORMERROR; 4081 byts[idx++] = len; 4082 4083 /* Read the byte values, flag/region bytes and shared indexes. */ 4084 for (i = 1; i <= len; ++i) 4085 { 4086 c = getc(fd); /* <byte> */ 4087 if (c < 0) 4088 return SP_TRUNCERROR; 4089 if (c <= BY_SPECIAL) 4090 { 4091 if (c == BY_NOFLAGS && !prefixtree) 4092 { 4093 /* No flags, all regions. */ 4094 idxs[idx] = 0; 4095 c = 0; 4096 } 4097 else if (c != BY_INDEX) 4098 { 4099 if (prefixtree) 4100 { 4101 /* Read the optional pflags byte, the prefix ID and the 4102 * condition nr. In idxs[] store the prefix ID in the low 4103 * byte, the condition index shifted up 8 bits, the flags 4104 * shifted up 24 bits. */ 4105 if (c == BY_FLAGS) 4106 c = getc(fd) << 24; /* <pflags> */ 4107 else 4108 c = 0; 4109 4110 c |= getc(fd); /* <affixID> */ 4111 4112 n = get2c(fd); /* <prefcondnr> */ 4113 if (n >= maxprefcondnr) 4114 return SP_FORMERROR; 4115 c |= (n << 8); 4116 } 4117 else /* c must be BY_FLAGS or BY_FLAGS2 */ 4118 { 4119 /* Read flags and optional region and prefix ID. In 4120 * idxs[] the flags go in the low two bytes, region above 4121 * that and prefix ID above the region. */ 4122 c2 = c; 4123 c = getc(fd); /* <flags> */ 4124 if (c2 == BY_FLAGS2) 4125 c = (getc(fd) << 8) + c; /* <flags2> */ 4126 if (c & WF_REGION) 4127 c = (getc(fd) << 16) + c; /* <region> */ 4128 if (c & WF_AFX) 4129 c = (getc(fd) << 24) + c; /* <affixID> */ 4130 } 4131 4132 idxs[idx] = c; 4133 c = 0; 4134 } 4135 else /* c == BY_INDEX */ 4136 { 4137 /* <nodeidx> */ 4138 n = get3c(fd); 4139 if (n < 0 || n >= maxidx) 4140 return SP_FORMERROR; 4141 idxs[idx] = n + SHARED_MASK; 4142 c = getc(fd); /* <xbyte> */ 4143 } 4144 } 4145 byts[idx++] = c; 4146 } 4147 4148 /* Recursively read the children for non-shared siblings. 4149 * Skip the end-of-word ones (zero byte value) and the shared ones (and 4150 * remove SHARED_MASK) */ 4151 for (i = 1; i <= len; ++i) 4152 if (byts[startidx + i] != 0) 4153 { 4154 if (idxs[startidx + i] & SHARED_MASK) 4155 idxs[startidx + i] &= ~SHARED_MASK; 4156 else 4157 { 4158 idxs[startidx + i] = idx; 4159 idx = read_tree_node(fd, byts, idxs, maxidx, idx, 4160 prefixtree, maxprefcondnr); 4161 if (idx < 0) 4162 break; 4163 } 4164 } 4165 4166 return idx; 4167} 4168 4169/* 4170 * Parse 'spelllang' and set w_s->b_langp accordingly. 4171 * Returns NULL if it's OK, an error message otherwise. 4172 */ 4173 char_u * 4174did_set_spelllang(wp) 4175 win_T *wp; 4176{ 4177 garray_T ga; 4178 char_u *splp; 4179 char_u *region; 4180 char_u region_cp[3]; 4181 int filename; 4182 int region_mask; 4183 slang_T *slang; 4184 int c; 4185 char_u lang[MAXWLEN + 1]; 4186 char_u spf_name[MAXPATHL]; 4187 int len; 4188 char_u *p; 4189 int round; 4190 char_u *spf; 4191 char_u *use_region = NULL; 4192 int dont_use_region = FALSE; 4193 int nobreak = FALSE; 4194 int i, j; 4195 langp_T *lp, *lp2; 4196 static int recursive = FALSE; 4197 char_u *ret_msg = NULL; 4198 char_u *spl_copy; 4199 4200 /* We don't want to do this recursively. May happen when a language is 4201 * not available and the SpellFileMissing autocommand opens a new buffer 4202 * in which 'spell' is set. */ 4203 if (recursive) 4204 return NULL; 4205 recursive = TRUE; 4206 4207 ga_init2(&ga, sizeof(langp_T), 2); 4208 clear_midword(wp); 4209 4210 /* Make a copy of 'spellang', the SpellFileMissing autocommands may change 4211 * it under our fingers. */ 4212 spl_copy = vim_strsave(wp->w_s->b_p_spl); 4213 if (spl_copy == NULL) 4214 goto theend; 4215 4216 /* loop over comma separated language names. */ 4217 for (splp = spl_copy; *splp != NUL; ) 4218 { 4219 /* Get one language name. */ 4220 copy_option_part(&splp, lang, MAXWLEN, ","); 4221 region = NULL; 4222 len = (int)STRLEN(lang); 4223 4224 /* If the name ends in ".spl" use it as the name of the spell file. 4225 * If there is a region name let "region" point to it and remove it 4226 * from the name. */ 4227 if (len > 4 && fnamecmp(lang + len - 4, ".spl") == 0) 4228 { 4229 filename = TRUE; 4230 4231 /* Locate a region and remove it from the file name. */ 4232 p = vim_strchr(gettail(lang), '_'); 4233 if (p != NULL && ASCII_ISALPHA(p[1]) && ASCII_ISALPHA(p[2]) 4234 && !ASCII_ISALPHA(p[3])) 4235 { 4236 vim_strncpy(region_cp, p + 1, 2); 4237 mch_memmove(p, p + 3, len - (p - lang) - 2); 4238 len -= 3; 4239 region = region_cp; 4240 } 4241 else 4242 dont_use_region = TRUE; 4243 4244 /* Check if we loaded this language before. */ 4245 for (slang = first_lang; slang != NULL; slang = slang->sl_next) 4246 if (fullpathcmp(lang, slang->sl_fname, FALSE) == FPC_SAME) 4247 break; 4248 } 4249 else 4250 { 4251 filename = FALSE; 4252 if (len > 3 && lang[len - 3] == '_') 4253 { 4254 region = lang + len - 2; 4255 len -= 3; 4256 lang[len] = NUL; 4257 } 4258 else 4259 dont_use_region = TRUE; 4260 4261 /* Check if we loaded this language before. */ 4262 for (slang = first_lang; slang != NULL; slang = slang->sl_next) 4263 if (STRICMP(lang, slang->sl_name) == 0) 4264 break; 4265 } 4266 4267 if (region != NULL) 4268 { 4269 /* If the region differs from what was used before then don't 4270 * use it for 'spellfile'. */ 4271 if (use_region != NULL && STRCMP(region, use_region) != 0) 4272 dont_use_region = TRUE; 4273 use_region = region; 4274 } 4275 4276 /* If not found try loading the language now. */ 4277 if (slang == NULL) 4278 { 4279 if (filename) 4280 (void)spell_load_file(lang, lang, NULL, FALSE); 4281 else 4282 { 4283 spell_load_lang(lang); 4284#ifdef FEAT_AUTOCMD 4285 /* SpellFileMissing autocommands may do anything, including 4286 * destroying the buffer we are using... */ 4287 if (!buf_valid(wp->w_buffer)) 4288 { 4289 ret_msg = (char_u *)"E797: SpellFileMissing autocommand deleted buffer"; 4290 goto theend; 4291 } 4292#endif 4293 } 4294 } 4295 4296 /* 4297 * Loop over the languages, there can be several files for "lang". 4298 */ 4299 for (slang = first_lang; slang != NULL; slang = slang->sl_next) 4300 if (filename ? fullpathcmp(lang, slang->sl_fname, FALSE) == FPC_SAME 4301 : STRICMP(lang, slang->sl_name) == 0) 4302 { 4303 region_mask = REGION_ALL; 4304 if (!filename && region != NULL) 4305 { 4306 /* find region in sl_regions */ 4307 c = find_region(slang->sl_regions, region); 4308 if (c == REGION_ALL) 4309 { 4310 if (slang->sl_add) 4311 { 4312 if (*slang->sl_regions != NUL) 4313 /* This addition file is for other regions. */ 4314 region_mask = 0; 4315 } 4316 else 4317 /* This is probably an error. Give a warning and 4318 * accept the words anyway. */ 4319 smsg((char_u *) 4320 _("Warning: region %s not supported"), 4321 region); 4322 } 4323 else 4324 region_mask = 1 << c; 4325 } 4326 4327 if (region_mask != 0) 4328 { 4329 if (ga_grow(&ga, 1) == FAIL) 4330 { 4331 ga_clear(&ga); 4332 ret_msg = e_outofmem; 4333 goto theend; 4334 } 4335 LANGP_ENTRY(ga, ga.ga_len)->lp_slang = slang; 4336 LANGP_ENTRY(ga, ga.ga_len)->lp_region = region_mask; 4337 ++ga.ga_len; 4338 use_midword(slang, wp); 4339 if (slang->sl_nobreak) 4340 nobreak = TRUE; 4341 } 4342 } 4343 } 4344 4345 /* round 0: load int_wordlist, if possible. 4346 * round 1: load first name in 'spellfile'. 4347 * round 2: load second name in 'spellfile. 4348 * etc. */ 4349 spf = curwin->w_s->b_p_spf; 4350 for (round = 0; round == 0 || *spf != NUL; ++round) 4351 { 4352 if (round == 0) 4353 { 4354 /* Internal wordlist, if there is one. */ 4355 if (int_wordlist == NULL) 4356 continue; 4357 int_wordlist_spl(spf_name); 4358 } 4359 else 4360 { 4361 /* One entry in 'spellfile'. */ 4362 copy_option_part(&spf, spf_name, MAXPATHL - 5, ","); 4363 STRCAT(spf_name, ".spl"); 4364 4365 /* If it was already found above then skip it. */ 4366 for (c = 0; c < ga.ga_len; ++c) 4367 { 4368 p = LANGP_ENTRY(ga, c)->lp_slang->sl_fname; 4369 if (p != NULL && fullpathcmp(spf_name, p, FALSE) == FPC_SAME) 4370 break; 4371 } 4372 if (c < ga.ga_len) 4373 continue; 4374 } 4375 4376 /* Check if it was loaded already. */ 4377 for (slang = first_lang; slang != NULL; slang = slang->sl_next) 4378 if (fullpathcmp(spf_name, slang->sl_fname, FALSE) == FPC_SAME) 4379 break; 4380 if (slang == NULL) 4381 { 4382 /* Not loaded, try loading it now. The language name includes the 4383 * region name, the region is ignored otherwise. for int_wordlist 4384 * use an arbitrary name. */ 4385 if (round == 0) 4386 STRCPY(lang, "internal wordlist"); 4387 else 4388 { 4389 vim_strncpy(lang, gettail(spf_name), MAXWLEN); 4390 p = vim_strchr(lang, '.'); 4391 if (p != NULL) 4392 *p = NUL; /* truncate at ".encoding.add" */ 4393 } 4394 slang = spell_load_file(spf_name, lang, NULL, TRUE); 4395 4396 /* If one of the languages has NOBREAK we assume the addition 4397 * files also have this. */ 4398 if (slang != NULL && nobreak) 4399 slang->sl_nobreak = TRUE; 4400 } 4401 if (slang != NULL && ga_grow(&ga, 1) == OK) 4402 { 4403 region_mask = REGION_ALL; 4404 if (use_region != NULL && !dont_use_region) 4405 { 4406 /* find region in sl_regions */ 4407 c = find_region(slang->sl_regions, use_region); 4408 if (c != REGION_ALL) 4409 region_mask = 1 << c; 4410 else if (*slang->sl_regions != NUL) 4411 /* This spell file is for other regions. */ 4412 region_mask = 0; 4413 } 4414 4415 if (region_mask != 0) 4416 { 4417 LANGP_ENTRY(ga, ga.ga_len)->lp_slang = slang; 4418 LANGP_ENTRY(ga, ga.ga_len)->lp_sallang = NULL; 4419 LANGP_ENTRY(ga, ga.ga_len)->lp_replang = NULL; 4420 LANGP_ENTRY(ga, ga.ga_len)->lp_region = region_mask; 4421 ++ga.ga_len; 4422 use_midword(slang, wp); 4423 } 4424 } 4425 } 4426 4427 /* Everything is fine, store the new b_langp value. */ 4428 ga_clear(&wp->w_s->b_langp); 4429 wp->w_s->b_langp = ga; 4430 4431 /* For each language figure out what language to use for sound folding and 4432 * REP items. If the language doesn't support it itself use another one 4433 * with the same name. E.g. for "en-math" use "en". */ 4434 for (i = 0; i < ga.ga_len; ++i) 4435 { 4436 lp = LANGP_ENTRY(ga, i); 4437 4438 /* sound folding */ 4439 if (lp->lp_slang->sl_sal.ga_len > 0) 4440 /* language does sound folding itself */ 4441 lp->lp_sallang = lp->lp_slang; 4442 else 4443 /* find first similar language that does sound folding */ 4444 for (j = 0; j < ga.ga_len; ++j) 4445 { 4446 lp2 = LANGP_ENTRY(ga, j); 4447 if (lp2->lp_slang->sl_sal.ga_len > 0 4448 && STRNCMP(lp->lp_slang->sl_name, 4449 lp2->lp_slang->sl_name, 2) == 0) 4450 { 4451 lp->lp_sallang = lp2->lp_slang; 4452 break; 4453 } 4454 } 4455 4456 /* REP items */ 4457 if (lp->lp_slang->sl_rep.ga_len > 0) 4458 /* language has REP items itself */ 4459 lp->lp_replang = lp->lp_slang; 4460 else 4461 /* find first similar language that has REP items */ 4462 for (j = 0; j < ga.ga_len; ++j) 4463 { 4464 lp2 = LANGP_ENTRY(ga, j); 4465 if (lp2->lp_slang->sl_rep.ga_len > 0 4466 && STRNCMP(lp->lp_slang->sl_name, 4467 lp2->lp_slang->sl_name, 2) == 0) 4468 { 4469 lp->lp_replang = lp2->lp_slang; 4470 break; 4471 } 4472 } 4473 } 4474 4475theend: 4476 vim_free(spl_copy); 4477 recursive = FALSE; 4478 return ret_msg; 4479} 4480 4481/* 4482 * Clear the midword characters for buffer "buf". 4483 */ 4484 static void 4485clear_midword(wp) 4486 win_T *wp; 4487{ 4488 vim_memset(wp->w_s->b_spell_ismw, 0, 256); 4489#ifdef FEAT_MBYTE 4490 vim_free(wp->w_s->b_spell_ismw_mb); 4491 wp->w_s->b_spell_ismw_mb = NULL; 4492#endif 4493} 4494 4495/* 4496 * Use the "sl_midword" field of language "lp" for buffer "buf". 4497 * They add up to any currently used midword characters. 4498 */ 4499 static void 4500use_midword(lp, wp) 4501 slang_T *lp; 4502 win_T *wp; 4503{ 4504 char_u *p; 4505 4506 if (lp->sl_midword == NULL) /* there aren't any */ 4507 return; 4508 4509 for (p = lp->sl_midword; *p != NUL; ) 4510#ifdef FEAT_MBYTE 4511 if (has_mbyte) 4512 { 4513 int c, l, n; 4514 char_u *bp; 4515 4516 c = mb_ptr2char(p); 4517 l = (*mb_ptr2len)(p); 4518 if (c < 256 && l <= 2) 4519 wp->w_s->b_spell_ismw[c] = TRUE; 4520 else if (wp->w_s->b_spell_ismw_mb == NULL) 4521 /* First multi-byte char in "b_spell_ismw_mb". */ 4522 wp->w_s->b_spell_ismw_mb = vim_strnsave(p, l); 4523 else 4524 { 4525 /* Append multi-byte chars to "b_spell_ismw_mb". */ 4526 n = (int)STRLEN(wp->w_s->b_spell_ismw_mb); 4527 bp = vim_strnsave(wp->w_s->b_spell_ismw_mb, n + l); 4528 if (bp != NULL) 4529 { 4530 vim_free(wp->w_s->b_spell_ismw_mb); 4531 wp->w_s->b_spell_ismw_mb = bp; 4532 vim_strncpy(bp + n, p, l); 4533 } 4534 } 4535 p += l; 4536 } 4537 else 4538#endif 4539 wp->w_s->b_spell_ismw[*p++] = TRUE; 4540} 4541 4542/* 4543 * Find the region "region[2]" in "rp" (points to "sl_regions"). 4544 * Each region is simply stored as the two characters of it's name. 4545 * Returns the index if found (first is 0), REGION_ALL if not found. 4546 */ 4547 static int 4548find_region(rp, region) 4549 char_u *rp; 4550 char_u *region; 4551{ 4552 int i; 4553 4554 for (i = 0; ; i += 2) 4555 { 4556 if (rp[i] == NUL) 4557 return REGION_ALL; 4558 if (rp[i] == region[0] && rp[i + 1] == region[1]) 4559 break; 4560 } 4561 return i / 2; 4562} 4563 4564/* 4565 * Return case type of word: 4566 * w word 0 4567 * Word WF_ONECAP 4568 * W WORD WF_ALLCAP 4569 * WoRd wOrd WF_KEEPCAP 4570 */ 4571 static int 4572captype(word, end) 4573 char_u *word; 4574 char_u *end; /* When NULL use up to NUL byte. */ 4575{ 4576 char_u *p; 4577 int c; 4578 int firstcap; 4579 int allcap; 4580 int past_second = FALSE; /* past second word char */ 4581 4582 /* find first letter */ 4583 for (p = word; !spell_iswordp_nmw(p); mb_ptr_adv(p)) 4584 if (end == NULL ? *p == NUL : p >= end) 4585 return 0; /* only non-word characters, illegal word */ 4586#ifdef FEAT_MBYTE 4587 if (has_mbyte) 4588 c = mb_ptr2char_adv(&p); 4589 else 4590#endif 4591 c = *p++; 4592 firstcap = allcap = SPELL_ISUPPER(c); 4593 4594 /* 4595 * Need to check all letters to find a word with mixed upper/lower. 4596 * But a word with an upper char only at start is a ONECAP. 4597 */ 4598 for ( ; end == NULL ? *p != NUL : p < end; mb_ptr_adv(p)) 4599 if (spell_iswordp_nmw(p)) 4600 { 4601 c = PTR2CHAR(p); 4602 if (!SPELL_ISUPPER(c)) 4603 { 4604 /* UUl -> KEEPCAP */ 4605 if (past_second && allcap) 4606 return WF_KEEPCAP; 4607 allcap = FALSE; 4608 } 4609 else if (!allcap) 4610 /* UlU -> KEEPCAP */ 4611 return WF_KEEPCAP; 4612 past_second = TRUE; 4613 } 4614 4615 if (allcap) 4616 return WF_ALLCAP; 4617 if (firstcap) 4618 return WF_ONECAP; 4619 return 0; 4620} 4621 4622/* 4623 * Like captype() but for a KEEPCAP word add ONECAP if the word starts with a 4624 * capital. So that make_case_word() can turn WOrd into Word. 4625 * Add ALLCAP for "WOrD". 4626 */ 4627 static int 4628badword_captype(word, end) 4629 char_u *word; 4630 char_u *end; 4631{ 4632 int flags = captype(word, end); 4633 int c; 4634 int l, u; 4635 int first; 4636 char_u *p; 4637 4638 if (flags & WF_KEEPCAP) 4639 { 4640 /* Count the number of UPPER and lower case letters. */ 4641 l = u = 0; 4642 first = FALSE; 4643 for (p = word; p < end; mb_ptr_adv(p)) 4644 { 4645 c = PTR2CHAR(p); 4646 if (SPELL_ISUPPER(c)) 4647 { 4648 ++u; 4649 if (p == word) 4650 first = TRUE; 4651 } 4652 else 4653 ++l; 4654 } 4655 4656 /* If there are more UPPER than lower case letters suggest an 4657 * ALLCAP word. Otherwise, if the first letter is UPPER then 4658 * suggest ONECAP. Exception: "ALl" most likely should be "All", 4659 * require three upper case letters. */ 4660 if (u > l && u > 2) 4661 flags |= WF_ALLCAP; 4662 else if (first) 4663 flags |= WF_ONECAP; 4664 4665 if (u >= 2 && l >= 2) /* maCARONI maCAroni */ 4666 flags |= WF_MIXCAP; 4667 } 4668 return flags; 4669} 4670 4671# if defined(FEAT_MBYTE) || defined(EXITFREE) || defined(PROTO) 4672/* 4673 * Free all languages. 4674 */ 4675 void 4676spell_free_all() 4677{ 4678 slang_T *slang; 4679 buf_T *buf; 4680 char_u fname[MAXPATHL]; 4681 4682 /* Go through all buffers and handle 'spelllang'. */ //<VN> 4683 for (buf = firstbuf; buf != NULL; buf = buf->b_next) 4684 ga_clear(&buf->b_s.b_langp); 4685 4686 while (first_lang != NULL) 4687 { 4688 slang = first_lang; 4689 first_lang = slang->sl_next; 4690 slang_free(slang); 4691 } 4692 4693 if (int_wordlist != NULL) 4694 { 4695 /* Delete the internal wordlist and its .spl file */ 4696 mch_remove(int_wordlist); 4697 int_wordlist_spl(fname); 4698 mch_remove(fname); 4699 vim_free(int_wordlist); 4700 int_wordlist = NULL; 4701 } 4702 4703 init_spell_chartab(); 4704 4705 vim_free(repl_to); 4706 repl_to = NULL; 4707 vim_free(repl_from); 4708 repl_from = NULL; 4709} 4710# endif 4711 4712# if defined(FEAT_MBYTE) || defined(PROTO) 4713/* 4714 * Clear all spelling tables and reload them. 4715 * Used after 'encoding' is set and when ":mkspell" was used. 4716 */ 4717 void 4718spell_reload() 4719{ 4720 win_T *wp; 4721 4722 /* Initialize the table for spell_iswordp(). */ 4723 init_spell_chartab(); 4724 4725 /* Unload all allocated memory. */ 4726 spell_free_all(); 4727 4728 /* Go through all buffers and handle 'spelllang'. */ 4729 for (wp = firstwin; wp != NULL; wp = wp->w_next) 4730 { 4731 /* Only load the wordlists when 'spelllang' is set and there is a 4732 * window for this buffer in which 'spell' is set. */ 4733 if (*wp->w_s->b_p_spl != NUL) 4734 { 4735 if (wp->w_p_spell) 4736 { 4737 (void)did_set_spelllang(wp); 4738# ifdef FEAT_WINDOWS 4739 break; 4740# endif 4741 } 4742 } 4743 } 4744} 4745# endif 4746 4747/* 4748 * Reload the spell file "fname" if it's loaded. 4749 */ 4750 static void 4751spell_reload_one(fname, added_word) 4752 char_u *fname; 4753 int added_word; /* invoked through "zg" */ 4754{ 4755 slang_T *slang; 4756 int didit = FALSE; 4757 4758 for (slang = first_lang; slang != NULL; slang = slang->sl_next) 4759 { 4760 if (fullpathcmp(fname, slang->sl_fname, FALSE) == FPC_SAME) 4761 { 4762 slang_clear(slang); 4763 if (spell_load_file(fname, NULL, slang, FALSE) == NULL) 4764 /* reloading failed, clear the language */ 4765 slang_clear(slang); 4766 redraw_all_later(SOME_VALID); 4767 didit = TRUE; 4768 } 4769 } 4770 4771 /* When "zg" was used and the file wasn't loaded yet, should redo 4772 * 'spelllang' to load it now. */ 4773 if (added_word && !didit) 4774 did_set_spelllang(curwin); 4775} 4776 4777 4778/* 4779 * Functions for ":mkspell". 4780 */ 4781 4782#define MAXLINELEN 500 /* Maximum length in bytes of a line in a .aff 4783 and .dic file. */ 4784/* 4785 * Main structure to store the contents of a ".aff" file. 4786 */ 4787typedef struct afffile_S 4788{ 4789 char_u *af_enc; /* "SET", normalized, alloc'ed string or NULL */ 4790 int af_flagtype; /* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */ 4791 unsigned af_rare; /* RARE ID for rare word */ 4792 unsigned af_keepcase; /* KEEPCASE ID for keep-case word */ 4793 unsigned af_bad; /* BAD ID for banned word */ 4794 unsigned af_needaffix; /* NEEDAFFIX ID */ 4795 unsigned af_circumfix; /* CIRCUMFIX ID */ 4796 unsigned af_needcomp; /* NEEDCOMPOUND ID */ 4797 unsigned af_comproot; /* COMPOUNDROOT ID */ 4798 unsigned af_compforbid; /* COMPOUNDFORBIDFLAG ID */ 4799 unsigned af_comppermit; /* COMPOUNDPERMITFLAG ID */ 4800 unsigned af_nosuggest; /* NOSUGGEST ID */ 4801 int af_pfxpostpone; /* postpone prefixes without chop string and 4802 without flags */ 4803 hashtab_T af_pref; /* hashtable for prefixes, affheader_T */ 4804 hashtab_T af_suff; /* hashtable for suffixes, affheader_T */ 4805 hashtab_T af_comp; /* hashtable for compound flags, compitem_T */ 4806} afffile_T; 4807 4808#define AFT_CHAR 0 /* flags are one character */ 4809#define AFT_LONG 1 /* flags are two characters */ 4810#define AFT_CAPLONG 2 /* flags are one or two characters */ 4811#define AFT_NUM 3 /* flags are numbers, comma separated */ 4812 4813typedef struct affentry_S affentry_T; 4814/* Affix entry from ".aff" file. Used for prefixes and suffixes. */ 4815struct affentry_S 4816{ 4817 affentry_T *ae_next; /* next affix with same name/number */ 4818 char_u *ae_chop; /* text to chop off basic word (can be NULL) */ 4819 char_u *ae_add; /* text to add to basic word (can be NULL) */ 4820 char_u *ae_flags; /* flags on the affix (can be NULL) */ 4821 char_u *ae_cond; /* condition (NULL for ".") */ 4822 regprog_T *ae_prog; /* regexp program for ae_cond or NULL */ 4823 char ae_compforbid; /* COMPOUNDFORBIDFLAG found */ 4824 char ae_comppermit; /* COMPOUNDPERMITFLAG found */ 4825}; 4826 4827#ifdef FEAT_MBYTE 4828# define AH_KEY_LEN 17 /* 2 x 8 bytes + NUL */ 4829#else 4830# define AH_KEY_LEN 7 /* 6 digits + NUL */ 4831#endif 4832 4833/* Affix header from ".aff" file. Used for af_pref and af_suff. */ 4834typedef struct affheader_S 4835{ 4836 char_u ah_key[AH_KEY_LEN]; /* key for hashtab == name of affix */ 4837 unsigned ah_flag; /* affix name as number, uses "af_flagtype" */ 4838 int ah_newID; /* prefix ID after renumbering; 0 if not used */ 4839 int ah_combine; /* suffix may combine with prefix */ 4840 int ah_follows; /* another affix block should be following */ 4841 affentry_T *ah_first; /* first affix entry */ 4842} affheader_T; 4843 4844#define HI2AH(hi) ((affheader_T *)(hi)->hi_key) 4845 4846/* Flag used in compound items. */ 4847typedef struct compitem_S 4848{ 4849 char_u ci_key[AH_KEY_LEN]; /* key for hashtab == name of compound */ 4850 unsigned ci_flag; /* affix name as number, uses "af_flagtype" */ 4851 int ci_newID; /* affix ID after renumbering. */ 4852} compitem_T; 4853 4854#define HI2CI(hi) ((compitem_T *)(hi)->hi_key) 4855 4856/* 4857 * Structure that is used to store the items in the word tree. This avoids 4858 * the need to keep track of each allocated thing, everything is freed all at 4859 * once after ":mkspell" is done. 4860 * Note: "sb_next" must be just before "sb_data" to make sure the alignment of 4861 * "sb_data" is correct for systems where pointers must be aligned on 4862 * pointer-size boundaries and sizeof(pointer) > sizeof(int) (e.g., Sparc). 4863 */ 4864#define SBLOCKSIZE 16000 /* size of sb_data */ 4865typedef struct sblock_S sblock_T; 4866struct sblock_S 4867{ 4868 int sb_used; /* nr of bytes already in use */ 4869 sblock_T *sb_next; /* next block in list */ 4870 char_u sb_data[1]; /* data, actually longer */ 4871}; 4872 4873/* 4874 * A node in the tree. 4875 */ 4876typedef struct wordnode_S wordnode_T; 4877struct wordnode_S 4878{ 4879 union /* shared to save space */ 4880 { 4881 char_u hashkey[6]; /* the hash key, only used while compressing */ 4882 int index; /* index in written nodes (valid after first 4883 round) */ 4884 } wn_u1; 4885 union /* shared to save space */ 4886 { 4887 wordnode_T *next; /* next node with same hash key */ 4888 wordnode_T *wnode; /* parent node that will write this node */ 4889 } wn_u2; 4890 wordnode_T *wn_child; /* child (next byte in word) */ 4891 wordnode_T *wn_sibling; /* next sibling (alternate byte in word, 4892 always sorted) */ 4893 int wn_refs; /* Nr. of references to this node. Only 4894 relevant for first node in a list of 4895 siblings, in following siblings it is 4896 always one. */ 4897 char_u wn_byte; /* Byte for this node. NUL for word end */ 4898 4899 /* Info for when "wn_byte" is NUL. 4900 * In PREFIXTREE "wn_region" is used for the prefcondnr. 4901 * In the soundfolded word tree "wn_flags" has the MSW of the wordnr and 4902 * "wn_region" the LSW of the wordnr. */ 4903 char_u wn_affixID; /* supported/required prefix ID or 0 */ 4904 short_u wn_flags; /* WF_ flags */ 4905 short wn_region; /* region mask */ 4906 4907#ifdef SPELL_PRINTTREE 4908 int wn_nr; /* sequence nr for printing */ 4909#endif 4910}; 4911 4912#define WN_MASK 0xffff /* mask relevant bits of "wn_flags" */ 4913 4914#define HI2WN(hi) (wordnode_T *)((hi)->hi_key) 4915 4916/* 4917 * Info used while reading the spell files. 4918 */ 4919typedef struct spellinfo_S 4920{ 4921 wordnode_T *si_foldroot; /* tree with case-folded words */ 4922 long si_foldwcount; /* nr of words in si_foldroot */ 4923 4924 wordnode_T *si_keeproot; /* tree with keep-case words */ 4925 long si_keepwcount; /* nr of words in si_keeproot */ 4926 4927 wordnode_T *si_prefroot; /* tree with postponed prefixes */ 4928 4929 long si_sugtree; /* creating the soundfolding trie */ 4930 4931 sblock_T *si_blocks; /* memory blocks used */ 4932 long si_blocks_cnt; /* memory blocks allocated */ 4933 long si_compress_cnt; /* words to add before lowering 4934 compression limit */ 4935 wordnode_T *si_first_free; /* List of nodes that have been freed during 4936 compression, linked by "wn_child" field. */ 4937 long si_free_count; /* number of nodes in si_first_free */ 4938#ifdef SPELL_PRINTTREE 4939 int si_wordnode_nr; /* sequence nr for nodes */ 4940#endif 4941 buf_T *si_spellbuf; /* buffer used to store soundfold word table */ 4942 4943 int si_ascii; /* handling only ASCII words */ 4944 int si_add; /* addition file */ 4945 int si_clear_chartab; /* when TRUE clear char tables */ 4946 int si_region; /* region mask */ 4947 vimconv_T si_conv; /* for conversion to 'encoding' */ 4948 int si_memtot; /* runtime memory used */ 4949 int si_verbose; /* verbose messages */ 4950 int si_msg_count; /* number of words added since last message */ 4951 char_u *si_info; /* info text chars or NULL */ 4952 int si_region_count; /* number of regions supported (1 when there 4953 are no regions) */ 4954 char_u si_region_name[16]; /* region names; used only if 4955 * si_region_count > 1) */ 4956 4957 garray_T si_rep; /* list of fromto_T entries from REP lines */ 4958 garray_T si_repsal; /* list of fromto_T entries from REPSAL lines */ 4959 garray_T si_sal; /* list of fromto_T entries from SAL lines */ 4960 char_u *si_sofofr; /* SOFOFROM text */ 4961 char_u *si_sofoto; /* SOFOTO text */ 4962 int si_nosugfile; /* NOSUGFILE item found */ 4963 int si_nosplitsugs; /* NOSPLITSUGS item found */ 4964 int si_followup; /* soundsalike: ? */ 4965 int si_collapse; /* soundsalike: ? */ 4966 hashtab_T si_commonwords; /* hashtable for common words */ 4967 time_t si_sugtime; /* timestamp for .sug file */ 4968 int si_rem_accents; /* soundsalike: remove accents */ 4969 garray_T si_map; /* MAP info concatenated */ 4970 char_u *si_midword; /* MIDWORD chars or NULL */ 4971 int si_compmax; /* max nr of words for compounding */ 4972 int si_compminlen; /* minimal length for compounding */ 4973 int si_compsylmax; /* max nr of syllables for compounding */ 4974 int si_compoptions; /* COMP_ flags */ 4975 garray_T si_comppat; /* CHECKCOMPOUNDPATTERN items, each stored as 4976 a string */ 4977 char_u *si_compflags; /* flags used for compounding */ 4978 char_u si_nobreak; /* NOBREAK */ 4979 char_u *si_syllable; /* syllable string */ 4980 garray_T si_prefcond; /* table with conditions for postponed 4981 * prefixes, each stored as a string */ 4982 int si_newprefID; /* current value for ah_newID */ 4983 int si_newcompID; /* current value for compound ID */ 4984} spellinfo_T; 4985 4986static afffile_T *spell_read_aff __ARGS((spellinfo_T *spin, char_u *fname)); 4987static int is_aff_rule __ARGS((char_u **items, int itemcnt, char *rulename, int mincount)); 4988static void aff_process_flags __ARGS((afffile_T *affile, affentry_T *entry)); 4989static int spell_info_item __ARGS((char_u *s)); 4990static unsigned affitem2flag __ARGS((int flagtype, char_u *item, char_u *fname, int lnum)); 4991static unsigned get_affitem __ARGS((int flagtype, char_u **pp)); 4992static void process_compflags __ARGS((spellinfo_T *spin, afffile_T *aff, char_u *compflags)); 4993static void check_renumber __ARGS((spellinfo_T *spin)); 4994static int flag_in_afflist __ARGS((int flagtype, char_u *afflist, unsigned flag)); 4995static void aff_check_number __ARGS((int spinval, int affval, char *name)); 4996static void aff_check_string __ARGS((char_u *spinval, char_u *affval, char *name)); 4997static int str_equal __ARGS((char_u *s1, char_u *s2)); 4998static void add_fromto __ARGS((spellinfo_T *spin, garray_T *gap, char_u *from, char_u *to)); 4999static int sal_to_bool __ARGS((char_u *s)); 5000static int has_non_ascii __ARGS((char_u *s)); 5001static void spell_free_aff __ARGS((afffile_T *aff)); 5002static int spell_read_dic __ARGS((spellinfo_T *spin, char_u *fname, afffile_T *affile)); 5003static int get_affix_flags __ARGS((afffile_T *affile, char_u *afflist)); 5004static int get_pfxlist __ARGS((afffile_T *affile, char_u *afflist, char_u *store_afflist)); 5005static void get_compflags __ARGS((afffile_T *affile, char_u *afflist, char_u *store_afflist)); 5006static int store_aff_word __ARGS((spellinfo_T *spin, char_u *word, char_u *afflist, afffile_T *affile, hashtab_T *ht, hashtab_T *xht, int condit, int flags, char_u *pfxlist, int pfxlen)); 5007static int spell_read_wordfile __ARGS((spellinfo_T *spin, char_u *fname)); 5008static void *getroom __ARGS((spellinfo_T *spin, size_t len, int align)); 5009static char_u *getroom_save __ARGS((spellinfo_T *spin, char_u *s)); 5010static void free_blocks __ARGS((sblock_T *bl)); 5011static wordnode_T *wordtree_alloc __ARGS((spellinfo_T *spin)); 5012static int store_word __ARGS((spellinfo_T *spin, char_u *word, int flags, int region, char_u *pfxlist, int need_affix)); 5013static int tree_add_word __ARGS((spellinfo_T *spin, char_u *word, wordnode_T *tree, int flags, int region, int affixID)); 5014static wordnode_T *get_wordnode __ARGS((spellinfo_T *spin)); 5015static int deref_wordnode __ARGS((spellinfo_T *spin, wordnode_T *node)); 5016static void free_wordnode __ARGS((spellinfo_T *spin, wordnode_T *n)); 5017static void wordtree_compress __ARGS((spellinfo_T *spin, wordnode_T *root)); 5018static int node_compress __ARGS((spellinfo_T *spin, wordnode_T *node, hashtab_T *ht, int *tot)); 5019static int node_equal __ARGS((wordnode_T *n1, wordnode_T *n2)); 5020static int write_vim_spell __ARGS((spellinfo_T *spin, char_u *fname)); 5021static void clear_node __ARGS((wordnode_T *node)); 5022static int put_node __ARGS((FILE *fd, wordnode_T *node, int idx, int regionmask, int prefixtree)); 5023static void spell_make_sugfile __ARGS((spellinfo_T *spin, char_u *wfname)); 5024static int sug_filltree __ARGS((spellinfo_T *spin, slang_T *slang)); 5025static int sug_maketable __ARGS((spellinfo_T *spin)); 5026static int sug_filltable __ARGS((spellinfo_T *spin, wordnode_T *node, int startwordnr, garray_T *gap)); 5027static int offset2bytes __ARGS((int nr, char_u *buf)); 5028static int bytes2offset __ARGS((char_u **pp)); 5029static void sug_write __ARGS((spellinfo_T *spin, char_u *fname)); 5030static void mkspell __ARGS((int fcount, char_u **fnames, int ascii, int overwrite, int added_word)); 5031static void spell_message __ARGS((spellinfo_T *spin, char_u *str)); 5032static void init_spellfile __ARGS((void)); 5033 5034/* In the postponed prefixes tree wn_flags is used to store the WFP_ flags, 5035 * but it must be negative to indicate the prefix tree to tree_add_word(). 5036 * Use a negative number with the lower 8 bits zero. */ 5037#define PFX_FLAGS -256 5038 5039/* flags for "condit" argument of store_aff_word() */ 5040#define CONDIT_COMB 1 /* affix must combine */ 5041#define CONDIT_CFIX 2 /* affix must have CIRCUMFIX flag */ 5042#define CONDIT_SUF 4 /* add a suffix for matching flags */ 5043#define CONDIT_AFF 8 /* word already has an affix */ 5044 5045/* 5046 * Tunable parameters for when the tree is compressed. See 'mkspellmem'. 5047 */ 5048static long compress_start = 30000; /* memory / SBLOCKSIZE */ 5049static long compress_inc = 100; /* memory / SBLOCKSIZE */ 5050static long compress_added = 500000; /* word count */ 5051 5052#ifdef SPELL_PRINTTREE 5053/* 5054 * For debugging the tree code: print the current tree in a (more or less) 5055 * readable format, so that we can see what happens when adding a word and/or 5056 * compressing the tree. 5057 * Based on code from Olaf Seibert. 5058 */ 5059#define PRINTLINESIZE 1000 5060#define PRINTWIDTH 6 5061 5062#define PRINTSOME(l, depth, fmt, a1, a2) vim_snprintf(l + depth * PRINTWIDTH, \ 5063 PRINTLINESIZE - PRINTWIDTH * depth, fmt, a1, a2) 5064 5065static char line1[PRINTLINESIZE]; 5066static char line2[PRINTLINESIZE]; 5067static char line3[PRINTLINESIZE]; 5068 5069 static void 5070spell_clear_flags(wordnode_T *node) 5071{ 5072 wordnode_T *np; 5073 5074 for (np = node; np != NULL; np = np->wn_sibling) 5075 { 5076 np->wn_u1.index = FALSE; 5077 spell_clear_flags(np->wn_child); 5078 } 5079} 5080 5081 static void 5082spell_print_node(wordnode_T *node, int depth) 5083{ 5084 if (node->wn_u1.index) 5085 { 5086 /* Done this node before, print the reference. */ 5087 PRINTSOME(line1, depth, "(%d)", node->wn_nr, 0); 5088 PRINTSOME(line2, depth, " ", 0, 0); 5089 PRINTSOME(line3, depth, " ", 0, 0); 5090 msg(line1); 5091 msg(line2); 5092 msg(line3); 5093 } 5094 else 5095 { 5096 node->wn_u1.index = TRUE; 5097 5098 if (node->wn_byte != NUL) 5099 { 5100 if (node->wn_child != NULL) 5101 PRINTSOME(line1, depth, " %c -> ", node->wn_byte, 0); 5102 else 5103 /* Cannot happen? */ 5104 PRINTSOME(line1, depth, " %c ???", node->wn_byte, 0); 5105 } 5106 else 5107 PRINTSOME(line1, depth, " $ ", 0, 0); 5108 5109 PRINTSOME(line2, depth, "%d/%d ", node->wn_nr, node->wn_refs); 5110 5111 if (node->wn_sibling != NULL) 5112 PRINTSOME(line3, depth, " | ", 0, 0); 5113 else 5114 PRINTSOME(line3, depth, " ", 0, 0); 5115 5116 if (node->wn_byte == NUL) 5117 { 5118 msg(line1); 5119 msg(line2); 5120 msg(line3); 5121 } 5122 5123 /* do the children */ 5124 if (node->wn_byte != NUL && node->wn_child != NULL) 5125 spell_print_node(node->wn_child, depth + 1); 5126 5127 /* do the siblings */ 5128 if (node->wn_sibling != NULL) 5129 { 5130 /* get rid of all parent details except | */ 5131 STRCPY(line1, line3); 5132 STRCPY(line2, line3); 5133 spell_print_node(node->wn_sibling, depth); 5134 } 5135 } 5136} 5137 5138 static void 5139spell_print_tree(wordnode_T *root) 5140{ 5141 if (root != NULL) 5142 { 5143 /* Clear the "wn_u1.index" fields, used to remember what has been 5144 * done. */ 5145 spell_clear_flags(root); 5146 5147 /* Recursively print the tree. */ 5148 spell_print_node(root, 0); 5149 } 5150} 5151#endif /* SPELL_PRINTTREE */ 5152 5153/* 5154 * Read the affix file "fname". 5155 * Returns an afffile_T, NULL for complete failure. 5156 */ 5157 static afffile_T * 5158spell_read_aff(spin, fname) 5159 spellinfo_T *spin; 5160 char_u *fname; 5161{ 5162 FILE *fd; 5163 afffile_T *aff; 5164 char_u rline[MAXLINELEN]; 5165 char_u *line; 5166 char_u *pc = NULL; 5167#define MAXITEMCNT 30 5168 char_u *(items[MAXITEMCNT]); 5169 int itemcnt; 5170 char_u *p; 5171 int lnum = 0; 5172 affheader_T *cur_aff = NULL; 5173 int did_postpone_prefix = FALSE; 5174 int aff_todo = 0; 5175 hashtab_T *tp; 5176 char_u *low = NULL; 5177 char_u *fol = NULL; 5178 char_u *upp = NULL; 5179 int do_rep; 5180 int do_repsal; 5181 int do_sal; 5182 int do_mapline; 5183 int found_map = FALSE; 5184 hashitem_T *hi; 5185 int l; 5186 int compminlen = 0; /* COMPOUNDMIN value */ 5187 int compsylmax = 0; /* COMPOUNDSYLMAX value */ 5188 int compoptions = 0; /* COMP_ flags */ 5189 int compmax = 0; /* COMPOUNDWORDMAX value */ 5190 char_u *compflags = NULL; /* COMPOUNDFLAG and COMPOUNDRULE 5191 concatenated */ 5192 char_u *midword = NULL; /* MIDWORD value */ 5193 char_u *syllable = NULL; /* SYLLABLE value */ 5194 char_u *sofofrom = NULL; /* SOFOFROM value */ 5195 char_u *sofoto = NULL; /* SOFOTO value */ 5196 5197 /* 5198 * Open the file. 5199 */ 5200 fd = mch_fopen((char *)fname, "r"); 5201 if (fd == NULL) 5202 { 5203 EMSG2(_(e_notopen), fname); 5204 return NULL; 5205 } 5206 5207 vim_snprintf((char *)IObuff, IOSIZE, _("Reading affix file %s ..."), fname); 5208 spell_message(spin, IObuff); 5209 5210 /* Only do REP lines when not done in another .aff file already. */ 5211 do_rep = spin->si_rep.ga_len == 0; 5212 5213 /* Only do REPSAL lines when not done in another .aff file already. */ 5214 do_repsal = spin->si_repsal.ga_len == 0; 5215 5216 /* Only do SAL lines when not done in another .aff file already. */ 5217 do_sal = spin->si_sal.ga_len == 0; 5218 5219 /* Only do MAP lines when not done in another .aff file already. */ 5220 do_mapline = spin->si_map.ga_len == 0; 5221 5222 /* 5223 * Allocate and init the afffile_T structure. 5224 */ 5225 aff = (afffile_T *)getroom(spin, sizeof(afffile_T), TRUE); 5226 if (aff == NULL) 5227 { 5228 fclose(fd); 5229 return NULL; 5230 } 5231 hash_init(&aff->af_pref); 5232 hash_init(&aff->af_suff); 5233 hash_init(&aff->af_comp); 5234 5235 /* 5236 * Read all the lines in the file one by one. 5237 */ 5238 while (!vim_fgets(rline, MAXLINELEN, fd) && !got_int) 5239 { 5240 line_breakcheck(); 5241 ++lnum; 5242 5243 /* Skip comment lines. */ 5244 if (*rline == '#') 5245 continue; 5246 5247 /* Convert from "SET" to 'encoding' when needed. */ 5248 vim_free(pc); 5249#ifdef FEAT_MBYTE 5250 if (spin->si_conv.vc_type != CONV_NONE) 5251 { 5252 pc = string_convert(&spin->si_conv, rline, NULL); 5253 if (pc == NULL) 5254 { 5255 smsg((char_u *)_("Conversion failure for word in %s line %d: %s"), 5256 fname, lnum, rline); 5257 continue; 5258 } 5259 line = pc; 5260 } 5261 else 5262#endif 5263 { 5264 pc = NULL; 5265 line = rline; 5266 } 5267 5268 /* Split the line up in white separated items. Put a NUL after each 5269 * item. */ 5270 itemcnt = 0; 5271 for (p = line; ; ) 5272 { 5273 while (*p != NUL && *p <= ' ') /* skip white space and CR/NL */ 5274 ++p; 5275 if (*p == NUL) 5276 break; 5277 if (itemcnt == MAXITEMCNT) /* too many items */ 5278 break; 5279 items[itemcnt++] = p; 5280 /* A few items have arbitrary text argument, don't split them. */ 5281 if (itemcnt == 2 && spell_info_item(items[0])) 5282 while (*p >= ' ' || *p == TAB) /* skip until CR/NL */ 5283 ++p; 5284 else 5285 while (*p > ' ') /* skip until white space or CR/NL */ 5286 ++p; 5287 if (*p == NUL) 5288 break; 5289 *p++ = NUL; 5290 } 5291 5292 /* Handle non-empty lines. */ 5293 if (itemcnt > 0) 5294 { 5295 if (is_aff_rule(items, itemcnt, "SET", 2) && aff->af_enc == NULL) 5296 { 5297#ifdef FEAT_MBYTE 5298 /* Setup for conversion from "ENC" to 'encoding'. */ 5299 aff->af_enc = enc_canonize(items[1]); 5300 if (aff->af_enc != NULL && !spin->si_ascii 5301 && convert_setup(&spin->si_conv, aff->af_enc, 5302 p_enc) == FAIL) 5303 smsg((char_u *)_("Conversion in %s not supported: from %s to %s"), 5304 fname, aff->af_enc, p_enc); 5305 spin->si_conv.vc_fail = TRUE; 5306#else 5307 smsg((char_u *)_("Conversion in %s not supported"), fname); 5308#endif 5309 } 5310 else if (is_aff_rule(items, itemcnt, "FLAG", 2) 5311 && aff->af_flagtype == AFT_CHAR) 5312 { 5313 if (STRCMP(items[1], "long") == 0) 5314 aff->af_flagtype = AFT_LONG; 5315 else if (STRCMP(items[1], "num") == 0) 5316 aff->af_flagtype = AFT_NUM; 5317 else if (STRCMP(items[1], "caplong") == 0) 5318 aff->af_flagtype = AFT_CAPLONG; 5319 else 5320 smsg((char_u *)_("Invalid value for FLAG in %s line %d: %s"), 5321 fname, lnum, items[1]); 5322 if (aff->af_rare != 0 5323 || aff->af_keepcase != 0 5324 || aff->af_bad != 0 5325 || aff->af_needaffix != 0 5326 || aff->af_circumfix != 0 5327 || aff->af_needcomp != 0 5328 || aff->af_comproot != 0 5329 || aff->af_nosuggest != 0 5330 || compflags != NULL 5331 || aff->af_suff.ht_used > 0 5332 || aff->af_pref.ht_used > 0) 5333 smsg((char_u *)_("FLAG after using flags in %s line %d: %s"), 5334 fname, lnum, items[1]); 5335 } 5336 else if (spell_info_item(items[0])) 5337 { 5338 p = (char_u *)getroom(spin, 5339 (spin->si_info == NULL ? 0 : STRLEN(spin->si_info)) 5340 + STRLEN(items[0]) 5341 + STRLEN(items[1]) + 3, FALSE); 5342 if (p != NULL) 5343 { 5344 if (spin->si_info != NULL) 5345 { 5346 STRCPY(p, spin->si_info); 5347 STRCAT(p, "\n"); 5348 } 5349 STRCAT(p, items[0]); 5350 STRCAT(p, " "); 5351 STRCAT(p, items[1]); 5352 spin->si_info = p; 5353 } 5354 } 5355 else if (is_aff_rule(items, itemcnt, "MIDWORD", 2) 5356 && midword == NULL) 5357 { 5358 midword = getroom_save(spin, items[1]); 5359 } 5360 else if (is_aff_rule(items, itemcnt, "TRY", 2)) 5361 { 5362 /* ignored, we look in the tree for what chars may appear */ 5363 } 5364 /* TODO: remove "RAR" later */ 5365 else if ((is_aff_rule(items, itemcnt, "RAR", 2) 5366 || is_aff_rule(items, itemcnt, "RARE", 2)) 5367 && aff->af_rare == 0) 5368 { 5369 aff->af_rare = affitem2flag(aff->af_flagtype, items[1], 5370 fname, lnum); 5371 } 5372 /* TODO: remove "KEP" later */ 5373 else if ((is_aff_rule(items, itemcnt, "KEP", 2) 5374 || is_aff_rule(items, itemcnt, "KEEPCASE", 2)) 5375 && aff->af_keepcase == 0) 5376 { 5377 aff->af_keepcase = affitem2flag(aff->af_flagtype, items[1], 5378 fname, lnum); 5379 } 5380 else if ((is_aff_rule(items, itemcnt, "BAD", 2) 5381 || is_aff_rule(items, itemcnt, "FORBIDDENWORD", 2)) 5382 && aff->af_bad == 0) 5383 { 5384 aff->af_bad = affitem2flag(aff->af_flagtype, items[1], 5385 fname, lnum); 5386 } 5387 else if (is_aff_rule(items, itemcnt, "NEEDAFFIX", 2) 5388 && aff->af_needaffix == 0) 5389 { 5390 aff->af_needaffix = affitem2flag(aff->af_flagtype, items[1], 5391 fname, lnum); 5392 } 5393 else if (is_aff_rule(items, itemcnt, "CIRCUMFIX", 2) 5394 && aff->af_circumfix == 0) 5395 { 5396 aff->af_circumfix = affitem2flag(aff->af_flagtype, items[1], 5397 fname, lnum); 5398 } 5399 else if (is_aff_rule(items, itemcnt, "NOSUGGEST", 2) 5400 && aff->af_nosuggest == 0) 5401 { 5402 aff->af_nosuggest = affitem2flag(aff->af_flagtype, items[1], 5403 fname, lnum); 5404 } 5405 else if ((is_aff_rule(items, itemcnt, "NEEDCOMPOUND", 2) 5406 || is_aff_rule(items, itemcnt, "ONLYINCOMPOUND", 2)) 5407 && aff->af_needcomp == 0) 5408 { 5409 aff->af_needcomp = affitem2flag(aff->af_flagtype, items[1], 5410 fname, lnum); 5411 } 5412 else if (is_aff_rule(items, itemcnt, "COMPOUNDROOT", 2) 5413 && aff->af_comproot == 0) 5414 { 5415 aff->af_comproot = affitem2flag(aff->af_flagtype, items[1], 5416 fname, lnum); 5417 } 5418 else if (is_aff_rule(items, itemcnt, "COMPOUNDFORBIDFLAG", 2) 5419 && aff->af_compforbid == 0) 5420 { 5421 aff->af_compforbid = affitem2flag(aff->af_flagtype, items[1], 5422 fname, lnum); 5423 if (aff->af_pref.ht_used > 0) 5424 smsg((char_u *)_("Defining COMPOUNDFORBIDFLAG after PFX item may give wrong results in %s line %d"), 5425 fname, lnum); 5426 } 5427 else if (is_aff_rule(items, itemcnt, "COMPOUNDPERMITFLAG", 2) 5428 && aff->af_comppermit == 0) 5429 { 5430 aff->af_comppermit = affitem2flag(aff->af_flagtype, items[1], 5431 fname, lnum); 5432 if (aff->af_pref.ht_used > 0) 5433 smsg((char_u *)_("Defining COMPOUNDPERMITFLAG after PFX item may give wrong results in %s line %d"), 5434 fname, lnum); 5435 } 5436 else if (is_aff_rule(items, itemcnt, "COMPOUNDFLAG", 2) 5437 && compflags == NULL) 5438 { 5439 /* Turn flag "c" into COMPOUNDRULE compatible string "c+", 5440 * "Na" into "Na+", "1234" into "1234+". */ 5441 p = getroom(spin, STRLEN(items[1]) + 2, FALSE); 5442 if (p != NULL) 5443 { 5444 STRCPY(p, items[1]); 5445 STRCAT(p, "+"); 5446 compflags = p; 5447 } 5448 } 5449 else if (is_aff_rule(items, itemcnt, "COMPOUNDRULES", 2)) 5450 { 5451 /* We don't use the count, but do check that it's a number and 5452 * not COMPOUNDRULE mistyped. */ 5453 if (atoi((char *)items[1]) == 0) 5454 smsg((char_u *)_("Wrong COMPOUNDRULES value in %s line %d: %s"), 5455 fname, lnum, items[1]); 5456 } 5457 else if (is_aff_rule(items, itemcnt, "COMPOUNDRULE", 2)) 5458 { 5459 /* Concatenate this string to previously defined ones, using a 5460 * slash to separate them. */ 5461 l = (int)STRLEN(items[1]) + 1; 5462 if (compflags != NULL) 5463 l += (int)STRLEN(compflags) + 1; 5464 p = getroom(spin, l, FALSE); 5465 if (p != NULL) 5466 { 5467 if (compflags != NULL) 5468 { 5469 STRCPY(p, compflags); 5470 STRCAT(p, "/"); 5471 } 5472 STRCAT(p, items[1]); 5473 compflags = p; 5474 } 5475 } 5476 else if (is_aff_rule(items, itemcnt, "COMPOUNDWORDMAX", 2) 5477 && compmax == 0) 5478 { 5479 compmax = atoi((char *)items[1]); 5480 if (compmax == 0) 5481 smsg((char_u *)_("Wrong COMPOUNDWORDMAX value in %s line %d: %s"), 5482 fname, lnum, items[1]); 5483 } 5484 else if (is_aff_rule(items, itemcnt, "COMPOUNDMIN", 2) 5485 && compminlen == 0) 5486 { 5487 compminlen = atoi((char *)items[1]); 5488 if (compminlen == 0) 5489 smsg((char_u *)_("Wrong COMPOUNDMIN value in %s line %d: %s"), 5490 fname, lnum, items[1]); 5491 } 5492 else if (is_aff_rule(items, itemcnt, "COMPOUNDSYLMAX", 2) 5493 && compsylmax == 0) 5494 { 5495 compsylmax = atoi((char *)items[1]); 5496 if (compsylmax == 0) 5497 smsg((char_u *)_("Wrong COMPOUNDSYLMAX value in %s line %d: %s"), 5498 fname, lnum, items[1]); 5499 } 5500 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDDUP", 1)) 5501 { 5502 compoptions |= COMP_CHECKDUP; 5503 } 5504 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDREP", 1)) 5505 { 5506 compoptions |= COMP_CHECKREP; 5507 } 5508 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDCASE", 1)) 5509 { 5510 compoptions |= COMP_CHECKCASE; 5511 } 5512 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDTRIPLE", 1)) 5513 { 5514 compoptions |= COMP_CHECKTRIPLE; 5515 } 5516 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDPATTERN", 2)) 5517 { 5518 if (atoi((char *)items[1]) == 0) 5519 smsg((char_u *)_("Wrong CHECKCOMPOUNDPATTERN value in %s line %d: %s"), 5520 fname, lnum, items[1]); 5521 } 5522 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDPATTERN", 3)) 5523 { 5524 garray_T *gap = &spin->si_comppat; 5525 int i; 5526 5527 /* Only add the couple if it isn't already there. */ 5528 for (i = 0; i < gap->ga_len - 1; i += 2) 5529 if (STRCMP(((char_u **)(gap->ga_data))[i], items[1]) == 0 5530 && STRCMP(((char_u **)(gap->ga_data))[i + 1], 5531 items[2]) == 0) 5532 break; 5533 if (i >= gap->ga_len && ga_grow(gap, 2) == OK) 5534 { 5535 ((char_u **)(gap->ga_data))[gap->ga_len++] 5536 = getroom_save(spin, items[1]); 5537 ((char_u **)(gap->ga_data))[gap->ga_len++] 5538 = getroom_save(spin, items[2]); 5539 } 5540 } 5541 else if (is_aff_rule(items, itemcnt, "SYLLABLE", 2) 5542 && syllable == NULL) 5543 { 5544 syllable = getroom_save(spin, items[1]); 5545 } 5546 else if (is_aff_rule(items, itemcnt, "NOBREAK", 1)) 5547 { 5548 spin->si_nobreak = TRUE; 5549 } 5550 else if (is_aff_rule(items, itemcnt, "NOSPLITSUGS", 1)) 5551 { 5552 spin->si_nosplitsugs = TRUE; 5553 } 5554 else if (is_aff_rule(items, itemcnt, "NOSUGFILE", 1)) 5555 { 5556 spin->si_nosugfile = TRUE; 5557 } 5558 else if (is_aff_rule(items, itemcnt, "PFXPOSTPONE", 1)) 5559 { 5560 aff->af_pfxpostpone = TRUE; 5561 } 5562 else if ((STRCMP(items[0], "PFX") == 0 5563 || STRCMP(items[0], "SFX") == 0) 5564 && aff_todo == 0 5565 && itemcnt >= 4) 5566 { 5567 int lasti = 4; 5568 char_u key[AH_KEY_LEN]; 5569 5570 if (*items[0] == 'P') 5571 tp = &aff->af_pref; 5572 else 5573 tp = &aff->af_suff; 5574 5575 /* Myspell allows the same affix name to be used multiple 5576 * times. The affix files that do this have an undocumented 5577 * "S" flag on all but the last block, thus we check for that 5578 * and store it in ah_follows. */ 5579 vim_strncpy(key, items[1], AH_KEY_LEN - 1); 5580 hi = hash_find(tp, key); 5581 if (!HASHITEM_EMPTY(hi)) 5582 { 5583 cur_aff = HI2AH(hi); 5584 if (cur_aff->ah_combine != (*items[2] == 'Y')) 5585 smsg((char_u *)_("Different combining flag in continued affix block in %s line %d: %s"), 5586 fname, lnum, items[1]); 5587 if (!cur_aff->ah_follows) 5588 smsg((char_u *)_("Duplicate affix in %s line %d: %s"), 5589 fname, lnum, items[1]); 5590 } 5591 else 5592 { 5593 /* New affix letter. */ 5594 cur_aff = (affheader_T *)getroom(spin, 5595 sizeof(affheader_T), TRUE); 5596 if (cur_aff == NULL) 5597 break; 5598 cur_aff->ah_flag = affitem2flag(aff->af_flagtype, items[1], 5599 fname, lnum); 5600 if (cur_aff->ah_flag == 0 || STRLEN(items[1]) >= AH_KEY_LEN) 5601 break; 5602 if (cur_aff->ah_flag == aff->af_bad 5603 || cur_aff->ah_flag == aff->af_rare 5604 || cur_aff->ah_flag == aff->af_keepcase 5605 || cur_aff->ah_flag == aff->af_needaffix 5606 || cur_aff->ah_flag == aff->af_circumfix 5607 || cur_aff->ah_flag == aff->af_nosuggest 5608 || cur_aff->ah_flag == aff->af_needcomp 5609 || cur_aff->ah_flag == aff->af_comproot) 5610 smsg((char_u *)_("Affix also used for BAD/RARE/KEEPCASE/NEEDAFFIX/NEEDCOMPOUND/NOSUGGEST in %s line %d: %s"), 5611 fname, lnum, items[1]); 5612 STRCPY(cur_aff->ah_key, items[1]); 5613 hash_add(tp, cur_aff->ah_key); 5614 5615 cur_aff->ah_combine = (*items[2] == 'Y'); 5616 } 5617 5618 /* Check for the "S" flag, which apparently means that another 5619 * block with the same affix name is following. */ 5620 if (itemcnt > lasti && STRCMP(items[lasti], "S") == 0) 5621 { 5622 ++lasti; 5623 cur_aff->ah_follows = TRUE; 5624 } 5625 else 5626 cur_aff->ah_follows = FALSE; 5627 5628 /* Myspell allows extra text after the item, but that might 5629 * mean mistakes go unnoticed. Require a comment-starter. */ 5630 if (itemcnt > lasti && *items[lasti] != '#') 5631 smsg((char_u *)_(e_afftrailing), fname, lnum, items[lasti]); 5632 5633 if (STRCMP(items[2], "Y") != 0 && STRCMP(items[2], "N") != 0) 5634 smsg((char_u *)_("Expected Y or N in %s line %d: %s"), 5635 fname, lnum, items[2]); 5636 5637 if (*items[0] == 'P' && aff->af_pfxpostpone) 5638 { 5639 if (cur_aff->ah_newID == 0) 5640 { 5641 /* Use a new number in the .spl file later, to be able 5642 * to handle multiple .aff files. */ 5643 check_renumber(spin); 5644 cur_aff->ah_newID = ++spin->si_newprefID; 5645 5646 /* We only really use ah_newID if the prefix is 5647 * postponed. We know that only after handling all 5648 * the items. */ 5649 did_postpone_prefix = FALSE; 5650 } 5651 else 5652 /* Did use the ID in a previous block. */ 5653 did_postpone_prefix = TRUE; 5654 } 5655 5656 aff_todo = atoi((char *)items[3]); 5657 } 5658 else if ((STRCMP(items[0], "PFX") == 0 5659 || STRCMP(items[0], "SFX") == 0) 5660 && aff_todo > 0 5661 && STRCMP(cur_aff->ah_key, items[1]) == 0 5662 && itemcnt >= 5) 5663 { 5664 affentry_T *aff_entry; 5665 int upper = FALSE; 5666 int lasti = 5; 5667 5668 /* Myspell allows extra text after the item, but that might 5669 * mean mistakes go unnoticed. Require a comment-starter. 5670 * Hunspell uses a "-" item. */ 5671 if (itemcnt > lasti && *items[lasti] != '#' 5672 && (STRCMP(items[lasti], "-") != 0 5673 || itemcnt != lasti + 1)) 5674 smsg((char_u *)_(e_afftrailing), fname, lnum, items[lasti]); 5675 5676 /* New item for an affix letter. */ 5677 --aff_todo; 5678 aff_entry = (affentry_T *)getroom(spin, 5679 sizeof(affentry_T), TRUE); 5680 if (aff_entry == NULL) 5681 break; 5682 5683 if (STRCMP(items[2], "0") != 0) 5684 aff_entry->ae_chop = getroom_save(spin, items[2]); 5685 if (STRCMP(items[3], "0") != 0) 5686 { 5687 aff_entry->ae_add = getroom_save(spin, items[3]); 5688 5689 /* Recognize flags on the affix: abcd/XYZ */ 5690 aff_entry->ae_flags = vim_strchr(aff_entry->ae_add, '/'); 5691 if (aff_entry->ae_flags != NULL) 5692 { 5693 *aff_entry->ae_flags++ = NUL; 5694 aff_process_flags(aff, aff_entry); 5695 } 5696 } 5697 5698 /* Don't use an affix entry with non-ASCII characters when 5699 * "spin->si_ascii" is TRUE. */ 5700 if (!spin->si_ascii || !(has_non_ascii(aff_entry->ae_chop) 5701 || has_non_ascii(aff_entry->ae_add))) 5702 { 5703 aff_entry->ae_next = cur_aff->ah_first; 5704 cur_aff->ah_first = aff_entry; 5705 5706 if (STRCMP(items[4], ".") != 0) 5707 { 5708 char_u buf[MAXLINELEN]; 5709 5710 aff_entry->ae_cond = getroom_save(spin, items[4]); 5711 if (*items[0] == 'P') 5712 sprintf((char *)buf, "^%s", items[4]); 5713 else 5714 sprintf((char *)buf, "%s$", items[4]); 5715 aff_entry->ae_prog = vim_regcomp(buf, 5716 RE_MAGIC + RE_STRING + RE_STRICT); 5717 if (aff_entry->ae_prog == NULL) 5718 smsg((char_u *)_("Broken condition in %s line %d: %s"), 5719 fname, lnum, items[4]); 5720 } 5721 5722 /* For postponed prefixes we need an entry in si_prefcond 5723 * for the condition. Use an existing one if possible. 5724 * Can't be done for an affix with flags, ignoring 5725 * COMPOUNDFORBIDFLAG and COMPOUNDPERMITFLAG. */ 5726 if (*items[0] == 'P' && aff->af_pfxpostpone 5727 && aff_entry->ae_flags == NULL) 5728 { 5729 /* When the chop string is one lower-case letter and 5730 * the add string ends in the upper-case letter we set 5731 * the "upper" flag, clear "ae_chop" and remove the 5732 * letters from "ae_add". The condition must either 5733 * be empty or start with the same letter. */ 5734 if (aff_entry->ae_chop != NULL 5735 && aff_entry->ae_add != NULL 5736#ifdef FEAT_MBYTE 5737 && aff_entry->ae_chop[(*mb_ptr2len)( 5738 aff_entry->ae_chop)] == NUL 5739#else 5740 && aff_entry->ae_chop[1] == NUL 5741#endif 5742 ) 5743 { 5744 int c, c_up; 5745 5746 c = PTR2CHAR(aff_entry->ae_chop); 5747 c_up = SPELL_TOUPPER(c); 5748 if (c_up != c 5749 && (aff_entry->ae_cond == NULL 5750 || PTR2CHAR(aff_entry->ae_cond) == c)) 5751 { 5752 p = aff_entry->ae_add 5753 + STRLEN(aff_entry->ae_add); 5754 mb_ptr_back(aff_entry->ae_add, p); 5755 if (PTR2CHAR(p) == c_up) 5756 { 5757 upper = TRUE; 5758 aff_entry->ae_chop = NULL; 5759 *p = NUL; 5760 5761 /* The condition is matched with the 5762 * actual word, thus must check for the 5763 * upper-case letter. */ 5764 if (aff_entry->ae_cond != NULL) 5765 { 5766 char_u buf[MAXLINELEN]; 5767#ifdef FEAT_MBYTE 5768 if (has_mbyte) 5769 { 5770 onecap_copy(items[4], buf, TRUE); 5771 aff_entry->ae_cond = getroom_save( 5772 spin, buf); 5773 } 5774 else 5775#endif 5776 *aff_entry->ae_cond = c_up; 5777 if (aff_entry->ae_cond != NULL) 5778 { 5779 sprintf((char *)buf, "^%s", 5780 aff_entry->ae_cond); 5781 vim_free(aff_entry->ae_prog); 5782 aff_entry->ae_prog = vim_regcomp( 5783 buf, RE_MAGIC + RE_STRING); 5784 } 5785 } 5786 } 5787 } 5788 } 5789 5790 if (aff_entry->ae_chop == NULL 5791 && aff_entry->ae_flags == NULL) 5792 { 5793 int idx; 5794 char_u **pp; 5795 int n; 5796 5797 /* Find a previously used condition. */ 5798 for (idx = spin->si_prefcond.ga_len - 1; idx >= 0; 5799 --idx) 5800 { 5801 p = ((char_u **)spin->si_prefcond.ga_data)[idx]; 5802 if (str_equal(p, aff_entry->ae_cond)) 5803 break; 5804 } 5805 if (idx < 0 && ga_grow(&spin->si_prefcond, 1) == OK) 5806 { 5807 /* Not found, add a new condition. */ 5808 idx = spin->si_prefcond.ga_len++; 5809 pp = ((char_u **)spin->si_prefcond.ga_data) 5810 + idx; 5811 if (aff_entry->ae_cond == NULL) 5812 *pp = NULL; 5813 else 5814 *pp = getroom_save(spin, 5815 aff_entry->ae_cond); 5816 } 5817 5818 /* Add the prefix to the prefix tree. */ 5819 if (aff_entry->ae_add == NULL) 5820 p = (char_u *)""; 5821 else 5822 p = aff_entry->ae_add; 5823 5824 /* PFX_FLAGS is a negative number, so that 5825 * tree_add_word() knows this is the prefix tree. */ 5826 n = PFX_FLAGS; 5827 if (!cur_aff->ah_combine) 5828 n |= WFP_NC; 5829 if (upper) 5830 n |= WFP_UP; 5831 if (aff_entry->ae_comppermit) 5832 n |= WFP_COMPPERMIT; 5833 if (aff_entry->ae_compforbid) 5834 n |= WFP_COMPFORBID; 5835 tree_add_word(spin, p, spin->si_prefroot, n, 5836 idx, cur_aff->ah_newID); 5837 did_postpone_prefix = TRUE; 5838 } 5839 5840 /* Didn't actually use ah_newID, backup si_newprefID. */ 5841 if (aff_todo == 0 && !did_postpone_prefix) 5842 { 5843 --spin->si_newprefID; 5844 cur_aff->ah_newID = 0; 5845 } 5846 } 5847 } 5848 } 5849 else if (is_aff_rule(items, itemcnt, "FOL", 2) && fol == NULL) 5850 { 5851 fol = vim_strsave(items[1]); 5852 } 5853 else if (is_aff_rule(items, itemcnt, "LOW", 2) && low == NULL) 5854 { 5855 low = vim_strsave(items[1]); 5856 } 5857 else if (is_aff_rule(items, itemcnt, "UPP", 2) && upp == NULL) 5858 { 5859 upp = vim_strsave(items[1]); 5860 } 5861 else if (is_aff_rule(items, itemcnt, "REP", 2) 5862 || is_aff_rule(items, itemcnt, "REPSAL", 2)) 5863 { 5864 /* Ignore REP/REPSAL count */; 5865 if (!isdigit(*items[1])) 5866 smsg((char_u *)_("Expected REP(SAL) count in %s line %d"), 5867 fname, lnum); 5868 } 5869 else if ((STRCMP(items[0], "REP") == 0 5870 || STRCMP(items[0], "REPSAL") == 0) 5871 && itemcnt >= 3) 5872 { 5873 /* REP/REPSAL item */ 5874 /* Myspell ignores extra arguments, we require it starts with 5875 * # to detect mistakes. */ 5876 if (itemcnt > 3 && items[3][0] != '#') 5877 smsg((char_u *)_(e_afftrailing), fname, lnum, items[3]); 5878 if (items[0][3] == 'S' ? do_repsal : do_rep) 5879 { 5880 /* Replace underscore with space (can't include a space 5881 * directly). */ 5882 for (p = items[1]; *p != NUL; mb_ptr_adv(p)) 5883 if (*p == '_') 5884 *p = ' '; 5885 for (p = items[2]; *p != NUL; mb_ptr_adv(p)) 5886 if (*p == '_') 5887 *p = ' '; 5888 add_fromto(spin, items[0][3] == 'S' 5889 ? &spin->si_repsal 5890 : &spin->si_rep, items[1], items[2]); 5891 } 5892 } 5893 else if (is_aff_rule(items, itemcnt, "MAP", 2)) 5894 { 5895 /* MAP item or count */ 5896 if (!found_map) 5897 { 5898 /* First line contains the count. */ 5899 found_map = TRUE; 5900 if (!isdigit(*items[1])) 5901 smsg((char_u *)_("Expected MAP count in %s line %d"), 5902 fname, lnum); 5903 } 5904 else if (do_mapline) 5905 { 5906 int c; 5907 5908 /* Check that every character appears only once. */ 5909 for (p = items[1]; *p != NUL; ) 5910 { 5911#ifdef FEAT_MBYTE 5912 c = mb_ptr2char_adv(&p); 5913#else 5914 c = *p++; 5915#endif 5916 if ((spin->si_map.ga_len > 0 5917 && vim_strchr(spin->si_map.ga_data, c) 5918 != NULL) 5919 || vim_strchr(p, c) != NULL) 5920 smsg((char_u *)_("Duplicate character in MAP in %s line %d"), 5921 fname, lnum); 5922 } 5923 5924 /* We simply concatenate all the MAP strings, separated by 5925 * slashes. */ 5926 ga_concat(&spin->si_map, items[1]); 5927 ga_append(&spin->si_map, '/'); 5928 } 5929 } 5930 /* Accept "SAL from to" and "SAL from to #comment". */ 5931 else if (is_aff_rule(items, itemcnt, "SAL", 3)) 5932 { 5933 if (do_sal) 5934 { 5935 /* SAL item (sounds-a-like) 5936 * Either one of the known keys or a from-to pair. */ 5937 if (STRCMP(items[1], "followup") == 0) 5938 spin->si_followup = sal_to_bool(items[2]); 5939 else if (STRCMP(items[1], "collapse_result") == 0) 5940 spin->si_collapse = sal_to_bool(items[2]); 5941 else if (STRCMP(items[1], "remove_accents") == 0) 5942 spin->si_rem_accents = sal_to_bool(items[2]); 5943 else 5944 /* when "to" is "_" it means empty */ 5945 add_fromto(spin, &spin->si_sal, items[1], 5946 STRCMP(items[2], "_") == 0 ? (char_u *)"" 5947 : items[2]); 5948 } 5949 } 5950 else if (is_aff_rule(items, itemcnt, "SOFOFROM", 2) 5951 && sofofrom == NULL) 5952 { 5953 sofofrom = getroom_save(spin, items[1]); 5954 } 5955 else if (is_aff_rule(items, itemcnt, "SOFOTO", 2) 5956 && sofoto == NULL) 5957 { 5958 sofoto = getroom_save(spin, items[1]); 5959 } 5960 else if (STRCMP(items[0], "COMMON") == 0) 5961 { 5962 int i; 5963 5964 for (i = 1; i < itemcnt; ++i) 5965 { 5966 if (HASHITEM_EMPTY(hash_find(&spin->si_commonwords, 5967 items[i]))) 5968 { 5969 p = vim_strsave(items[i]); 5970 if (p == NULL) 5971 break; 5972 hash_add(&spin->si_commonwords, p); 5973 } 5974 } 5975 } 5976 else 5977 smsg((char_u *)_("Unrecognized or duplicate item in %s line %d: %s"), 5978 fname, lnum, items[0]); 5979 } 5980 } 5981 5982 if (fol != NULL || low != NULL || upp != NULL) 5983 { 5984 if (spin->si_clear_chartab) 5985 { 5986 /* Clear the char type tables, don't want to use any of the 5987 * currently used spell properties. */ 5988 init_spell_chartab(); 5989 spin->si_clear_chartab = FALSE; 5990 } 5991 5992 /* 5993 * Don't write a word table for an ASCII file, so that we don't check 5994 * for conflicts with a word table that matches 'encoding'. 5995 * Don't write one for utf-8 either, we use utf_*() and 5996 * mb_get_class(), the list of chars in the file will be incomplete. 5997 */ 5998 if (!spin->si_ascii 5999#ifdef FEAT_MBYTE 6000 && !enc_utf8 6001#endif 6002 ) 6003 { 6004 if (fol == NULL || low == NULL || upp == NULL) 6005 smsg((char_u *)_("Missing FOL/LOW/UPP line in %s"), fname); 6006 else 6007 (void)set_spell_chartab(fol, low, upp); 6008 } 6009 6010 vim_free(fol); 6011 vim_free(low); 6012 vim_free(upp); 6013 } 6014 6015 /* Use compound specifications of the .aff file for the spell info. */ 6016 if (compmax != 0) 6017 { 6018 aff_check_number(spin->si_compmax, compmax, "COMPOUNDWORDMAX"); 6019 spin->si_compmax = compmax; 6020 } 6021 6022 if (compminlen != 0) 6023 { 6024 aff_check_number(spin->si_compminlen, compminlen, "COMPOUNDMIN"); 6025 spin->si_compminlen = compminlen; 6026 } 6027 6028 if (compsylmax != 0) 6029 { 6030 if (syllable == NULL) 6031 smsg((char_u *)_("COMPOUNDSYLMAX used without SYLLABLE")); 6032 aff_check_number(spin->si_compsylmax, compsylmax, "COMPOUNDSYLMAX"); 6033 spin->si_compsylmax = compsylmax; 6034 } 6035 6036 if (compoptions != 0) 6037 { 6038 aff_check_number(spin->si_compoptions, compoptions, "COMPOUND options"); 6039 spin->si_compoptions |= compoptions; 6040 } 6041 6042 if (compflags != NULL) 6043 process_compflags(spin, aff, compflags); 6044 6045 /* Check that we didn't use too many renumbered flags. */ 6046 if (spin->si_newcompID < spin->si_newprefID) 6047 { 6048 if (spin->si_newcompID == 127 || spin->si_newcompID == 255) 6049 MSG(_("Too many postponed prefixes")); 6050 else if (spin->si_newprefID == 0 || spin->si_newprefID == 127) 6051 MSG(_("Too many compound flags")); 6052 else 6053 MSG(_("Too many postponed prefixes and/or compound flags")); 6054 } 6055 6056 if (syllable != NULL) 6057 { 6058 aff_check_string(spin->si_syllable, syllable, "SYLLABLE"); 6059 spin->si_syllable = syllable; 6060 } 6061 6062 if (sofofrom != NULL || sofoto != NULL) 6063 { 6064 if (sofofrom == NULL || sofoto == NULL) 6065 smsg((char_u *)_("Missing SOFO%s line in %s"), 6066 sofofrom == NULL ? "FROM" : "TO", fname); 6067 else if (spin->si_sal.ga_len > 0) 6068 smsg((char_u *)_("Both SAL and SOFO lines in %s"), fname); 6069 else 6070 { 6071 aff_check_string(spin->si_sofofr, sofofrom, "SOFOFROM"); 6072 aff_check_string(spin->si_sofoto, sofoto, "SOFOTO"); 6073 spin->si_sofofr = sofofrom; 6074 spin->si_sofoto = sofoto; 6075 } 6076 } 6077 6078 if (midword != NULL) 6079 { 6080 aff_check_string(spin->si_midword, midword, "MIDWORD"); 6081 spin->si_midword = midword; 6082 } 6083 6084 vim_free(pc); 6085 fclose(fd); 6086 return aff; 6087} 6088 6089/* 6090 * Return TRUE when items[0] equals "rulename", there are "mincount" items or 6091 * a comment is following after item "mincount". 6092 */ 6093 static int 6094is_aff_rule(items, itemcnt, rulename, mincount) 6095 char_u **items; 6096 int itemcnt; 6097 char *rulename; 6098 int mincount; 6099{ 6100 return (STRCMP(items[0], rulename) == 0 6101 && (itemcnt == mincount 6102 || (itemcnt > mincount && items[mincount][0] == '#'))); 6103} 6104 6105/* 6106 * For affix "entry" move COMPOUNDFORBIDFLAG and COMPOUNDPERMITFLAG from 6107 * ae_flags to ae_comppermit and ae_compforbid. 6108 */ 6109 static void 6110aff_process_flags(affile, entry) 6111 afffile_T *affile; 6112 affentry_T *entry; 6113{ 6114 char_u *p; 6115 char_u *prevp; 6116 unsigned flag; 6117 6118 if (entry->ae_flags != NULL 6119 && (affile->af_compforbid != 0 || affile->af_comppermit != 0)) 6120 { 6121 for (p = entry->ae_flags; *p != NUL; ) 6122 { 6123 prevp = p; 6124 flag = get_affitem(affile->af_flagtype, &p); 6125 if (flag == affile->af_comppermit || flag == affile->af_compforbid) 6126 { 6127 STRMOVE(prevp, p); 6128 p = prevp; 6129 if (flag == affile->af_comppermit) 6130 entry->ae_comppermit = TRUE; 6131 else 6132 entry->ae_compforbid = TRUE; 6133 } 6134 if (affile->af_flagtype == AFT_NUM && *p == ',') 6135 ++p; 6136 } 6137 if (*entry->ae_flags == NUL) 6138 entry->ae_flags = NULL; /* nothing left */ 6139 } 6140} 6141 6142/* 6143 * Return TRUE if "s" is the name of an info item in the affix file. 6144 */ 6145 static int 6146spell_info_item(s) 6147 char_u *s; 6148{ 6149 return STRCMP(s, "NAME") == 0 6150 || STRCMP(s, "HOME") == 0 6151 || STRCMP(s, "VERSION") == 0 6152 || STRCMP(s, "AUTHOR") == 0 6153 || STRCMP(s, "EMAIL") == 0 6154 || STRCMP(s, "COPYRIGHT") == 0; 6155} 6156 6157/* 6158 * Turn an affix flag name into a number, according to the FLAG type. 6159 * returns zero for failure. 6160 */ 6161 static unsigned 6162affitem2flag(flagtype, item, fname, lnum) 6163 int flagtype; 6164 char_u *item; 6165 char_u *fname; 6166 int lnum; 6167{ 6168 unsigned res; 6169 char_u *p = item; 6170 6171 res = get_affitem(flagtype, &p); 6172 if (res == 0) 6173 { 6174 if (flagtype == AFT_NUM) 6175 smsg((char_u *)_("Flag is not a number in %s line %d: %s"), 6176 fname, lnum, item); 6177 else 6178 smsg((char_u *)_("Illegal flag in %s line %d: %s"), 6179 fname, lnum, item); 6180 } 6181 if (*p != NUL) 6182 { 6183 smsg((char_u *)_(e_affname), fname, lnum, item); 6184 return 0; 6185 } 6186 6187 return res; 6188} 6189 6190/* 6191 * Get one affix name from "*pp" and advance the pointer. 6192 * Returns zero for an error, still advances the pointer then. 6193 */ 6194 static unsigned 6195get_affitem(flagtype, pp) 6196 int flagtype; 6197 char_u **pp; 6198{ 6199 int res; 6200 6201 if (flagtype == AFT_NUM) 6202 { 6203 if (!VIM_ISDIGIT(**pp)) 6204 { 6205 ++*pp; /* always advance, avoid getting stuck */ 6206 return 0; 6207 } 6208 res = getdigits(pp); 6209 } 6210 else 6211 { 6212#ifdef FEAT_MBYTE 6213 res = mb_ptr2char_adv(pp); 6214#else 6215 res = *(*pp)++; 6216#endif 6217 if (flagtype == AFT_LONG || (flagtype == AFT_CAPLONG 6218 && res >= 'A' && res <= 'Z')) 6219 { 6220 if (**pp == NUL) 6221 return 0; 6222#ifdef FEAT_MBYTE 6223 res = mb_ptr2char_adv(pp) + (res << 16); 6224#else 6225 res = *(*pp)++ + (res << 16); 6226#endif 6227 } 6228 } 6229 return res; 6230} 6231 6232/* 6233 * Process the "compflags" string used in an affix file and append it to 6234 * spin->si_compflags. 6235 * The processing involves changing the affix names to ID numbers, so that 6236 * they fit in one byte. 6237 */ 6238 static void 6239process_compflags(spin, aff, compflags) 6240 spellinfo_T *spin; 6241 afffile_T *aff; 6242 char_u *compflags; 6243{ 6244 char_u *p; 6245 char_u *prevp; 6246 unsigned flag; 6247 compitem_T *ci; 6248 int id; 6249 int len; 6250 char_u *tp; 6251 char_u key[AH_KEY_LEN]; 6252 hashitem_T *hi; 6253 6254 /* Make room for the old and the new compflags, concatenated with a / in 6255 * between. Processing it makes it shorter, but we don't know by how 6256 * much, thus allocate the maximum. */ 6257 len = (int)STRLEN(compflags) + 1; 6258 if (spin->si_compflags != NULL) 6259 len += (int)STRLEN(spin->si_compflags) + 1; 6260 p = getroom(spin, len, FALSE); 6261 if (p == NULL) 6262 return; 6263 if (spin->si_compflags != NULL) 6264 { 6265 STRCPY(p, spin->si_compflags); 6266 STRCAT(p, "/"); 6267 } 6268 spin->si_compflags = p; 6269 tp = p + STRLEN(p); 6270 6271 for (p = compflags; *p != NUL; ) 6272 { 6273 if (vim_strchr((char_u *)"/*+[]", *p) != NULL) 6274 /* Copy non-flag characters directly. */ 6275 *tp++ = *p++; 6276 else 6277 { 6278 /* First get the flag number, also checks validity. */ 6279 prevp = p; 6280 flag = get_affitem(aff->af_flagtype, &p); 6281 if (flag != 0) 6282 { 6283 /* Find the flag in the hashtable. If it was used before, use 6284 * the existing ID. Otherwise add a new entry. */ 6285 vim_strncpy(key, prevp, p - prevp); 6286 hi = hash_find(&aff->af_comp, key); 6287 if (!HASHITEM_EMPTY(hi)) 6288 id = HI2CI(hi)->ci_newID; 6289 else 6290 { 6291 ci = (compitem_T *)getroom(spin, sizeof(compitem_T), TRUE); 6292 if (ci == NULL) 6293 break; 6294 STRCPY(ci->ci_key, key); 6295 ci->ci_flag = flag; 6296 /* Avoid using a flag ID that has a special meaning in a 6297 * regexp (also inside []). */ 6298 do 6299 { 6300 check_renumber(spin); 6301 id = spin->si_newcompID--; 6302 } while (vim_strchr((char_u *)"/+*[]\\-^", id) != NULL); 6303 ci->ci_newID = id; 6304 hash_add(&aff->af_comp, ci->ci_key); 6305 } 6306 *tp++ = id; 6307 } 6308 if (aff->af_flagtype == AFT_NUM && *p == ',') 6309 ++p; 6310 } 6311 } 6312 6313 *tp = NUL; 6314} 6315 6316/* 6317 * Check that the new IDs for postponed affixes and compounding don't overrun 6318 * each other. We have almost 255 available, but start at 0-127 to avoid 6319 * using two bytes for utf-8. When the 0-127 range is used up go to 128-255. 6320 * When that is used up an error message is given. 6321 */ 6322 static void 6323check_renumber(spin) 6324 spellinfo_T *spin; 6325{ 6326 if (spin->si_newprefID == spin->si_newcompID && spin->si_newcompID < 128) 6327 { 6328 spin->si_newprefID = 127; 6329 spin->si_newcompID = 255; 6330 } 6331} 6332 6333/* 6334 * Return TRUE if flag "flag" appears in affix list "afflist". 6335 */ 6336 static int 6337flag_in_afflist(flagtype, afflist, flag) 6338 int flagtype; 6339 char_u *afflist; 6340 unsigned flag; 6341{ 6342 char_u *p; 6343 unsigned n; 6344 6345 switch (flagtype) 6346 { 6347 case AFT_CHAR: 6348 return vim_strchr(afflist, flag) != NULL; 6349 6350 case AFT_CAPLONG: 6351 case AFT_LONG: 6352 for (p = afflist; *p != NUL; ) 6353 { 6354#ifdef FEAT_MBYTE 6355 n = mb_ptr2char_adv(&p); 6356#else 6357 n = *p++; 6358#endif 6359 if ((flagtype == AFT_LONG || (n >= 'A' && n <= 'Z')) 6360 && *p != NUL) 6361#ifdef FEAT_MBYTE 6362 n = mb_ptr2char_adv(&p) + (n << 16); 6363#else 6364 n = *p++ + (n << 16); 6365#endif 6366 if (n == flag) 6367 return TRUE; 6368 } 6369 break; 6370 6371 case AFT_NUM: 6372 for (p = afflist; *p != NUL; ) 6373 { 6374 n = getdigits(&p); 6375 if (n == flag) 6376 return TRUE; 6377 if (*p != NUL) /* skip over comma */ 6378 ++p; 6379 } 6380 break; 6381 } 6382 return FALSE; 6383} 6384 6385/* 6386 * Give a warning when "spinval" and "affval" numbers are set and not the same. 6387 */ 6388 static void 6389aff_check_number(spinval, affval, name) 6390 int spinval; 6391 int affval; 6392 char *name; 6393{ 6394 if (spinval != 0 && spinval != affval) 6395 smsg((char_u *)_("%s value differs from what is used in another .aff file"), name); 6396} 6397 6398/* 6399 * Give a warning when "spinval" and "affval" strings are set and not the same. 6400 */ 6401 static void 6402aff_check_string(spinval, affval, name) 6403 char_u *spinval; 6404 char_u *affval; 6405 char *name; 6406{ 6407 if (spinval != NULL && STRCMP(spinval, affval) != 0) 6408 smsg((char_u *)_("%s value differs from what is used in another .aff file"), name); 6409} 6410 6411/* 6412 * Return TRUE if strings "s1" and "s2" are equal. Also consider both being 6413 * NULL as equal. 6414 */ 6415 static int 6416str_equal(s1, s2) 6417 char_u *s1; 6418 char_u *s2; 6419{ 6420 if (s1 == NULL || s2 == NULL) 6421 return s1 == s2; 6422 return STRCMP(s1, s2) == 0; 6423} 6424 6425/* 6426 * Add a from-to item to "gap". Used for REP and SAL items. 6427 * They are stored case-folded. 6428 */ 6429 static void 6430add_fromto(spin, gap, from, to) 6431 spellinfo_T *spin; 6432 garray_T *gap; 6433 char_u *from; 6434 char_u *to; 6435{ 6436 fromto_T *ftp; 6437 char_u word[MAXWLEN]; 6438 6439 if (ga_grow(gap, 1) == OK) 6440 { 6441 ftp = ((fromto_T *)gap->ga_data) + gap->ga_len; 6442 (void)spell_casefold(from, (int)STRLEN(from), word, MAXWLEN); 6443 ftp->ft_from = getroom_save(spin, word); 6444 (void)spell_casefold(to, (int)STRLEN(to), word, MAXWLEN); 6445 ftp->ft_to = getroom_save(spin, word); 6446 ++gap->ga_len; 6447 } 6448} 6449 6450/* 6451 * Convert a boolean argument in a SAL line to TRUE or FALSE; 6452 */ 6453 static int 6454sal_to_bool(s) 6455 char_u *s; 6456{ 6457 return STRCMP(s, "1") == 0 || STRCMP(s, "true") == 0; 6458} 6459 6460/* 6461 * Return TRUE if string "s" contains a non-ASCII character (128 or higher). 6462 * When "s" is NULL FALSE is returned. 6463 */ 6464 static int 6465has_non_ascii(s) 6466 char_u *s; 6467{ 6468 char_u *p; 6469 6470 if (s != NULL) 6471 for (p = s; *p != NUL; ++p) 6472 if (*p >= 128) 6473 return TRUE; 6474 return FALSE; 6475} 6476 6477/* 6478 * Free the structure filled by spell_read_aff(). 6479 */ 6480 static void 6481spell_free_aff(aff) 6482 afffile_T *aff; 6483{ 6484 hashtab_T *ht; 6485 hashitem_T *hi; 6486 int todo; 6487 affheader_T *ah; 6488 affentry_T *ae; 6489 6490 vim_free(aff->af_enc); 6491 6492 /* All this trouble to free the "ae_prog" items... */ 6493 for (ht = &aff->af_pref; ; ht = &aff->af_suff) 6494 { 6495 todo = (int)ht->ht_used; 6496 for (hi = ht->ht_array; todo > 0; ++hi) 6497 { 6498 if (!HASHITEM_EMPTY(hi)) 6499 { 6500 --todo; 6501 ah = HI2AH(hi); 6502 for (ae = ah->ah_first; ae != NULL; ae = ae->ae_next) 6503 vim_free(ae->ae_prog); 6504 } 6505 } 6506 if (ht == &aff->af_suff) 6507 break; 6508 } 6509 6510 hash_clear(&aff->af_pref); 6511 hash_clear(&aff->af_suff); 6512 hash_clear(&aff->af_comp); 6513} 6514 6515/* 6516 * Read dictionary file "fname". 6517 * Returns OK or FAIL; 6518 */ 6519 static int 6520spell_read_dic(spin, fname, affile) 6521 spellinfo_T *spin; 6522 char_u *fname; 6523 afffile_T *affile; 6524{ 6525 hashtab_T ht; 6526 char_u line[MAXLINELEN]; 6527 char_u *p; 6528 char_u *afflist; 6529 char_u store_afflist[MAXWLEN]; 6530 int pfxlen; 6531 int need_affix; 6532 char_u *dw; 6533 char_u *pc; 6534 char_u *w; 6535 int l; 6536 hash_T hash; 6537 hashitem_T *hi; 6538 FILE *fd; 6539 int lnum = 1; 6540 int non_ascii = 0; 6541 int retval = OK; 6542 char_u message[MAXLINELEN + MAXWLEN]; 6543 int flags; 6544 int duplicate = 0; 6545 6546 /* 6547 * Open the file. 6548 */ 6549 fd = mch_fopen((char *)fname, "r"); 6550 if (fd == NULL) 6551 { 6552 EMSG2(_(e_notopen), fname); 6553 return FAIL; 6554 } 6555 6556 /* The hashtable is only used to detect duplicated words. */ 6557 hash_init(&ht); 6558 6559 vim_snprintf((char *)IObuff, IOSIZE, 6560 _("Reading dictionary file %s ..."), fname); 6561 spell_message(spin, IObuff); 6562 6563 /* start with a message for the first line */ 6564 spin->si_msg_count = 999999; 6565 6566 /* Read and ignore the first line: word count. */ 6567 (void)vim_fgets(line, MAXLINELEN, fd); 6568 if (!vim_isdigit(*skipwhite(line))) 6569 EMSG2(_("E760: No word count in %s"), fname); 6570 6571 /* 6572 * Read all the lines in the file one by one. 6573 * The words are converted to 'encoding' here, before being added to 6574 * the hashtable. 6575 */ 6576 while (!vim_fgets(line, MAXLINELEN, fd) && !got_int) 6577 { 6578 line_breakcheck(); 6579 ++lnum; 6580 if (line[0] == '#' || line[0] == '/') 6581 continue; /* comment line */ 6582 6583 /* Remove CR, LF and white space from the end. White space halfway 6584 * the word is kept to allow e.g., "et al.". */ 6585 l = (int)STRLEN(line); 6586 while (l > 0 && line[l - 1] <= ' ') 6587 --l; 6588 if (l == 0) 6589 continue; /* empty line */ 6590 line[l] = NUL; 6591 6592#ifdef FEAT_MBYTE 6593 /* Convert from "SET" to 'encoding' when needed. */ 6594 if (spin->si_conv.vc_type != CONV_NONE) 6595 { 6596 pc = string_convert(&spin->si_conv, line, NULL); 6597 if (pc == NULL) 6598 { 6599 smsg((char_u *)_("Conversion failure for word in %s line %d: %s"), 6600 fname, lnum, line); 6601 continue; 6602 } 6603 w = pc; 6604 } 6605 else 6606#endif 6607 { 6608 pc = NULL; 6609 w = line; 6610 } 6611 6612 /* Truncate the word at the "/", set "afflist" to what follows. 6613 * Replace "\/" by "/" and "\\" by "\". */ 6614 afflist = NULL; 6615 for (p = w; *p != NUL; mb_ptr_adv(p)) 6616 { 6617 if (*p == '\\' && (p[1] == '\\' || p[1] == '/')) 6618 STRMOVE(p, p + 1); 6619 else if (*p == '/') 6620 { 6621 *p = NUL; 6622 afflist = p + 1; 6623 break; 6624 } 6625 } 6626 6627 /* Skip non-ASCII words when "spin->si_ascii" is TRUE. */ 6628 if (spin->si_ascii && has_non_ascii(w)) 6629 { 6630 ++non_ascii; 6631 vim_free(pc); 6632 continue; 6633 } 6634 6635 /* This takes time, print a message every 10000 words. */ 6636 if (spin->si_verbose && spin->si_msg_count > 10000) 6637 { 6638 spin->si_msg_count = 0; 6639 vim_snprintf((char *)message, sizeof(message), 6640 _("line %6d, word %6d - %s"), 6641 lnum, spin->si_foldwcount + spin->si_keepwcount, w); 6642 msg_start(); 6643 msg_puts_long_attr(message, 0); 6644 msg_clr_eos(); 6645 msg_didout = FALSE; 6646 msg_col = 0; 6647 out_flush(); 6648 } 6649 6650 /* Store the word in the hashtable to be able to find duplicates. */ 6651 dw = (char_u *)getroom_save(spin, w); 6652 if (dw == NULL) 6653 { 6654 retval = FAIL; 6655 vim_free(pc); 6656 break; 6657 } 6658 6659 hash = hash_hash(dw); 6660 hi = hash_lookup(&ht, dw, hash); 6661 if (!HASHITEM_EMPTY(hi)) 6662 { 6663 if (p_verbose > 0) 6664 smsg((char_u *)_("Duplicate word in %s line %d: %s"), 6665 fname, lnum, dw); 6666 else if (duplicate == 0) 6667 smsg((char_u *)_("First duplicate word in %s line %d: %s"), 6668 fname, lnum, dw); 6669 ++duplicate; 6670 } 6671 else 6672 hash_add_item(&ht, hi, dw, hash); 6673 6674 flags = 0; 6675 store_afflist[0] = NUL; 6676 pfxlen = 0; 6677 need_affix = FALSE; 6678 if (afflist != NULL) 6679 { 6680 /* Extract flags from the affix list. */ 6681 flags |= get_affix_flags(affile, afflist); 6682 6683 if (affile->af_needaffix != 0 && flag_in_afflist( 6684 affile->af_flagtype, afflist, affile->af_needaffix)) 6685 need_affix = TRUE; 6686 6687 if (affile->af_pfxpostpone) 6688 /* Need to store the list of prefix IDs with the word. */ 6689 pfxlen = get_pfxlist(affile, afflist, store_afflist); 6690 6691 if (spin->si_compflags != NULL) 6692 /* Need to store the list of compound flags with the word. 6693 * Concatenate them to the list of prefix IDs. */ 6694 get_compflags(affile, afflist, store_afflist + pfxlen); 6695 } 6696 6697 /* Add the word to the word tree(s). */ 6698 if (store_word(spin, dw, flags, spin->si_region, 6699 store_afflist, need_affix) == FAIL) 6700 retval = FAIL; 6701 6702 if (afflist != NULL) 6703 { 6704 /* Find all matching suffixes and add the resulting words. 6705 * Additionally do matching prefixes that combine. */ 6706 if (store_aff_word(spin, dw, afflist, affile, 6707 &affile->af_suff, &affile->af_pref, 6708 CONDIT_SUF, flags, store_afflist, pfxlen) == FAIL) 6709 retval = FAIL; 6710 6711 /* Find all matching prefixes and add the resulting words. */ 6712 if (store_aff_word(spin, dw, afflist, affile, 6713 &affile->af_pref, NULL, 6714 CONDIT_SUF, flags, store_afflist, pfxlen) == FAIL) 6715 retval = FAIL; 6716 } 6717 6718 vim_free(pc); 6719 } 6720 6721 if (duplicate > 0) 6722 smsg((char_u *)_("%d duplicate word(s) in %s"), duplicate, fname); 6723 if (spin->si_ascii && non_ascii > 0) 6724 smsg((char_u *)_("Ignored %d word(s) with non-ASCII characters in %s"), 6725 non_ascii, fname); 6726 hash_clear(&ht); 6727 6728 fclose(fd); 6729 return retval; 6730} 6731 6732/* 6733 * Check for affix flags in "afflist" that are turned into word flags. 6734 * Return WF_ flags. 6735 */ 6736 static int 6737get_affix_flags(affile, afflist) 6738 afffile_T *affile; 6739 char_u *afflist; 6740{ 6741 int flags = 0; 6742 6743 if (affile->af_keepcase != 0 && flag_in_afflist( 6744 affile->af_flagtype, afflist, affile->af_keepcase)) 6745 flags |= WF_KEEPCAP | WF_FIXCAP; 6746 if (affile->af_rare != 0 && flag_in_afflist( 6747 affile->af_flagtype, afflist, affile->af_rare)) 6748 flags |= WF_RARE; 6749 if (affile->af_bad != 0 && flag_in_afflist( 6750 affile->af_flagtype, afflist, affile->af_bad)) 6751 flags |= WF_BANNED; 6752 if (affile->af_needcomp != 0 && flag_in_afflist( 6753 affile->af_flagtype, afflist, affile->af_needcomp)) 6754 flags |= WF_NEEDCOMP; 6755 if (affile->af_comproot != 0 && flag_in_afflist( 6756 affile->af_flagtype, afflist, affile->af_comproot)) 6757 flags |= WF_COMPROOT; 6758 if (affile->af_nosuggest != 0 && flag_in_afflist( 6759 affile->af_flagtype, afflist, affile->af_nosuggest)) 6760 flags |= WF_NOSUGGEST; 6761 return flags; 6762} 6763 6764/* 6765 * Get the list of prefix IDs from the affix list "afflist". 6766 * Used for PFXPOSTPONE. 6767 * Put the resulting flags in "store_afflist[MAXWLEN]" with a terminating NUL 6768 * and return the number of affixes. 6769 */ 6770 static int 6771get_pfxlist(affile, afflist, store_afflist) 6772 afffile_T *affile; 6773 char_u *afflist; 6774 char_u *store_afflist; 6775{ 6776 char_u *p; 6777 char_u *prevp; 6778 int cnt = 0; 6779 int id; 6780 char_u key[AH_KEY_LEN]; 6781 hashitem_T *hi; 6782 6783 for (p = afflist; *p != NUL; ) 6784 { 6785 prevp = p; 6786 if (get_affitem(affile->af_flagtype, &p) != 0) 6787 { 6788 /* A flag is a postponed prefix flag if it appears in "af_pref" 6789 * and it's ID is not zero. */ 6790 vim_strncpy(key, prevp, p - prevp); 6791 hi = hash_find(&affile->af_pref, key); 6792 if (!HASHITEM_EMPTY(hi)) 6793 { 6794 id = HI2AH(hi)->ah_newID; 6795 if (id != 0) 6796 store_afflist[cnt++] = id; 6797 } 6798 } 6799 if (affile->af_flagtype == AFT_NUM && *p == ',') 6800 ++p; 6801 } 6802 6803 store_afflist[cnt] = NUL; 6804 return cnt; 6805} 6806 6807/* 6808 * Get the list of compound IDs from the affix list "afflist" that are used 6809 * for compound words. 6810 * Puts the flags in "store_afflist[]". 6811 */ 6812 static void 6813get_compflags(affile, afflist, store_afflist) 6814 afffile_T *affile; 6815 char_u *afflist; 6816 char_u *store_afflist; 6817{ 6818 char_u *p; 6819 char_u *prevp; 6820 int cnt = 0; 6821 char_u key[AH_KEY_LEN]; 6822 hashitem_T *hi; 6823 6824 for (p = afflist; *p != NUL; ) 6825 { 6826 prevp = p; 6827 if (get_affitem(affile->af_flagtype, &p) != 0) 6828 { 6829 /* A flag is a compound flag if it appears in "af_comp". */ 6830 vim_strncpy(key, prevp, p - prevp); 6831 hi = hash_find(&affile->af_comp, key); 6832 if (!HASHITEM_EMPTY(hi)) 6833 store_afflist[cnt++] = HI2CI(hi)->ci_newID; 6834 } 6835 if (affile->af_flagtype == AFT_NUM && *p == ',') 6836 ++p; 6837 } 6838 6839 store_afflist[cnt] = NUL; 6840} 6841 6842/* 6843 * Apply affixes to a word and store the resulting words. 6844 * "ht" is the hashtable with affentry_T that need to be applied, either 6845 * prefixes or suffixes. 6846 * "xht", when not NULL, is the prefix hashtable, to be used additionally on 6847 * the resulting words for combining affixes. 6848 * 6849 * Returns FAIL when out of memory. 6850 */ 6851 static int 6852store_aff_word(spin, word, afflist, affile, ht, xht, condit, flags, 6853 pfxlist, pfxlen) 6854 spellinfo_T *spin; /* spell info */ 6855 char_u *word; /* basic word start */ 6856 char_u *afflist; /* list of names of supported affixes */ 6857 afffile_T *affile; 6858 hashtab_T *ht; 6859 hashtab_T *xht; 6860 int condit; /* CONDIT_SUF et al. */ 6861 int flags; /* flags for the word */ 6862 char_u *pfxlist; /* list of prefix IDs */ 6863 int pfxlen; /* nr of flags in "pfxlist" for prefixes, rest 6864 * is compound flags */ 6865{ 6866 int todo; 6867 hashitem_T *hi; 6868 affheader_T *ah; 6869 affentry_T *ae; 6870 regmatch_T regmatch; 6871 char_u newword[MAXWLEN]; 6872 int retval = OK; 6873 int i, j; 6874 char_u *p; 6875 int use_flags; 6876 char_u *use_pfxlist; 6877 int use_pfxlen; 6878 int need_affix; 6879 char_u store_afflist[MAXWLEN]; 6880 char_u pfx_pfxlist[MAXWLEN]; 6881 size_t wordlen = STRLEN(word); 6882 int use_condit; 6883 6884 todo = (int)ht->ht_used; 6885 for (hi = ht->ht_array; todo > 0 && retval == OK; ++hi) 6886 { 6887 if (!HASHITEM_EMPTY(hi)) 6888 { 6889 --todo; 6890 ah = HI2AH(hi); 6891 6892 /* Check that the affix combines, if required, and that the word 6893 * supports this affix. */ 6894 if (((condit & CONDIT_COMB) == 0 || ah->ah_combine) 6895 && flag_in_afflist(affile->af_flagtype, afflist, 6896 ah->ah_flag)) 6897 { 6898 /* Loop over all affix entries with this name. */ 6899 for (ae = ah->ah_first; ae != NULL; ae = ae->ae_next) 6900 { 6901 /* Check the condition. It's not logical to match case 6902 * here, but it is required for compatibility with 6903 * Myspell. 6904 * Another requirement from Myspell is that the chop 6905 * string is shorter than the word itself. 6906 * For prefixes, when "PFXPOSTPONE" was used, only do 6907 * prefixes with a chop string and/or flags. 6908 * When a previously added affix had CIRCUMFIX this one 6909 * must have it too, if it had not then this one must not 6910 * have one either. */ 6911 regmatch.regprog = ae->ae_prog; 6912 regmatch.rm_ic = FALSE; 6913 if ((xht != NULL || !affile->af_pfxpostpone 6914 || ae->ae_chop != NULL 6915 || ae->ae_flags != NULL) 6916 && (ae->ae_chop == NULL 6917 || STRLEN(ae->ae_chop) < wordlen) 6918 && (ae->ae_prog == NULL 6919 || vim_regexec(®match, word, (colnr_T)0)) 6920 && (((condit & CONDIT_CFIX) == 0) 6921 == ((condit & CONDIT_AFF) == 0 6922 || ae->ae_flags == NULL 6923 || !flag_in_afflist(affile->af_flagtype, 6924 ae->ae_flags, affile->af_circumfix)))) 6925 { 6926 /* Match. Remove the chop and add the affix. */ 6927 if (xht == NULL) 6928 { 6929 /* prefix: chop/add at the start of the word */ 6930 if (ae->ae_add == NULL) 6931 *newword = NUL; 6932 else 6933 STRCPY(newword, ae->ae_add); 6934 p = word; 6935 if (ae->ae_chop != NULL) 6936 { 6937 /* Skip chop string. */ 6938#ifdef FEAT_MBYTE 6939 if (has_mbyte) 6940 { 6941 i = mb_charlen(ae->ae_chop); 6942 for ( ; i > 0; --i) 6943 mb_ptr_adv(p); 6944 } 6945 else 6946#endif 6947 p += STRLEN(ae->ae_chop); 6948 } 6949 STRCAT(newword, p); 6950 } 6951 else 6952 { 6953 /* suffix: chop/add at the end of the word */ 6954 STRCPY(newword, word); 6955 if (ae->ae_chop != NULL) 6956 { 6957 /* Remove chop string. */ 6958 p = newword + STRLEN(newword); 6959 i = (int)MB_CHARLEN(ae->ae_chop); 6960 for ( ; i > 0; --i) 6961 mb_ptr_back(newword, p); 6962 *p = NUL; 6963 } 6964 if (ae->ae_add != NULL) 6965 STRCAT(newword, ae->ae_add); 6966 } 6967 6968 use_flags = flags; 6969 use_pfxlist = pfxlist; 6970 use_pfxlen = pfxlen; 6971 need_affix = FALSE; 6972 use_condit = condit | CONDIT_COMB | CONDIT_AFF; 6973 if (ae->ae_flags != NULL) 6974 { 6975 /* Extract flags from the affix list. */ 6976 use_flags |= get_affix_flags(affile, ae->ae_flags); 6977 6978 if (affile->af_needaffix != 0 && flag_in_afflist( 6979 affile->af_flagtype, ae->ae_flags, 6980 affile->af_needaffix)) 6981 need_affix = TRUE; 6982 6983 /* When there is a CIRCUMFIX flag the other affix 6984 * must also have it and we don't add the word 6985 * with one affix. */ 6986 if (affile->af_circumfix != 0 && flag_in_afflist( 6987 affile->af_flagtype, ae->ae_flags, 6988 affile->af_circumfix)) 6989 { 6990 use_condit |= CONDIT_CFIX; 6991 if ((condit & CONDIT_CFIX) == 0) 6992 need_affix = TRUE; 6993 } 6994 6995 if (affile->af_pfxpostpone 6996 || spin->si_compflags != NULL) 6997 { 6998 if (affile->af_pfxpostpone) 6999 /* Get prefix IDS from the affix list. */ 7000 use_pfxlen = get_pfxlist(affile, 7001 ae->ae_flags, store_afflist); 7002 else 7003 use_pfxlen = 0; 7004 use_pfxlist = store_afflist; 7005 7006 /* Combine the prefix IDs. Avoid adding the 7007 * same ID twice. */ 7008 for (i = 0; i < pfxlen; ++i) 7009 { 7010 for (j = 0; j < use_pfxlen; ++j) 7011 if (pfxlist[i] == use_pfxlist[j]) 7012 break; 7013 if (j == use_pfxlen) 7014 use_pfxlist[use_pfxlen++] = pfxlist[i]; 7015 } 7016 7017 if (spin->si_compflags != NULL) 7018 /* Get compound IDS from the affix list. */ 7019 get_compflags(affile, ae->ae_flags, 7020 use_pfxlist + use_pfxlen); 7021 7022 /* Combine the list of compound flags. 7023 * Concatenate them to the prefix IDs list. 7024 * Avoid adding the same ID twice. */ 7025 for (i = pfxlen; pfxlist[i] != NUL; ++i) 7026 { 7027 for (j = use_pfxlen; 7028 use_pfxlist[j] != NUL; ++j) 7029 if (pfxlist[i] == use_pfxlist[j]) 7030 break; 7031 if (use_pfxlist[j] == NUL) 7032 { 7033 use_pfxlist[j++] = pfxlist[i]; 7034 use_pfxlist[j] = NUL; 7035 } 7036 } 7037 } 7038 } 7039 7040 /* Obey a "COMPOUNDFORBIDFLAG" of the affix: don't 7041 * use the compound flags. */ 7042 if (use_pfxlist != NULL && ae->ae_compforbid) 7043 { 7044 vim_strncpy(pfx_pfxlist, use_pfxlist, use_pfxlen); 7045 use_pfxlist = pfx_pfxlist; 7046 } 7047 7048 /* When there are postponed prefixes... */ 7049 if (spin->si_prefroot != NULL 7050 && spin->si_prefroot->wn_sibling != NULL) 7051 { 7052 /* ... add a flag to indicate an affix was used. */ 7053 use_flags |= WF_HAS_AFF; 7054 7055 /* ... don't use a prefix list if combining 7056 * affixes is not allowed. But do use the 7057 * compound flags after them. */ 7058 if (!ah->ah_combine && use_pfxlist != NULL) 7059 use_pfxlist += use_pfxlen; 7060 } 7061 7062 /* When compounding is supported and there is no 7063 * "COMPOUNDPERMITFLAG" then forbid compounding on the 7064 * side where the affix is applied. */ 7065 if (spin->si_compflags != NULL && !ae->ae_comppermit) 7066 { 7067 if (xht != NULL) 7068 use_flags |= WF_NOCOMPAFT; 7069 else 7070 use_flags |= WF_NOCOMPBEF; 7071 } 7072 7073 /* Store the modified word. */ 7074 if (store_word(spin, newword, use_flags, 7075 spin->si_region, use_pfxlist, 7076 need_affix) == FAIL) 7077 retval = FAIL; 7078 7079 /* When added a prefix or a first suffix and the affix 7080 * has flags may add a(nother) suffix. RECURSIVE! */ 7081 if ((condit & CONDIT_SUF) && ae->ae_flags != NULL) 7082 if (store_aff_word(spin, newword, ae->ae_flags, 7083 affile, &affile->af_suff, xht, 7084 use_condit & (xht == NULL 7085 ? ~0 : ~CONDIT_SUF), 7086 use_flags, use_pfxlist, pfxlen) == FAIL) 7087 retval = FAIL; 7088 7089 /* When added a suffix and combining is allowed also 7090 * try adding a prefix additionally. Both for the 7091 * word flags and for the affix flags. RECURSIVE! */ 7092 if (xht != NULL && ah->ah_combine) 7093 { 7094 if (store_aff_word(spin, newword, 7095 afflist, affile, 7096 xht, NULL, use_condit, 7097 use_flags, use_pfxlist, 7098 pfxlen) == FAIL 7099 || (ae->ae_flags != NULL 7100 && store_aff_word(spin, newword, 7101 ae->ae_flags, affile, 7102 xht, NULL, use_condit, 7103 use_flags, use_pfxlist, 7104 pfxlen) == FAIL)) 7105 retval = FAIL; 7106 } 7107 } 7108 } 7109 } 7110 } 7111 } 7112 7113 return retval; 7114} 7115 7116/* 7117 * Read a file with a list of words. 7118 */ 7119 static int 7120spell_read_wordfile(spin, fname) 7121 spellinfo_T *spin; 7122 char_u *fname; 7123{ 7124 FILE *fd; 7125 long lnum = 0; 7126 char_u rline[MAXLINELEN]; 7127 char_u *line; 7128 char_u *pc = NULL; 7129 char_u *p; 7130 int l; 7131 int retval = OK; 7132 int did_word = FALSE; 7133 int non_ascii = 0; 7134 int flags; 7135 int regionmask; 7136 7137 /* 7138 * Open the file. 7139 */ 7140 fd = mch_fopen((char *)fname, "r"); 7141 if (fd == NULL) 7142 { 7143 EMSG2(_(e_notopen), fname); 7144 return FAIL; 7145 } 7146 7147 vim_snprintf((char *)IObuff, IOSIZE, _("Reading word file %s ..."), fname); 7148 spell_message(spin, IObuff); 7149 7150 /* 7151 * Read all the lines in the file one by one. 7152 */ 7153 while (!vim_fgets(rline, MAXLINELEN, fd) && !got_int) 7154 { 7155 line_breakcheck(); 7156 ++lnum; 7157 7158 /* Skip comment lines. */ 7159 if (*rline == '#') 7160 continue; 7161 7162 /* Remove CR, LF and white space from the end. */ 7163 l = (int)STRLEN(rline); 7164 while (l > 0 && rline[l - 1] <= ' ') 7165 --l; 7166 if (l == 0) 7167 continue; /* empty or blank line */ 7168 rline[l] = NUL; 7169 7170 /* Convert from "/encoding={encoding}" to 'encoding' when needed. */ 7171 vim_free(pc); 7172#ifdef FEAT_MBYTE 7173 if (spin->si_conv.vc_type != CONV_NONE) 7174 { 7175 pc = string_convert(&spin->si_conv, rline, NULL); 7176 if (pc == NULL) 7177 { 7178 smsg((char_u *)_("Conversion failure for word in %s line %d: %s"), 7179 fname, lnum, rline); 7180 continue; 7181 } 7182 line = pc; 7183 } 7184 else 7185#endif 7186 { 7187 pc = NULL; 7188 line = rline; 7189 } 7190 7191 if (*line == '/') 7192 { 7193 ++line; 7194 if (STRNCMP(line, "encoding=", 9) == 0) 7195 { 7196 if (spin->si_conv.vc_type != CONV_NONE) 7197 smsg((char_u *)_("Duplicate /encoding= line ignored in %s line %d: %s"), 7198 fname, lnum, line - 1); 7199 else if (did_word) 7200 smsg((char_u *)_("/encoding= line after word ignored in %s line %d: %s"), 7201 fname, lnum, line - 1); 7202 else 7203 { 7204#ifdef FEAT_MBYTE 7205 char_u *enc; 7206 7207 /* Setup for conversion to 'encoding'. */ 7208 line += 9; 7209 enc = enc_canonize(line); 7210 if (enc != NULL && !spin->si_ascii 7211 && convert_setup(&spin->si_conv, enc, 7212 p_enc) == FAIL) 7213 smsg((char_u *)_("Conversion in %s not supported: from %s to %s"), 7214 fname, line, p_enc); 7215 vim_free(enc); 7216 spin->si_conv.vc_fail = TRUE; 7217#else 7218 smsg((char_u *)_("Conversion in %s not supported"), fname); 7219#endif 7220 } 7221 continue; 7222 } 7223 7224 if (STRNCMP(line, "regions=", 8) == 0) 7225 { 7226 if (spin->si_region_count > 1) 7227 smsg((char_u *)_("Duplicate /regions= line ignored in %s line %d: %s"), 7228 fname, lnum, line); 7229 else 7230 { 7231 line += 8; 7232 if (STRLEN(line) > 16) 7233 smsg((char_u *)_("Too many regions in %s line %d: %s"), 7234 fname, lnum, line); 7235 else 7236 { 7237 spin->si_region_count = (int)STRLEN(line) / 2; 7238 STRCPY(spin->si_region_name, line); 7239 7240 /* Adjust the mask for a word valid in all regions. */ 7241 spin->si_region = (1 << spin->si_region_count) - 1; 7242 } 7243 } 7244 continue; 7245 } 7246 7247 smsg((char_u *)_("/ line ignored in %s line %d: %s"), 7248 fname, lnum, line - 1); 7249 continue; 7250 } 7251 7252 flags = 0; 7253 regionmask = spin->si_region; 7254 7255 /* Check for flags and region after a slash. */ 7256 p = vim_strchr(line, '/'); 7257 if (p != NULL) 7258 { 7259 *p++ = NUL; 7260 while (*p != NUL) 7261 { 7262 if (*p == '=') /* keep-case word */ 7263 flags |= WF_KEEPCAP | WF_FIXCAP; 7264 else if (*p == '!') /* Bad, bad, wicked word. */ 7265 flags |= WF_BANNED; 7266 else if (*p == '?') /* Rare word. */ 7267 flags |= WF_RARE; 7268 else if (VIM_ISDIGIT(*p)) /* region number(s) */ 7269 { 7270 if ((flags & WF_REGION) == 0) /* first one */ 7271 regionmask = 0; 7272 flags |= WF_REGION; 7273 7274 l = *p - '0'; 7275 if (l > spin->si_region_count) 7276 { 7277 smsg((char_u *)_("Invalid region nr in %s line %d: %s"), 7278 fname, lnum, p); 7279 break; 7280 } 7281 regionmask |= 1 << (l - 1); 7282 } 7283 else 7284 { 7285 smsg((char_u *)_("Unrecognized flags in %s line %d: %s"), 7286 fname, lnum, p); 7287 break; 7288 } 7289 ++p; 7290 } 7291 } 7292 7293 /* Skip non-ASCII words when "spin->si_ascii" is TRUE. */ 7294 if (spin->si_ascii && has_non_ascii(line)) 7295 { 7296 ++non_ascii; 7297 continue; 7298 } 7299 7300 /* Normal word: store it. */ 7301 if (store_word(spin, line, flags, regionmask, NULL, FALSE) == FAIL) 7302 { 7303 retval = FAIL; 7304 break; 7305 } 7306 did_word = TRUE; 7307 } 7308 7309 vim_free(pc); 7310 fclose(fd); 7311 7312 if (spin->si_ascii && non_ascii > 0) 7313 { 7314 vim_snprintf((char *)IObuff, IOSIZE, 7315 _("Ignored %d words with non-ASCII characters"), non_ascii); 7316 spell_message(spin, IObuff); 7317 } 7318 7319 return retval; 7320} 7321 7322/* 7323 * Get part of an sblock_T, "len" bytes long. 7324 * This avoids calling free() for every little struct we use (and keeping 7325 * track of them). 7326 * The memory is cleared to all zeros. 7327 * Returns NULL when out of memory. 7328 */ 7329 static void * 7330getroom(spin, len, align) 7331 spellinfo_T *spin; 7332 size_t len; /* length needed */ 7333 int align; /* align for pointer */ 7334{ 7335 char_u *p; 7336 sblock_T *bl = spin->si_blocks; 7337 7338 if (align && bl != NULL) 7339 /* Round size up for alignment. On some systems structures need to be 7340 * aligned to the size of a pointer (e.g., SPARC). */ 7341 bl->sb_used = (bl->sb_used + sizeof(char *) - 1) 7342 & ~(sizeof(char *) - 1); 7343 7344 if (bl == NULL || bl->sb_used + len > SBLOCKSIZE) 7345 { 7346 /* Allocate a block of memory. This is not freed until much later. */ 7347 bl = (sblock_T *)alloc_clear((unsigned)(sizeof(sblock_T) + SBLOCKSIZE)); 7348 if (bl == NULL) 7349 return NULL; 7350 bl->sb_next = spin->si_blocks; 7351 spin->si_blocks = bl; 7352 bl->sb_used = 0; 7353 ++spin->si_blocks_cnt; 7354 } 7355 7356 p = bl->sb_data + bl->sb_used; 7357 bl->sb_used += (int)len; 7358 7359 return p; 7360} 7361 7362/* 7363 * Make a copy of a string into memory allocated with getroom(). 7364 */ 7365 static char_u * 7366getroom_save(spin, s) 7367 spellinfo_T *spin; 7368 char_u *s; 7369{ 7370 char_u *sc; 7371 7372 sc = (char_u *)getroom(spin, STRLEN(s) + 1, FALSE); 7373 if (sc != NULL) 7374 STRCPY(sc, s); 7375 return sc; 7376} 7377 7378 7379/* 7380 * Free the list of allocated sblock_T. 7381 */ 7382 static void 7383free_blocks(bl) 7384 sblock_T *bl; 7385{ 7386 sblock_T *next; 7387 7388 while (bl != NULL) 7389 { 7390 next = bl->sb_next; 7391 vim_free(bl); 7392 bl = next; 7393 } 7394} 7395 7396/* 7397 * Allocate the root of a word tree. 7398 */ 7399 static wordnode_T * 7400wordtree_alloc(spin) 7401 spellinfo_T *spin; 7402{ 7403 return (wordnode_T *)getroom(spin, sizeof(wordnode_T), TRUE); 7404} 7405 7406/* 7407 * Store a word in the tree(s). 7408 * Always store it in the case-folded tree. For a keep-case word this is 7409 * useful when the word can also be used with all caps (no WF_FIXCAP flag) and 7410 * used to find suggestions. 7411 * For a keep-case word also store it in the keep-case tree. 7412 * When "pfxlist" is not NULL store the word for each postponed prefix ID and 7413 * compound flag. 7414 */ 7415 static int 7416store_word(spin, word, flags, region, pfxlist, need_affix) 7417 spellinfo_T *spin; 7418 char_u *word; 7419 int flags; /* extra flags, WF_BANNED */ 7420 int region; /* supported region(s) */ 7421 char_u *pfxlist; /* list of prefix IDs or NULL */ 7422 int need_affix; /* only store word with affix ID */ 7423{ 7424 int len = (int)STRLEN(word); 7425 int ct = captype(word, word + len); 7426 char_u foldword[MAXWLEN]; 7427 int res = OK; 7428 char_u *p; 7429 7430 (void)spell_casefold(word, len, foldword, MAXWLEN); 7431 for (p = pfxlist; res == OK; ++p) 7432 { 7433 if (!need_affix || (p != NULL && *p != NUL)) 7434 res = tree_add_word(spin, foldword, spin->si_foldroot, ct | flags, 7435 region, p == NULL ? 0 : *p); 7436 if (p == NULL || *p == NUL) 7437 break; 7438 } 7439 ++spin->si_foldwcount; 7440 7441 if (res == OK && (ct == WF_KEEPCAP || (flags & WF_KEEPCAP))) 7442 { 7443 for (p = pfxlist; res == OK; ++p) 7444 { 7445 if (!need_affix || (p != NULL && *p != NUL)) 7446 res = tree_add_word(spin, word, spin->si_keeproot, flags, 7447 region, p == NULL ? 0 : *p); 7448 if (p == NULL || *p == NUL) 7449 break; 7450 } 7451 ++spin->si_keepwcount; 7452 } 7453 return res; 7454} 7455 7456/* 7457 * Add word "word" to a word tree at "root". 7458 * When "flags" < 0 we are adding to the prefix tree where "flags" is used for 7459 * "rare" and "region" is the condition nr. 7460 * Returns FAIL when out of memory. 7461 */ 7462 static int 7463tree_add_word(spin, word, root, flags, region, affixID) 7464 spellinfo_T *spin; 7465 char_u *word; 7466 wordnode_T *root; 7467 int flags; 7468 int region; 7469 int affixID; 7470{ 7471 wordnode_T *node = root; 7472 wordnode_T *np; 7473 wordnode_T *copyp, **copyprev; 7474 wordnode_T **prev = NULL; 7475 int i; 7476 7477 /* Add each byte of the word to the tree, including the NUL at the end. */ 7478 for (i = 0; ; ++i) 7479 { 7480 /* When there is more than one reference to this node we need to make 7481 * a copy, so that we can modify it. Copy the whole list of siblings 7482 * (we don't optimize for a partly shared list of siblings). */ 7483 if (node != NULL && node->wn_refs > 1) 7484 { 7485 --node->wn_refs; 7486 copyprev = prev; 7487 for (copyp = node; copyp != NULL; copyp = copyp->wn_sibling) 7488 { 7489 /* Allocate a new node and copy the info. */ 7490 np = get_wordnode(spin); 7491 if (np == NULL) 7492 return FAIL; 7493 np->wn_child = copyp->wn_child; 7494 if (np->wn_child != NULL) 7495 ++np->wn_child->wn_refs; /* child gets extra ref */ 7496 np->wn_byte = copyp->wn_byte; 7497 if (np->wn_byte == NUL) 7498 { 7499 np->wn_flags = copyp->wn_flags; 7500 np->wn_region = copyp->wn_region; 7501 np->wn_affixID = copyp->wn_affixID; 7502 } 7503 7504 /* Link the new node in the list, there will be one ref. */ 7505 np->wn_refs = 1; 7506 if (copyprev != NULL) 7507 *copyprev = np; 7508 copyprev = &np->wn_sibling; 7509 7510 /* Let "node" point to the head of the copied list. */ 7511 if (copyp == node) 7512 node = np; 7513 } 7514 } 7515 7516 /* Look for the sibling that has the same character. They are sorted 7517 * on byte value, thus stop searching when a sibling is found with a 7518 * higher byte value. For zero bytes (end of word) the sorting is 7519 * done on flags and then on affixID. */ 7520 while (node != NULL 7521 && (node->wn_byte < word[i] 7522 || (node->wn_byte == NUL 7523 && (flags < 0 7524 ? node->wn_affixID < (unsigned)affixID 7525 : (node->wn_flags < (unsigned)(flags & WN_MASK) 7526 || (node->wn_flags == (flags & WN_MASK) 7527 && (spin->si_sugtree 7528 ? (node->wn_region & 0xffff) < region 7529 : node->wn_affixID 7530 < (unsigned)affixID))))))) 7531 { 7532 prev = &node->wn_sibling; 7533 node = *prev; 7534 } 7535 if (node == NULL 7536 || node->wn_byte != word[i] 7537 || (word[i] == NUL 7538 && (flags < 0 7539 || spin->si_sugtree 7540 || node->wn_flags != (flags & WN_MASK) 7541 || node->wn_affixID != affixID))) 7542 { 7543 /* Allocate a new node. */ 7544 np = get_wordnode(spin); 7545 if (np == NULL) 7546 return FAIL; 7547 np->wn_byte = word[i]; 7548 7549 /* If "node" is NULL this is a new child or the end of the sibling 7550 * list: ref count is one. Otherwise use ref count of sibling and 7551 * make ref count of sibling one (matters when inserting in front 7552 * of the list of siblings). */ 7553 if (node == NULL) 7554 np->wn_refs = 1; 7555 else 7556 { 7557 np->wn_refs = node->wn_refs; 7558 node->wn_refs = 1; 7559 } 7560 if (prev != NULL) 7561 *prev = np; 7562 np->wn_sibling = node; 7563 node = np; 7564 } 7565 7566 if (word[i] == NUL) 7567 { 7568 node->wn_flags = flags; 7569 node->wn_region |= region; 7570 node->wn_affixID = affixID; 7571 break; 7572 } 7573 prev = &node->wn_child; 7574 node = *prev; 7575 } 7576#ifdef SPELL_PRINTTREE 7577 smsg("Added \"%s\"", word); 7578 spell_print_tree(root->wn_sibling); 7579#endif 7580 7581 /* count nr of words added since last message */ 7582 ++spin->si_msg_count; 7583 7584 if (spin->si_compress_cnt > 1) 7585 { 7586 if (--spin->si_compress_cnt == 1) 7587 /* Did enough words to lower the block count limit. */ 7588 spin->si_blocks_cnt += compress_inc; 7589 } 7590 7591 /* 7592 * When we have allocated lots of memory we need to compress the word tree 7593 * to free up some room. But compression is slow, and we might actually 7594 * need that room, thus only compress in the following situations: 7595 * 1. When not compressed before (si_compress_cnt == 0): when using 7596 * "compress_start" blocks. 7597 * 2. When compressed before and used "compress_inc" blocks before 7598 * adding "compress_added" words (si_compress_cnt > 1). 7599 * 3. When compressed before, added "compress_added" words 7600 * (si_compress_cnt == 1) and the number of free nodes drops below the 7601 * maximum word length. 7602 */ 7603#ifndef SPELL_PRINTTREE 7604 if (spin->si_compress_cnt == 1 7605 ? spin->si_free_count < MAXWLEN 7606 : spin->si_blocks_cnt >= compress_start) 7607#endif 7608 { 7609 /* Decrement the block counter. The effect is that we compress again 7610 * when the freed up room has been used and another "compress_inc" 7611 * blocks have been allocated. Unless "compress_added" words have 7612 * been added, then the limit is put back again. */ 7613 spin->si_blocks_cnt -= compress_inc; 7614 spin->si_compress_cnt = compress_added; 7615 7616 if (spin->si_verbose) 7617 { 7618 msg_start(); 7619 msg_puts((char_u *)_(msg_compressing)); 7620 msg_clr_eos(); 7621 msg_didout = FALSE; 7622 msg_col = 0; 7623 out_flush(); 7624 } 7625 7626 /* Compress both trees. Either they both have many nodes, which makes 7627 * compression useful, or one of them is small, which means 7628 * compression goes fast. But when filling the souldfold word tree 7629 * there is no keep-case tree. */ 7630 wordtree_compress(spin, spin->si_foldroot); 7631 if (affixID >= 0) 7632 wordtree_compress(spin, spin->si_keeproot); 7633 } 7634 7635 return OK; 7636} 7637 7638/* 7639 * Check the 'mkspellmem' option. Return FAIL if it's wrong. 7640 * Sets "sps_flags". 7641 */ 7642 int 7643spell_check_msm() 7644{ 7645 char_u *p = p_msm; 7646 long start = 0; 7647 long incr = 0; 7648 long added = 0; 7649 7650 if (!VIM_ISDIGIT(*p)) 7651 return FAIL; 7652 /* block count = (value * 1024) / SBLOCKSIZE (but avoid overflow)*/ 7653 start = (getdigits(&p) * 10) / (SBLOCKSIZE / 102); 7654 if (*p != ',') 7655 return FAIL; 7656 ++p; 7657 if (!VIM_ISDIGIT(*p)) 7658 return FAIL; 7659 incr = (getdigits(&p) * 102) / (SBLOCKSIZE / 10); 7660 if (*p != ',') 7661 return FAIL; 7662 ++p; 7663 if (!VIM_ISDIGIT(*p)) 7664 return FAIL; 7665 added = getdigits(&p) * 1024; 7666 if (*p != NUL) 7667 return FAIL; 7668 7669 if (start == 0 || incr == 0 || added == 0 || incr > start) 7670 return FAIL; 7671 7672 compress_start = start; 7673 compress_inc = incr; 7674 compress_added = added; 7675 return OK; 7676} 7677 7678 7679/* 7680 * Get a wordnode_T, either from the list of previously freed nodes or 7681 * allocate a new one. 7682 */ 7683 static wordnode_T * 7684get_wordnode(spin) 7685 spellinfo_T *spin; 7686{ 7687 wordnode_T *n; 7688 7689 if (spin->si_first_free == NULL) 7690 n = (wordnode_T *)getroom(spin, sizeof(wordnode_T), TRUE); 7691 else 7692 { 7693 n = spin->si_first_free; 7694 spin->si_first_free = n->wn_child; 7695 vim_memset(n, 0, sizeof(wordnode_T)); 7696 --spin->si_free_count; 7697 } 7698#ifdef SPELL_PRINTTREE 7699 n->wn_nr = ++spin->si_wordnode_nr; 7700#endif 7701 return n; 7702} 7703 7704/* 7705 * Decrement the reference count on a node (which is the head of a list of 7706 * siblings). If the reference count becomes zero free the node and its 7707 * siblings. 7708 * Returns the number of nodes actually freed. 7709 */ 7710 static int 7711deref_wordnode(spin, node) 7712 spellinfo_T *spin; 7713 wordnode_T *node; 7714{ 7715 wordnode_T *np; 7716 int cnt = 0; 7717 7718 if (--node->wn_refs == 0) 7719 { 7720 for (np = node; np != NULL; np = np->wn_sibling) 7721 { 7722 if (np->wn_child != NULL) 7723 cnt += deref_wordnode(spin, np->wn_child); 7724 free_wordnode(spin, np); 7725 ++cnt; 7726 } 7727 ++cnt; /* length field */ 7728 } 7729 return cnt; 7730} 7731 7732/* 7733 * Free a wordnode_T for re-use later. 7734 * Only the "wn_child" field becomes invalid. 7735 */ 7736 static void 7737free_wordnode(spin, n) 7738 spellinfo_T *spin; 7739 wordnode_T *n; 7740{ 7741 n->wn_child = spin->si_first_free; 7742 spin->si_first_free = n; 7743 ++spin->si_free_count; 7744} 7745 7746/* 7747 * Compress a tree: find tails that are identical and can be shared. 7748 */ 7749 static void 7750wordtree_compress(spin, root) 7751 spellinfo_T *spin; 7752 wordnode_T *root; 7753{ 7754 hashtab_T ht; 7755 int n; 7756 int tot = 0; 7757 int perc; 7758 7759 /* Skip the root itself, it's not actually used. The first sibling is the 7760 * start of the tree. */ 7761 if (root->wn_sibling != NULL) 7762 { 7763 hash_init(&ht); 7764 n = node_compress(spin, root->wn_sibling, &ht, &tot); 7765 7766#ifndef SPELL_PRINTTREE 7767 if (spin->si_verbose || p_verbose > 2) 7768#endif 7769 { 7770 if (tot > 1000000) 7771 perc = (tot - n) / (tot / 100); 7772 else if (tot == 0) 7773 perc = 0; 7774 else 7775 perc = (tot - n) * 100 / tot; 7776 vim_snprintf((char *)IObuff, IOSIZE, 7777 _("Compressed %d of %d nodes; %d (%d%%) remaining"), 7778 n, tot, tot - n, perc); 7779 spell_message(spin, IObuff); 7780 } 7781#ifdef SPELL_PRINTTREE 7782 spell_print_tree(root->wn_sibling); 7783#endif 7784 hash_clear(&ht); 7785 } 7786} 7787 7788/* 7789 * Compress a node, its siblings and its children, depth first. 7790 * Returns the number of compressed nodes. 7791 */ 7792 static int 7793node_compress(spin, node, ht, tot) 7794 spellinfo_T *spin; 7795 wordnode_T *node; 7796 hashtab_T *ht; 7797 int *tot; /* total count of nodes before compressing, 7798 incremented while going through the tree */ 7799{ 7800 wordnode_T *np; 7801 wordnode_T *tp; 7802 wordnode_T *child; 7803 hash_T hash; 7804 hashitem_T *hi; 7805 int len = 0; 7806 unsigned nr, n; 7807 int compressed = 0; 7808 7809 /* 7810 * Go through the list of siblings. Compress each child and then try 7811 * finding an identical child to replace it. 7812 * Note that with "child" we mean not just the node that is pointed to, 7813 * but the whole list of siblings of which the child node is the first. 7814 */ 7815 for (np = node; np != NULL && !got_int; np = np->wn_sibling) 7816 { 7817 ++len; 7818 if ((child = np->wn_child) != NULL) 7819 { 7820 /* Compress the child first. This fills hashkey. */ 7821 compressed += node_compress(spin, child, ht, tot); 7822 7823 /* Try to find an identical child. */ 7824 hash = hash_hash(child->wn_u1.hashkey); 7825 hi = hash_lookup(ht, child->wn_u1.hashkey, hash); 7826 if (!HASHITEM_EMPTY(hi)) 7827 { 7828 /* There are children we encountered before with a hash value 7829 * identical to the current child. Now check if there is one 7830 * that is really identical. */ 7831 for (tp = HI2WN(hi); tp != NULL; tp = tp->wn_u2.next) 7832 if (node_equal(child, tp)) 7833 { 7834 /* Found one! Now use that child in place of the 7835 * current one. This means the current child and all 7836 * its siblings is unlinked from the tree. */ 7837 ++tp->wn_refs; 7838 compressed += deref_wordnode(spin, child); 7839 np->wn_child = tp; 7840 break; 7841 } 7842 if (tp == NULL) 7843 { 7844 /* No other child with this hash value equals the child of 7845 * the node, add it to the linked list after the first 7846 * item. */ 7847 tp = HI2WN(hi); 7848 child->wn_u2.next = tp->wn_u2.next; 7849 tp->wn_u2.next = child; 7850 } 7851 } 7852 else 7853 /* No other child has this hash value, add it to the 7854 * hashtable. */ 7855 hash_add_item(ht, hi, child->wn_u1.hashkey, hash); 7856 } 7857 } 7858 *tot += len + 1; /* add one for the node that stores the length */ 7859 7860 /* 7861 * Make a hash key for the node and its siblings, so that we can quickly 7862 * find a lookalike node. This must be done after compressing the sibling 7863 * list, otherwise the hash key would become invalid by the compression. 7864 */ 7865 node->wn_u1.hashkey[0] = len; 7866 nr = 0; 7867 for (np = node; np != NULL; np = np->wn_sibling) 7868 { 7869 if (np->wn_byte == NUL) 7870 /* end node: use wn_flags, wn_region and wn_affixID */ 7871 n = np->wn_flags + (np->wn_region << 8) + (np->wn_affixID << 16); 7872 else 7873 /* byte node: use the byte value and the child pointer */ 7874 n = (unsigned)(np->wn_byte + ((long_u)np->wn_child << 8)); 7875 nr = nr * 101 + n; 7876 } 7877 7878 /* Avoid NUL bytes, it terminates the hash key. */ 7879 n = nr & 0xff; 7880 node->wn_u1.hashkey[1] = n == 0 ? 1 : n; 7881 n = (nr >> 8) & 0xff; 7882 node->wn_u1.hashkey[2] = n == 0 ? 1 : n; 7883 n = (nr >> 16) & 0xff; 7884 node->wn_u1.hashkey[3] = n == 0 ? 1 : n; 7885 n = (nr >> 24) & 0xff; 7886 node->wn_u1.hashkey[4] = n == 0 ? 1 : n; 7887 node->wn_u1.hashkey[5] = NUL; 7888 7889 /* Check for CTRL-C pressed now and then. */ 7890 fast_breakcheck(); 7891 7892 return compressed; 7893} 7894 7895/* 7896 * Return TRUE when two nodes have identical siblings and children. 7897 */ 7898 static int 7899node_equal(n1, n2) 7900 wordnode_T *n1; 7901 wordnode_T *n2; 7902{ 7903 wordnode_T *p1; 7904 wordnode_T *p2; 7905 7906 for (p1 = n1, p2 = n2; p1 != NULL && p2 != NULL; 7907 p1 = p1->wn_sibling, p2 = p2->wn_sibling) 7908 if (p1->wn_byte != p2->wn_byte 7909 || (p1->wn_byte == NUL 7910 ? (p1->wn_flags != p2->wn_flags 7911 || p1->wn_region != p2->wn_region 7912 || p1->wn_affixID != p2->wn_affixID) 7913 : (p1->wn_child != p2->wn_child))) 7914 break; 7915 7916 return p1 == NULL && p2 == NULL; 7917} 7918 7919static int 7920#ifdef __BORLANDC__ 7921_RTLENTRYF 7922#endif 7923rep_compare __ARGS((const void *s1, const void *s2)); 7924 7925/* 7926 * Function given to qsort() to sort the REP items on "from" string. 7927 */ 7928 static int 7929#ifdef __BORLANDC__ 7930_RTLENTRYF 7931#endif 7932rep_compare(s1, s2) 7933 const void *s1; 7934 const void *s2; 7935{ 7936 fromto_T *p1 = (fromto_T *)s1; 7937 fromto_T *p2 = (fromto_T *)s2; 7938 7939 return STRCMP(p1->ft_from, p2->ft_from); 7940} 7941 7942/* 7943 * Write the Vim .spl file "fname". 7944 * Return FAIL or OK; 7945 */ 7946 static int 7947write_vim_spell(spin, fname) 7948 spellinfo_T *spin; 7949 char_u *fname; 7950{ 7951 FILE *fd; 7952 int regionmask; 7953 int round; 7954 wordnode_T *tree; 7955 int nodecount; 7956 int i; 7957 int l; 7958 garray_T *gap; 7959 fromto_T *ftp; 7960 char_u *p; 7961 int rr; 7962 int retval = OK; 7963 size_t fwv = 1; /* collect return value of fwrite() to avoid 7964 warnings from picky compiler */ 7965 7966 fd = mch_fopen((char *)fname, "w"); 7967 if (fd == NULL) 7968 { 7969 EMSG2(_(e_notopen), fname); 7970 return FAIL; 7971 } 7972 7973 /* <HEADER>: <fileID> <versionnr> */ 7974 /* <fileID> */ 7975 fwv &= fwrite(VIMSPELLMAGIC, VIMSPELLMAGICL, (size_t)1, fd); 7976 if (fwv != (size_t)1) 7977 /* Catch first write error, don't try writing more. */ 7978 goto theend; 7979 7980 putc(VIMSPELLVERSION, fd); /* <versionnr> */ 7981 7982 /* 7983 * <SECTIONS>: <section> ... <sectionend> 7984 */ 7985 7986 /* SN_INFO: <infotext> */ 7987 if (spin->si_info != NULL) 7988 { 7989 putc(SN_INFO, fd); /* <sectionID> */ 7990 putc(0, fd); /* <sectionflags> */ 7991 7992 i = (int)STRLEN(spin->si_info); 7993 put_bytes(fd, (long_u)i, 4); /* <sectionlen> */ 7994 fwv &= fwrite(spin->si_info, (size_t)i, (size_t)1, fd); /* <infotext> */ 7995 } 7996 7997 /* SN_REGION: <regionname> ... 7998 * Write the region names only if there is more than one. */ 7999 if (spin->si_region_count > 1) 8000 { 8001 putc(SN_REGION, fd); /* <sectionID> */ 8002 putc(SNF_REQUIRED, fd); /* <sectionflags> */ 8003 l = spin->si_region_count * 2; 8004 put_bytes(fd, (long_u)l, 4); /* <sectionlen> */ 8005 fwv &= fwrite(spin->si_region_name, (size_t)l, (size_t)1, fd); 8006 /* <regionname> ... */ 8007 regionmask = (1 << spin->si_region_count) - 1; 8008 } 8009 else 8010 regionmask = 0; 8011 8012 /* SN_CHARFLAGS: <charflagslen> <charflags> <folcharslen> <folchars> 8013 * 8014 * The table with character flags and the table for case folding. 8015 * This makes sure the same characters are recognized as word characters 8016 * when generating an when using a spell file. 8017 * Skip this for ASCII, the table may conflict with the one used for 8018 * 'encoding'. 8019 * Also skip this for an .add.spl file, the main spell file must contain 8020 * the table (avoids that it conflicts). File is shorter too. 8021 */ 8022 if (!spin->si_ascii && !spin->si_add) 8023 { 8024 char_u folchars[128 * 8]; 8025 int flags; 8026 8027 putc(SN_CHARFLAGS, fd); /* <sectionID> */ 8028 putc(SNF_REQUIRED, fd); /* <sectionflags> */ 8029 8030 /* Form the <folchars> string first, we need to know its length. */ 8031 l = 0; 8032 for (i = 128; i < 256; ++i) 8033 { 8034#ifdef FEAT_MBYTE 8035 if (has_mbyte) 8036 l += mb_char2bytes(spelltab.st_fold[i], folchars + l); 8037 else 8038#endif 8039 folchars[l++] = spelltab.st_fold[i]; 8040 } 8041 put_bytes(fd, (long_u)(1 + 128 + 2 + l), 4); /* <sectionlen> */ 8042 8043 fputc(128, fd); /* <charflagslen> */ 8044 for (i = 128; i < 256; ++i) 8045 { 8046 flags = 0; 8047 if (spelltab.st_isw[i]) 8048 flags |= CF_WORD; 8049 if (spelltab.st_isu[i]) 8050 flags |= CF_UPPER; 8051 fputc(flags, fd); /* <charflags> */ 8052 } 8053 8054 put_bytes(fd, (long_u)l, 2); /* <folcharslen> */ 8055 fwv &= fwrite(folchars, (size_t)l, (size_t)1, fd); /* <folchars> */ 8056 } 8057 8058 /* SN_MIDWORD: <midword> */ 8059 if (spin->si_midword != NULL) 8060 { 8061 putc(SN_MIDWORD, fd); /* <sectionID> */ 8062 putc(SNF_REQUIRED, fd); /* <sectionflags> */ 8063 8064 i = (int)STRLEN(spin->si_midword); 8065 put_bytes(fd, (long_u)i, 4); /* <sectionlen> */ 8066 fwv &= fwrite(spin->si_midword, (size_t)i, (size_t)1, fd); 8067 /* <midword> */ 8068 } 8069 8070 /* SN_PREFCOND: <prefcondcnt> <prefcond> ... */ 8071 if (spin->si_prefcond.ga_len > 0) 8072 { 8073 putc(SN_PREFCOND, fd); /* <sectionID> */ 8074 putc(SNF_REQUIRED, fd); /* <sectionflags> */ 8075 8076 l = write_spell_prefcond(NULL, &spin->si_prefcond); 8077 put_bytes(fd, (long_u)l, 4); /* <sectionlen> */ 8078 8079 write_spell_prefcond(fd, &spin->si_prefcond); 8080 } 8081 8082 /* SN_REP: <repcount> <rep> ... 8083 * SN_SAL: <salflags> <salcount> <sal> ... 8084 * SN_REPSAL: <repcount> <rep> ... */ 8085 8086 /* round 1: SN_REP section 8087 * round 2: SN_SAL section (unless SN_SOFO is used) 8088 * round 3: SN_REPSAL section */ 8089 for (round = 1; round <= 3; ++round) 8090 { 8091 if (round == 1) 8092 gap = &spin->si_rep; 8093 else if (round == 2) 8094 { 8095 /* Don't write SN_SAL when using a SN_SOFO section */ 8096 if (spin->si_sofofr != NULL && spin->si_sofoto != NULL) 8097 continue; 8098 gap = &spin->si_sal; 8099 } 8100 else 8101 gap = &spin->si_repsal; 8102 8103 /* Don't write the section if there are no items. */ 8104 if (gap->ga_len == 0) 8105 continue; 8106 8107 /* Sort the REP/REPSAL items. */ 8108 if (round != 2) 8109 qsort(gap->ga_data, (size_t)gap->ga_len, 8110 sizeof(fromto_T), rep_compare); 8111 8112 i = round == 1 ? SN_REP : (round == 2 ? SN_SAL : SN_REPSAL); 8113 putc(i, fd); /* <sectionID> */ 8114 8115 /* This is for making suggestions, section is not required. */ 8116 putc(0, fd); /* <sectionflags> */ 8117 8118 /* Compute the length of what follows. */ 8119 l = 2; /* count <repcount> or <salcount> */ 8120 for (i = 0; i < gap->ga_len; ++i) 8121 { 8122 ftp = &((fromto_T *)gap->ga_data)[i]; 8123 l += 1 + (int)STRLEN(ftp->ft_from); /* count <*fromlen> and <*from> */ 8124 l += 1 + (int)STRLEN(ftp->ft_to); /* count <*tolen> and <*to> */ 8125 } 8126 if (round == 2) 8127 ++l; /* count <salflags> */ 8128 put_bytes(fd, (long_u)l, 4); /* <sectionlen> */ 8129 8130 if (round == 2) 8131 { 8132 i = 0; 8133 if (spin->si_followup) 8134 i |= SAL_F0LLOWUP; 8135 if (spin->si_collapse) 8136 i |= SAL_COLLAPSE; 8137 if (spin->si_rem_accents) 8138 i |= SAL_REM_ACCENTS; 8139 putc(i, fd); /* <salflags> */ 8140 } 8141 8142 put_bytes(fd, (long_u)gap->ga_len, 2); /* <repcount> or <salcount> */ 8143 for (i = 0; i < gap->ga_len; ++i) 8144 { 8145 /* <rep> : <repfromlen> <repfrom> <reptolen> <repto> */ 8146 /* <sal> : <salfromlen> <salfrom> <saltolen> <salto> */ 8147 ftp = &((fromto_T *)gap->ga_data)[i]; 8148 for (rr = 1; rr <= 2; ++rr) 8149 { 8150 p = rr == 1 ? ftp->ft_from : ftp->ft_to; 8151 l = (int)STRLEN(p); 8152 putc(l, fd); 8153 if (l > 0) 8154 fwv &= fwrite(p, l, (size_t)1, fd); 8155 } 8156 } 8157 8158 } 8159 8160 /* SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto> 8161 * This is for making suggestions, section is not required. */ 8162 if (spin->si_sofofr != NULL && spin->si_sofoto != NULL) 8163 { 8164 putc(SN_SOFO, fd); /* <sectionID> */ 8165 putc(0, fd); /* <sectionflags> */ 8166 8167 l = (int)STRLEN(spin->si_sofofr); 8168 put_bytes(fd, (long_u)(l + STRLEN(spin->si_sofoto) + 4), 4); 8169 /* <sectionlen> */ 8170 8171 put_bytes(fd, (long_u)l, 2); /* <sofofromlen> */ 8172 fwv &= fwrite(spin->si_sofofr, l, (size_t)1, fd); /* <sofofrom> */ 8173 8174 l = (int)STRLEN(spin->si_sofoto); 8175 put_bytes(fd, (long_u)l, 2); /* <sofotolen> */ 8176 fwv &= fwrite(spin->si_sofoto, l, (size_t)1, fd); /* <sofoto> */ 8177 } 8178 8179 /* SN_WORDS: <word> ... 8180 * This is for making suggestions, section is not required. */ 8181 if (spin->si_commonwords.ht_used > 0) 8182 { 8183 putc(SN_WORDS, fd); /* <sectionID> */ 8184 putc(0, fd); /* <sectionflags> */ 8185 8186 /* round 1: count the bytes 8187 * round 2: write the bytes */ 8188 for (round = 1; round <= 2; ++round) 8189 { 8190 int todo; 8191 int len = 0; 8192 hashitem_T *hi; 8193 8194 todo = (int)spin->si_commonwords.ht_used; 8195 for (hi = spin->si_commonwords.ht_array; todo > 0; ++hi) 8196 if (!HASHITEM_EMPTY(hi)) 8197 { 8198 l = (int)STRLEN(hi->hi_key) + 1; 8199 len += l; 8200 if (round == 2) /* <word> */ 8201 fwv &= fwrite(hi->hi_key, (size_t)l, (size_t)1, fd); 8202 --todo; 8203 } 8204 if (round == 1) 8205 put_bytes(fd, (long_u)len, 4); /* <sectionlen> */ 8206 } 8207 } 8208 8209 /* SN_MAP: <mapstr> 8210 * This is for making suggestions, section is not required. */ 8211 if (spin->si_map.ga_len > 0) 8212 { 8213 putc(SN_MAP, fd); /* <sectionID> */ 8214 putc(0, fd); /* <sectionflags> */ 8215 l = spin->si_map.ga_len; 8216 put_bytes(fd, (long_u)l, 4); /* <sectionlen> */ 8217 fwv &= fwrite(spin->si_map.ga_data, (size_t)l, (size_t)1, fd); 8218 /* <mapstr> */ 8219 } 8220 8221 /* SN_SUGFILE: <timestamp> 8222 * This is used to notify that a .sug file may be available and at the 8223 * same time allows for checking that a .sug file that is found matches 8224 * with this .spl file. That's because the word numbers must be exactly 8225 * right. */ 8226 if (!spin->si_nosugfile 8227 && (spin->si_sal.ga_len > 0 8228 || (spin->si_sofofr != NULL && spin->si_sofoto != NULL))) 8229 { 8230 putc(SN_SUGFILE, fd); /* <sectionID> */ 8231 putc(0, fd); /* <sectionflags> */ 8232 put_bytes(fd, (long_u)8, 4); /* <sectionlen> */ 8233 8234 /* Set si_sugtime and write it to the file. */ 8235 spin->si_sugtime = time(NULL); 8236 put_time(fd, spin->si_sugtime); /* <timestamp> */ 8237 } 8238 8239 /* SN_NOSPLITSUGS: nothing 8240 * This is used to notify that no suggestions with word splits are to be 8241 * made. */ 8242 if (spin->si_nosplitsugs) 8243 { 8244 putc(SN_NOSPLITSUGS, fd); /* <sectionID> */ 8245 putc(0, fd); /* <sectionflags> */ 8246 put_bytes(fd, (long_u)0, 4); /* <sectionlen> */ 8247 } 8248 8249 /* SN_COMPOUND: compound info. 8250 * We don't mark it required, when not supported all compound words will 8251 * be bad words. */ 8252 if (spin->si_compflags != NULL) 8253 { 8254 putc(SN_COMPOUND, fd); /* <sectionID> */ 8255 putc(0, fd); /* <sectionflags> */ 8256 8257 l = (int)STRLEN(spin->si_compflags); 8258 for (i = 0; i < spin->si_comppat.ga_len; ++i) 8259 l += (int)STRLEN(((char_u **)(spin->si_comppat.ga_data))[i]) + 1; 8260 put_bytes(fd, (long_u)(l + 7), 4); /* <sectionlen> */ 8261 8262 putc(spin->si_compmax, fd); /* <compmax> */ 8263 putc(spin->si_compminlen, fd); /* <compminlen> */ 8264 putc(spin->si_compsylmax, fd); /* <compsylmax> */ 8265 putc(0, fd); /* for Vim 7.0b compatibility */ 8266 putc(spin->si_compoptions, fd); /* <compoptions> */ 8267 put_bytes(fd, (long_u)spin->si_comppat.ga_len, 2); 8268 /* <comppatcount> */ 8269 for (i = 0; i < spin->si_comppat.ga_len; ++i) 8270 { 8271 p = ((char_u **)(spin->si_comppat.ga_data))[i]; 8272 putc((int)STRLEN(p), fd); /* <comppatlen> */ 8273 fwv &= fwrite(p, (size_t)STRLEN(p), (size_t)1, fd); 8274 /* <comppattext> */ 8275 } 8276 /* <compflags> */ 8277 fwv &= fwrite(spin->si_compflags, (size_t)STRLEN(spin->si_compflags), 8278 (size_t)1, fd); 8279 } 8280 8281 /* SN_NOBREAK: NOBREAK flag */ 8282 if (spin->si_nobreak) 8283 { 8284 putc(SN_NOBREAK, fd); /* <sectionID> */ 8285 putc(0, fd); /* <sectionflags> */ 8286 8287 /* It's empty, the presence of the section flags the feature. */ 8288 put_bytes(fd, (long_u)0, 4); /* <sectionlen> */ 8289 } 8290 8291 /* SN_SYLLABLE: syllable info. 8292 * We don't mark it required, when not supported syllables will not be 8293 * counted. */ 8294 if (spin->si_syllable != NULL) 8295 { 8296 putc(SN_SYLLABLE, fd); /* <sectionID> */ 8297 putc(0, fd); /* <sectionflags> */ 8298 8299 l = (int)STRLEN(spin->si_syllable); 8300 put_bytes(fd, (long_u)l, 4); /* <sectionlen> */ 8301 fwv &= fwrite(spin->si_syllable, (size_t)l, (size_t)1, fd); 8302 /* <syllable> */ 8303 } 8304 8305 /* end of <SECTIONS> */ 8306 putc(SN_END, fd); /* <sectionend> */ 8307 8308 8309 /* 8310 * <LWORDTREE> <KWORDTREE> <PREFIXTREE> 8311 */ 8312 spin->si_memtot = 0; 8313 for (round = 1; round <= 3; ++round) 8314 { 8315 if (round == 1) 8316 tree = spin->si_foldroot->wn_sibling; 8317 else if (round == 2) 8318 tree = spin->si_keeproot->wn_sibling; 8319 else 8320 tree = spin->si_prefroot->wn_sibling; 8321 8322 /* Clear the index and wnode fields in the tree. */ 8323 clear_node(tree); 8324 8325 /* Count the number of nodes. Needed to be able to allocate the 8326 * memory when reading the nodes. Also fills in index for shared 8327 * nodes. */ 8328 nodecount = put_node(NULL, tree, 0, regionmask, round == 3); 8329 8330 /* number of nodes in 4 bytes */ 8331 put_bytes(fd, (long_u)nodecount, 4); /* <nodecount> */ 8332 spin->si_memtot += nodecount + nodecount * sizeof(int); 8333 8334 /* Write the nodes. */ 8335 (void)put_node(fd, tree, 0, regionmask, round == 3); 8336 } 8337 8338 /* Write another byte to check for errors (file system full). */ 8339 if (putc(0, fd) == EOF) 8340 retval = FAIL; 8341theend: 8342 if (fclose(fd) == EOF) 8343 retval = FAIL; 8344 8345 if (fwv != (size_t)1) 8346 retval = FAIL; 8347 if (retval == FAIL) 8348 EMSG(_(e_write)); 8349 8350 return retval; 8351} 8352 8353/* 8354 * Clear the index and wnode fields of "node", it siblings and its 8355 * children. This is needed because they are a union with other items to save 8356 * space. 8357 */ 8358 static void 8359clear_node(node) 8360 wordnode_T *node; 8361{ 8362 wordnode_T *np; 8363 8364 if (node != NULL) 8365 for (np = node; np != NULL; np = np->wn_sibling) 8366 { 8367 np->wn_u1.index = 0; 8368 np->wn_u2.wnode = NULL; 8369 8370 if (np->wn_byte != NUL) 8371 clear_node(np->wn_child); 8372 } 8373} 8374 8375 8376/* 8377 * Dump a word tree at node "node". 8378 * 8379 * This first writes the list of possible bytes (siblings). Then for each 8380 * byte recursively write the children. 8381 * 8382 * NOTE: The code here must match the code in read_tree_node(), since 8383 * assumptions are made about the indexes (so that we don't have to write them 8384 * in the file). 8385 * 8386 * Returns the number of nodes used. 8387 */ 8388 static int 8389put_node(fd, node, idx, regionmask, prefixtree) 8390 FILE *fd; /* NULL when only counting */ 8391 wordnode_T *node; 8392 int idx; 8393 int regionmask; 8394 int prefixtree; /* TRUE for PREFIXTREE */ 8395{ 8396 int newindex = idx; 8397 int siblingcount = 0; 8398 wordnode_T *np; 8399 int flags; 8400 8401 /* If "node" is zero the tree is empty. */ 8402 if (node == NULL) 8403 return 0; 8404 8405 /* Store the index where this node is written. */ 8406 node->wn_u1.index = idx; 8407 8408 /* Count the number of siblings. */ 8409 for (np = node; np != NULL; np = np->wn_sibling) 8410 ++siblingcount; 8411 8412 /* Write the sibling count. */ 8413 if (fd != NULL) 8414 putc(siblingcount, fd); /* <siblingcount> */ 8415 8416 /* Write each sibling byte and optionally extra info. */ 8417 for (np = node; np != NULL; np = np->wn_sibling) 8418 { 8419 if (np->wn_byte == 0) 8420 { 8421 if (fd != NULL) 8422 { 8423 /* For a NUL byte (end of word) write the flags etc. */ 8424 if (prefixtree) 8425 { 8426 /* In PREFIXTREE write the required affixID and the 8427 * associated condition nr (stored in wn_region). The 8428 * byte value is misused to store the "rare" and "not 8429 * combining" flags */ 8430 if (np->wn_flags == (short_u)PFX_FLAGS) 8431 putc(BY_NOFLAGS, fd); /* <byte> */ 8432 else 8433 { 8434 putc(BY_FLAGS, fd); /* <byte> */ 8435 putc(np->wn_flags, fd); /* <pflags> */ 8436 } 8437 putc(np->wn_affixID, fd); /* <affixID> */ 8438 put_bytes(fd, (long_u)np->wn_region, 2); /* <prefcondnr> */ 8439 } 8440 else 8441 { 8442 /* For word trees we write the flag/region items. */ 8443 flags = np->wn_flags; 8444 if (regionmask != 0 && np->wn_region != regionmask) 8445 flags |= WF_REGION; 8446 if (np->wn_affixID != 0) 8447 flags |= WF_AFX; 8448 if (flags == 0) 8449 { 8450 /* word without flags or region */ 8451 putc(BY_NOFLAGS, fd); /* <byte> */ 8452 } 8453 else 8454 { 8455 if (np->wn_flags >= 0x100) 8456 { 8457 putc(BY_FLAGS2, fd); /* <byte> */ 8458 putc(flags, fd); /* <flags> */ 8459 putc((unsigned)flags >> 8, fd); /* <flags2> */ 8460 } 8461 else 8462 { 8463 putc(BY_FLAGS, fd); /* <byte> */ 8464 putc(flags, fd); /* <flags> */ 8465 } 8466 if (flags & WF_REGION) 8467 putc(np->wn_region, fd); /* <region> */ 8468 if (flags & WF_AFX) 8469 putc(np->wn_affixID, fd); /* <affixID> */ 8470 } 8471 } 8472 } 8473 } 8474 else 8475 { 8476 if (np->wn_child->wn_u1.index != 0 8477 && np->wn_child->wn_u2.wnode != node) 8478 { 8479 /* The child is written elsewhere, write the reference. */ 8480 if (fd != NULL) 8481 { 8482 putc(BY_INDEX, fd); /* <byte> */ 8483 /* <nodeidx> */ 8484 put_bytes(fd, (long_u)np->wn_child->wn_u1.index, 3); 8485 } 8486 } 8487 else if (np->wn_child->wn_u2.wnode == NULL) 8488 /* We will write the child below and give it an index. */ 8489 np->wn_child->wn_u2.wnode = node; 8490 8491 if (fd != NULL) 8492 if (putc(np->wn_byte, fd) == EOF) /* <byte> or <xbyte> */ 8493 { 8494 EMSG(_(e_write)); 8495 return 0; 8496 } 8497 } 8498 } 8499 8500 /* Space used in the array when reading: one for each sibling and one for 8501 * the count. */ 8502 newindex += siblingcount + 1; 8503 8504 /* Recursively dump the children of each sibling. */ 8505 for (np = node; np != NULL; np = np->wn_sibling) 8506 if (np->wn_byte != 0 && np->wn_child->wn_u2.wnode == node) 8507 newindex = put_node(fd, np->wn_child, newindex, regionmask, 8508 prefixtree); 8509 8510 return newindex; 8511} 8512 8513 8514/* 8515 * ":mkspell [-ascii] outfile infile ..." 8516 * ":mkspell [-ascii] addfile" 8517 */ 8518 void 8519ex_mkspell(eap) 8520 exarg_T *eap; 8521{ 8522 int fcount; 8523 char_u **fnames; 8524 char_u *arg = eap->arg; 8525 int ascii = FALSE; 8526 8527 if (STRNCMP(arg, "-ascii", 6) == 0) 8528 { 8529 ascii = TRUE; 8530 arg = skipwhite(arg + 6); 8531 } 8532 8533 /* Expand all the remaining arguments (e.g., $VIMRUNTIME). */ 8534 if (get_arglist_exp(arg, &fcount, &fnames) == OK) 8535 { 8536 mkspell(fcount, fnames, ascii, eap->forceit, FALSE); 8537 FreeWild(fcount, fnames); 8538 } 8539} 8540 8541/* 8542 * Create the .sug file. 8543 * Uses the soundfold info in "spin". 8544 * Writes the file with the name "wfname", with ".spl" changed to ".sug". 8545 */ 8546 static void 8547spell_make_sugfile(spin, wfname) 8548 spellinfo_T *spin; 8549 char_u *wfname; 8550{ 8551 char_u fname[MAXPATHL]; 8552 int len; 8553 slang_T *slang; 8554 int free_slang = FALSE; 8555 8556 /* 8557 * Read back the .spl file that was written. This fills the required 8558 * info for soundfolding. This also uses less memory than the 8559 * pointer-linked version of the trie. And it avoids having two versions 8560 * of the code for the soundfolding stuff. 8561 * It might have been done already by spell_reload_one(). 8562 */ 8563 for (slang = first_lang; slang != NULL; slang = slang->sl_next) 8564 if (fullpathcmp(wfname, slang->sl_fname, FALSE) == FPC_SAME) 8565 break; 8566 if (slang == NULL) 8567 { 8568 spell_message(spin, (char_u *)_("Reading back spell file...")); 8569 slang = spell_load_file(wfname, NULL, NULL, FALSE); 8570 if (slang == NULL) 8571 return; 8572 free_slang = TRUE; 8573 } 8574 8575 /* 8576 * Clear the info in "spin" that is used. 8577 */ 8578 spin->si_blocks = NULL; 8579 spin->si_blocks_cnt = 0; 8580 spin->si_compress_cnt = 0; /* will stay at 0 all the time*/ 8581 spin->si_free_count = 0; 8582 spin->si_first_free = NULL; 8583 spin->si_foldwcount = 0; 8584 8585 /* 8586 * Go through the trie of good words, soundfold each word and add it to 8587 * the soundfold trie. 8588 */ 8589 spell_message(spin, (char_u *)_("Performing soundfolding...")); 8590 if (sug_filltree(spin, slang) == FAIL) 8591 goto theend; 8592 8593 /* 8594 * Create the table which links each soundfold word with a list of the 8595 * good words it may come from. Creates buffer "spin->si_spellbuf". 8596 * This also removes the wordnr from the NUL byte entries to make 8597 * compression possible. 8598 */ 8599 if (sug_maketable(spin) == FAIL) 8600 goto theend; 8601 8602 smsg((char_u *)_("Number of words after soundfolding: %ld"), 8603 (long)spin->si_spellbuf->b_ml.ml_line_count); 8604 8605 /* 8606 * Compress the soundfold trie. 8607 */ 8608 spell_message(spin, (char_u *)_(msg_compressing)); 8609 wordtree_compress(spin, spin->si_foldroot); 8610 8611 /* 8612 * Write the .sug file. 8613 * Make the file name by changing ".spl" to ".sug". 8614 */ 8615 STRCPY(fname, wfname); 8616 len = (int)STRLEN(fname); 8617 fname[len - 2] = 'u'; 8618 fname[len - 1] = 'g'; 8619 sug_write(spin, fname); 8620 8621theend: 8622 if (free_slang) 8623 slang_free(slang); 8624 free_blocks(spin->si_blocks); 8625 close_spellbuf(spin->si_spellbuf); 8626} 8627 8628/* 8629 * Build the soundfold trie for language "slang". 8630 */ 8631 static int 8632sug_filltree(spin, slang) 8633 spellinfo_T *spin; 8634 slang_T *slang; 8635{ 8636 char_u *byts; 8637 idx_T *idxs; 8638 int depth; 8639 idx_T arridx[MAXWLEN]; 8640 int curi[MAXWLEN]; 8641 char_u tword[MAXWLEN]; 8642 char_u tsalword[MAXWLEN]; 8643 int c; 8644 idx_T n; 8645 unsigned words_done = 0; 8646 int wordcount[MAXWLEN]; 8647 8648 /* We use si_foldroot for the souldfolded trie. */ 8649 spin->si_foldroot = wordtree_alloc(spin); 8650 if (spin->si_foldroot == NULL) 8651 return FAIL; 8652 8653 /* let tree_add_word() know we're adding to the soundfolded tree */ 8654 spin->si_sugtree = TRUE; 8655 8656 /* 8657 * Go through the whole case-folded tree, soundfold each word and put it 8658 * in the trie. 8659 */ 8660 byts = slang->sl_fbyts; 8661 idxs = slang->sl_fidxs; 8662 8663 arridx[0] = 0; 8664 curi[0] = 1; 8665 wordcount[0] = 0; 8666 8667 depth = 0; 8668 while (depth >= 0 && !got_int) 8669 { 8670 if (curi[depth] > byts[arridx[depth]]) 8671 { 8672 /* Done all bytes at this node, go up one level. */ 8673 idxs[arridx[depth]] = wordcount[depth]; 8674 if (depth > 0) 8675 wordcount[depth - 1] += wordcount[depth]; 8676 8677 --depth; 8678 line_breakcheck(); 8679 } 8680 else 8681 { 8682 8683 /* Do one more byte at this node. */ 8684 n = arridx[depth] + curi[depth]; 8685 ++curi[depth]; 8686 8687 c = byts[n]; 8688 if (c == 0) 8689 { 8690 /* Sound-fold the word. */ 8691 tword[depth] = NUL; 8692 spell_soundfold(slang, tword, TRUE, tsalword); 8693 8694 /* We use the "flags" field for the MSB of the wordnr, 8695 * "region" for the LSB of the wordnr. */ 8696 if (tree_add_word(spin, tsalword, spin->si_foldroot, 8697 words_done >> 16, words_done & 0xffff, 8698 0) == FAIL) 8699 return FAIL; 8700 8701 ++words_done; 8702 ++wordcount[depth]; 8703 8704 /* Reset the block count each time to avoid compression 8705 * kicking in. */ 8706 spin->si_blocks_cnt = 0; 8707 8708 /* Skip over any other NUL bytes (same word with different 8709 * flags). */ 8710 while (byts[n + 1] == 0) 8711 { 8712 ++n; 8713 ++curi[depth]; 8714 } 8715 } 8716 else 8717 { 8718 /* Normal char, go one level deeper. */ 8719 tword[depth++] = c; 8720 arridx[depth] = idxs[n]; 8721 curi[depth] = 1; 8722 wordcount[depth] = 0; 8723 } 8724 } 8725 } 8726 8727 smsg((char_u *)_("Total number of words: %d"), words_done); 8728 8729 return OK; 8730} 8731 8732/* 8733 * Make the table that links each word in the soundfold trie to the words it 8734 * can be produced from. 8735 * This is not unlike lines in a file, thus use a memfile to be able to access 8736 * the table efficiently. 8737 * Returns FAIL when out of memory. 8738 */ 8739 static int 8740sug_maketable(spin) 8741 spellinfo_T *spin; 8742{ 8743 garray_T ga; 8744 int res = OK; 8745 8746 /* Allocate a buffer, open a memline for it and create the swap file 8747 * (uses a temp file, not a .swp file). */ 8748 spin->si_spellbuf = open_spellbuf(); 8749 if (spin->si_spellbuf == NULL) 8750 return FAIL; 8751 8752 /* Use a buffer to store the line info, avoids allocating many small 8753 * pieces of memory. */ 8754 ga_init2(&ga, 1, 100); 8755 8756 /* recursively go through the tree */ 8757 if (sug_filltable(spin, spin->si_foldroot->wn_sibling, 0, &ga) == -1) 8758 res = FAIL; 8759 8760 ga_clear(&ga); 8761 return res; 8762} 8763 8764/* 8765 * Fill the table for one node and its children. 8766 * Returns the wordnr at the start of the node. 8767 * Returns -1 when out of memory. 8768 */ 8769 static int 8770sug_filltable(spin, node, startwordnr, gap) 8771 spellinfo_T *spin; 8772 wordnode_T *node; 8773 int startwordnr; 8774 garray_T *gap; /* place to store line of numbers */ 8775{ 8776 wordnode_T *p, *np; 8777 int wordnr = startwordnr; 8778 int nr; 8779 int prev_nr; 8780 8781 for (p = node; p != NULL; p = p->wn_sibling) 8782 { 8783 if (p->wn_byte == NUL) 8784 { 8785 gap->ga_len = 0; 8786 prev_nr = 0; 8787 for (np = p; np != NULL && np->wn_byte == NUL; np = np->wn_sibling) 8788 { 8789 if (ga_grow(gap, 10) == FAIL) 8790 return -1; 8791 8792 nr = (np->wn_flags << 16) + (np->wn_region & 0xffff); 8793 /* Compute the offset from the previous nr and store the 8794 * offset in a way that it takes a minimum number of bytes. 8795 * It's a bit like utf-8, but without the need to mark 8796 * following bytes. */ 8797 nr -= prev_nr; 8798 prev_nr += nr; 8799 gap->ga_len += offset2bytes(nr, 8800 (char_u *)gap->ga_data + gap->ga_len); 8801 } 8802 8803 /* add the NUL byte */ 8804 ((char_u *)gap->ga_data)[gap->ga_len++] = NUL; 8805 8806 if (ml_append_buf(spin->si_spellbuf, (linenr_T)wordnr, 8807 gap->ga_data, gap->ga_len, TRUE) == FAIL) 8808 return -1; 8809 ++wordnr; 8810 8811 /* Remove extra NUL entries, we no longer need them. We don't 8812 * bother freeing the nodes, the won't be reused anyway. */ 8813 while (p->wn_sibling != NULL && p->wn_sibling->wn_byte == NUL) 8814 p->wn_sibling = p->wn_sibling->wn_sibling; 8815 8816 /* Clear the flags on the remaining NUL node, so that compression 8817 * works a lot better. */ 8818 p->wn_flags = 0; 8819 p->wn_region = 0; 8820 } 8821 else 8822 { 8823 wordnr = sug_filltable(spin, p->wn_child, wordnr, gap); 8824 if (wordnr == -1) 8825 return -1; 8826 } 8827 } 8828 return wordnr; 8829} 8830 8831/* 8832 * Convert an offset into a minimal number of bytes. 8833 * Similar to utf_char2byters, but use 8 bits in followup bytes and avoid NUL 8834 * bytes. 8835 */ 8836 static int 8837offset2bytes(nr, buf) 8838 int nr; 8839 char_u *buf; 8840{ 8841 int rem; 8842 int b1, b2, b3, b4; 8843 8844 /* Split the number in parts of base 255. We need to avoid NUL bytes. */ 8845 b1 = nr % 255 + 1; 8846 rem = nr / 255; 8847 b2 = rem % 255 + 1; 8848 rem = rem / 255; 8849 b3 = rem % 255 + 1; 8850 b4 = rem / 255 + 1; 8851 8852 if (b4 > 1 || b3 > 0x1f) /* 4 bytes */ 8853 { 8854 buf[0] = 0xe0 + b4; 8855 buf[1] = b3; 8856 buf[2] = b2; 8857 buf[3] = b1; 8858 return 4; 8859 } 8860 if (b3 > 1 || b2 > 0x3f ) /* 3 bytes */ 8861 { 8862 buf[0] = 0xc0 + b3; 8863 buf[1] = b2; 8864 buf[2] = b1; 8865 return 3; 8866 } 8867 if (b2 > 1 || b1 > 0x7f ) /* 2 bytes */ 8868 { 8869 buf[0] = 0x80 + b2; 8870 buf[1] = b1; 8871 return 2; 8872 } 8873 /* 1 byte */ 8874 buf[0] = b1; 8875 return 1; 8876} 8877 8878/* 8879 * Opposite of offset2bytes(). 8880 * "pp" points to the bytes and is advanced over it. 8881 * Returns the offset. 8882 */ 8883 static int 8884bytes2offset(pp) 8885 char_u **pp; 8886{ 8887 char_u *p = *pp; 8888 int nr; 8889 int c; 8890 8891 c = *p++; 8892 if ((c & 0x80) == 0x00) /* 1 byte */ 8893 { 8894 nr = c - 1; 8895 } 8896 else if ((c & 0xc0) == 0x80) /* 2 bytes */ 8897 { 8898 nr = (c & 0x3f) - 1; 8899 nr = nr * 255 + (*p++ - 1); 8900 } 8901 else if ((c & 0xe0) == 0xc0) /* 3 bytes */ 8902 { 8903 nr = (c & 0x1f) - 1; 8904 nr = nr * 255 + (*p++ - 1); 8905 nr = nr * 255 + (*p++ - 1); 8906 } 8907 else /* 4 bytes */ 8908 { 8909 nr = (c & 0x0f) - 1; 8910 nr = nr * 255 + (*p++ - 1); 8911 nr = nr * 255 + (*p++ - 1); 8912 nr = nr * 255 + (*p++ - 1); 8913 } 8914 8915 *pp = p; 8916 return nr; 8917} 8918 8919/* 8920 * Write the .sug file in "fname". 8921 */ 8922 static void 8923sug_write(spin, fname) 8924 spellinfo_T *spin; 8925 char_u *fname; 8926{ 8927 FILE *fd; 8928 wordnode_T *tree; 8929 int nodecount; 8930 int wcount; 8931 char_u *line; 8932 linenr_T lnum; 8933 int len; 8934 8935 /* Create the file. Note that an existing file is silently overwritten! */ 8936 fd = mch_fopen((char *)fname, "w"); 8937 if (fd == NULL) 8938 { 8939 EMSG2(_(e_notopen), fname); 8940 return; 8941 } 8942 8943 vim_snprintf((char *)IObuff, IOSIZE, 8944 _("Writing suggestion file %s ..."), fname); 8945 spell_message(spin, IObuff); 8946 8947 /* 8948 * <SUGHEADER>: <fileID> <versionnr> <timestamp> 8949 */ 8950 if (fwrite(VIMSUGMAGIC, VIMSUGMAGICL, (size_t)1, fd) != 1) /* <fileID> */ 8951 { 8952 EMSG(_(e_write)); 8953 goto theend; 8954 } 8955 putc(VIMSUGVERSION, fd); /* <versionnr> */ 8956 8957 /* Write si_sugtime to the file. */ 8958 put_time(fd, spin->si_sugtime); /* <timestamp> */ 8959 8960 /* 8961 * <SUGWORDTREE> 8962 */ 8963 spin->si_memtot = 0; 8964 tree = spin->si_foldroot->wn_sibling; 8965 8966 /* Clear the index and wnode fields in the tree. */ 8967 clear_node(tree); 8968 8969 /* Count the number of nodes. Needed to be able to allocate the 8970 * memory when reading the nodes. Also fills in index for shared 8971 * nodes. */ 8972 nodecount = put_node(NULL, tree, 0, 0, FALSE); 8973 8974 /* number of nodes in 4 bytes */ 8975 put_bytes(fd, (long_u)nodecount, 4); /* <nodecount> */ 8976 spin->si_memtot += nodecount + nodecount * sizeof(int); 8977 8978 /* Write the nodes. */ 8979 (void)put_node(fd, tree, 0, 0, FALSE); 8980 8981 /* 8982 * <SUGTABLE>: <sugwcount> <sugline> ... 8983 */ 8984 wcount = spin->si_spellbuf->b_ml.ml_line_count; 8985 put_bytes(fd, (long_u)wcount, 4); /* <sugwcount> */ 8986 8987 for (lnum = 1; lnum <= (linenr_T)wcount; ++lnum) 8988 { 8989 /* <sugline>: <sugnr> ... NUL */ 8990 line = ml_get_buf(spin->si_spellbuf, lnum, FALSE); 8991 len = (int)STRLEN(line) + 1; 8992 if (fwrite(line, (size_t)len, (size_t)1, fd) == 0) 8993 { 8994 EMSG(_(e_write)); 8995 goto theend; 8996 } 8997 spin->si_memtot += len; 8998 } 8999 9000 /* Write another byte to check for errors. */ 9001 if (putc(0, fd) == EOF) 9002 EMSG(_(e_write)); 9003 9004 vim_snprintf((char *)IObuff, IOSIZE, 9005 _("Estimated runtime memory use: %d bytes"), spin->si_memtot); 9006 spell_message(spin, IObuff); 9007 9008theend: 9009 /* close the file */ 9010 fclose(fd); 9011} 9012 9013/* 9014 * Open a spell buffer. This is a nameless buffer that is not in the buffer 9015 * list and only contains text lines. Can use a swapfile to reduce memory 9016 * use. 9017 * Most other fields are invalid! Esp. watch out for string options being 9018 * NULL and there is no undo info. 9019 * Returns NULL when out of memory. 9020 */ 9021 static buf_T * 9022open_spellbuf() 9023{ 9024 buf_T *buf; 9025 9026 buf = (buf_T *)alloc_clear(sizeof(buf_T)); 9027 if (buf != NULL) 9028 { 9029 buf->b_spell = TRUE; 9030 buf->b_p_swf = TRUE; /* may create a swap file */ 9031 ml_open(buf); 9032 ml_open_file(buf); /* create swap file now */ 9033 } 9034 return buf; 9035} 9036 9037/* 9038 * Close the buffer used for spell info. 9039 */ 9040 static void 9041close_spellbuf(buf) 9042 buf_T *buf; 9043{ 9044 if (buf != NULL) 9045 { 9046 ml_close(buf, TRUE); 9047 vim_free(buf); 9048 } 9049} 9050 9051 9052/* 9053 * Create a Vim spell file from one or more word lists. 9054 * "fnames[0]" is the output file name. 9055 * "fnames[fcount - 1]" is the last input file name. 9056 * Exception: when "fnames[0]" ends in ".add" it's used as the input file name 9057 * and ".spl" is appended to make the output file name. 9058 */ 9059 static void 9060mkspell(fcount, fnames, ascii, overwrite, added_word) 9061 int fcount; 9062 char_u **fnames; 9063 int ascii; /* -ascii argument given */ 9064 int overwrite; /* overwrite existing output file */ 9065 int added_word; /* invoked through "zg" */ 9066{ 9067 char_u fname[MAXPATHL]; 9068 char_u wfname[MAXPATHL]; 9069 char_u **innames; 9070 int incount; 9071 afffile_T *(afile[8]); 9072 int i; 9073 int len; 9074 struct stat st; 9075 int error = FALSE; 9076 spellinfo_T spin; 9077 9078 vim_memset(&spin, 0, sizeof(spin)); 9079 spin.si_verbose = !added_word; 9080 spin.si_ascii = ascii; 9081 spin.si_followup = TRUE; 9082 spin.si_rem_accents = TRUE; 9083 ga_init2(&spin.si_rep, (int)sizeof(fromto_T), 20); 9084 ga_init2(&spin.si_repsal, (int)sizeof(fromto_T), 20); 9085 ga_init2(&spin.si_sal, (int)sizeof(fromto_T), 20); 9086 ga_init2(&spin.si_map, (int)sizeof(char_u), 100); 9087 ga_init2(&spin.si_comppat, (int)sizeof(char_u *), 20); 9088 ga_init2(&spin.si_prefcond, (int)sizeof(char_u *), 50); 9089 hash_init(&spin.si_commonwords); 9090 spin.si_newcompID = 127; /* start compound ID at first maximum */ 9091 9092 /* default: fnames[0] is output file, following are input files */ 9093 innames = &fnames[1]; 9094 incount = fcount - 1; 9095 9096 if (fcount >= 1) 9097 { 9098 len = (int)STRLEN(fnames[0]); 9099 if (fcount == 1 && len > 4 && STRCMP(fnames[0] + len - 4, ".add") == 0) 9100 { 9101 /* For ":mkspell path/en.latin1.add" output file is 9102 * "path/en.latin1.add.spl". */ 9103 innames = &fnames[0]; 9104 incount = 1; 9105 vim_snprintf((char *)wfname, sizeof(wfname), "%s.spl", fnames[0]); 9106 } 9107 else if (fcount == 1) 9108 { 9109 /* For ":mkspell path/vim" output file is "path/vim.latin1.spl". */ 9110 innames = &fnames[0]; 9111 incount = 1; 9112 vim_snprintf((char *)wfname, sizeof(wfname), "%s.%s.spl", fnames[0], 9113 spin.si_ascii ? (char_u *)"ascii" : spell_enc()); 9114 } 9115 else if (len > 4 && STRCMP(fnames[0] + len - 4, ".spl") == 0) 9116 { 9117 /* Name ends in ".spl", use as the file name. */ 9118 vim_strncpy(wfname, fnames[0], sizeof(wfname) - 1); 9119 } 9120 else 9121 /* Name should be language, make the file name from it. */ 9122 vim_snprintf((char *)wfname, sizeof(wfname), "%s.%s.spl", fnames[0], 9123 spin.si_ascii ? (char_u *)"ascii" : spell_enc()); 9124 9125 /* Check for .ascii.spl. */ 9126 if (strstr((char *)gettail(wfname), ".ascii.") != NULL) 9127 spin.si_ascii = TRUE; 9128 9129 /* Check for .add.spl. */ 9130 if (strstr((char *)gettail(wfname), ".add.") != NULL) 9131 spin.si_add = TRUE; 9132 } 9133 9134 if (incount <= 0) 9135 EMSG(_(e_invarg)); /* need at least output and input names */ 9136 else if (vim_strchr(gettail(wfname), '_') != NULL) 9137 EMSG(_("E751: Output file name must not have region name")); 9138 else if (incount > 8) 9139 EMSG(_("E754: Only up to 8 regions supported")); 9140 else 9141 { 9142 /* Check for overwriting before doing things that may take a lot of 9143 * time. */ 9144 if (!overwrite && mch_stat((char *)wfname, &st) >= 0) 9145 { 9146 EMSG(_(e_exists)); 9147 return; 9148 } 9149 if (mch_isdir(wfname)) 9150 { 9151 EMSG2(_(e_isadir2), wfname); 9152 return; 9153 } 9154 9155 /* 9156 * Init the aff and dic pointers. 9157 * Get the region names if there are more than 2 arguments. 9158 */ 9159 for (i = 0; i < incount; ++i) 9160 { 9161 afile[i] = NULL; 9162 9163 if (incount > 1) 9164 { 9165 len = (int)STRLEN(innames[i]); 9166 if (STRLEN(gettail(innames[i])) < 5 9167 || innames[i][len - 3] != '_') 9168 { 9169 EMSG2(_("E755: Invalid region in %s"), innames[i]); 9170 return; 9171 } 9172 spin.si_region_name[i * 2] = TOLOWER_ASC(innames[i][len - 2]); 9173 spin.si_region_name[i * 2 + 1] = 9174 TOLOWER_ASC(innames[i][len - 1]); 9175 } 9176 } 9177 spin.si_region_count = incount; 9178 9179 spin.si_foldroot = wordtree_alloc(&spin); 9180 spin.si_keeproot = wordtree_alloc(&spin); 9181 spin.si_prefroot = wordtree_alloc(&spin); 9182 if (spin.si_foldroot == NULL 9183 || spin.si_keeproot == NULL 9184 || spin.si_prefroot == NULL) 9185 { 9186 free_blocks(spin.si_blocks); 9187 return; 9188 } 9189 9190 /* When not producing a .add.spl file clear the character table when 9191 * we encounter one in the .aff file. This means we dump the current 9192 * one in the .spl file if the .aff file doesn't define one. That's 9193 * better than guessing the contents, the table will match a 9194 * previously loaded spell file. */ 9195 if (!spin.si_add) 9196 spin.si_clear_chartab = TRUE; 9197 9198 /* 9199 * Read all the .aff and .dic files. 9200 * Text is converted to 'encoding'. 9201 * Words are stored in the case-folded and keep-case trees. 9202 */ 9203 for (i = 0; i < incount && !error; ++i) 9204 { 9205 spin.si_conv.vc_type = CONV_NONE; 9206 spin.si_region = 1 << i; 9207 9208 vim_snprintf((char *)fname, sizeof(fname), "%s.aff", innames[i]); 9209 if (mch_stat((char *)fname, &st) >= 0) 9210 { 9211 /* Read the .aff file. Will init "spin->si_conv" based on the 9212 * "SET" line. */ 9213 afile[i] = spell_read_aff(&spin, fname); 9214 if (afile[i] == NULL) 9215 error = TRUE; 9216 else 9217 { 9218 /* Read the .dic file and store the words in the trees. */ 9219 vim_snprintf((char *)fname, sizeof(fname), "%s.dic", 9220 innames[i]); 9221 if (spell_read_dic(&spin, fname, afile[i]) == FAIL) 9222 error = TRUE; 9223 } 9224 } 9225 else 9226 { 9227 /* No .aff file, try reading the file as a word list. Store 9228 * the words in the trees. */ 9229 if (spell_read_wordfile(&spin, innames[i]) == FAIL) 9230 error = TRUE; 9231 } 9232 9233#ifdef FEAT_MBYTE 9234 /* Free any conversion stuff. */ 9235 convert_setup(&spin.si_conv, NULL, NULL); 9236#endif 9237 } 9238 9239 if (spin.si_compflags != NULL && spin.si_nobreak) 9240 MSG(_("Warning: both compounding and NOBREAK specified")); 9241 9242 if (!error && !got_int) 9243 { 9244 /* 9245 * Combine tails in the tree. 9246 */ 9247 spell_message(&spin, (char_u *)_(msg_compressing)); 9248 wordtree_compress(&spin, spin.si_foldroot); 9249 wordtree_compress(&spin, spin.si_keeproot); 9250 wordtree_compress(&spin, spin.si_prefroot); 9251 } 9252 9253 if (!error && !got_int) 9254 { 9255 /* 9256 * Write the info in the spell file. 9257 */ 9258 vim_snprintf((char *)IObuff, IOSIZE, 9259 _("Writing spell file %s ..."), wfname); 9260 spell_message(&spin, IObuff); 9261 9262 error = write_vim_spell(&spin, wfname) == FAIL; 9263 9264 spell_message(&spin, (char_u *)_("Done!")); 9265 vim_snprintf((char *)IObuff, IOSIZE, 9266 _("Estimated runtime memory use: %d bytes"), spin.si_memtot); 9267 spell_message(&spin, IObuff); 9268 9269 /* 9270 * If the file is loaded need to reload it. 9271 */ 9272 if (!error) 9273 spell_reload_one(wfname, added_word); 9274 } 9275 9276 /* Free the allocated memory. */ 9277 ga_clear(&spin.si_rep); 9278 ga_clear(&spin.si_repsal); 9279 ga_clear(&spin.si_sal); 9280 ga_clear(&spin.si_map); 9281 ga_clear(&spin.si_comppat); 9282 ga_clear(&spin.si_prefcond); 9283 hash_clear_all(&spin.si_commonwords, 0); 9284 9285 /* Free the .aff file structures. */ 9286 for (i = 0; i < incount; ++i) 9287 if (afile[i] != NULL) 9288 spell_free_aff(afile[i]); 9289 9290 /* Free all the bits and pieces at once. */ 9291 free_blocks(spin.si_blocks); 9292 9293 /* 9294 * If there is soundfolding info and no NOSUGFILE item create the 9295 * .sug file with the soundfolded word trie. 9296 */ 9297 if (spin.si_sugtime != 0 && !error && !got_int) 9298 spell_make_sugfile(&spin, wfname); 9299 9300 } 9301} 9302 9303/* 9304 * Display a message for spell file processing when 'verbose' is set or using 9305 * ":mkspell". "str" can be IObuff. 9306 */ 9307 static void 9308spell_message(spin, str) 9309 spellinfo_T *spin; 9310 char_u *str; 9311{ 9312 if (spin->si_verbose || p_verbose > 2) 9313 { 9314 if (!spin->si_verbose) 9315 verbose_enter(); 9316 MSG(str); 9317 out_flush(); 9318 if (!spin->si_verbose) 9319 verbose_leave(); 9320 } 9321} 9322 9323/* 9324 * ":[count]spellgood {word}" 9325 * ":[count]spellwrong {word}" 9326 * ":[count]spellundo {word}" 9327 */ 9328 void 9329ex_spell(eap) 9330 exarg_T *eap; 9331{ 9332 spell_add_word(eap->arg, (int)STRLEN(eap->arg), eap->cmdidx == CMD_spellwrong, 9333 eap->forceit ? 0 : (int)eap->line2, 9334 eap->cmdidx == CMD_spellundo); 9335} 9336 9337/* 9338 * Add "word[len]" to 'spellfile' as a good or bad word. 9339 */ 9340 void 9341spell_add_word(word, len, bad, idx, undo) 9342 char_u *word; 9343 int len; 9344 int bad; 9345 int idx; /* "zG" and "zW": zero, otherwise index in 9346 'spellfile' */ 9347 int undo; /* TRUE for "zug", "zuG", "zuw" and "zuW" */ 9348{ 9349 FILE *fd = NULL; 9350 buf_T *buf = NULL; 9351 int new_spf = FALSE; 9352 char_u *fname; 9353 char_u fnamebuf[MAXPATHL]; 9354 char_u line[MAXWLEN * 2]; 9355 long fpos, fpos_next = 0; 9356 int i; 9357 char_u *spf; 9358 9359 if (idx == 0) /* use internal wordlist */ 9360 { 9361 if (int_wordlist == NULL) 9362 { 9363 int_wordlist = vim_tempname('s'); 9364 if (int_wordlist == NULL) 9365 return; 9366 } 9367 fname = int_wordlist; 9368 } 9369 else 9370 { 9371 /* If 'spellfile' isn't set figure out a good default value. */ 9372 if (*curwin->w_s->b_p_spf == NUL) 9373 { 9374 init_spellfile(); 9375 new_spf = TRUE; 9376 } 9377 9378 if (*curwin->w_s->b_p_spf == NUL) 9379 { 9380 EMSG2(_(e_notset), "spellfile"); 9381 return; 9382 } 9383 9384 for (spf = curwin->w_s->b_p_spf, i = 1; *spf != NUL; ++i) 9385 { 9386 copy_option_part(&spf, fnamebuf, MAXPATHL, ","); 9387 if (i == idx) 9388 break; 9389 if (*spf == NUL) 9390 { 9391 EMSGN(_("E765: 'spellfile' does not have %ld entries"), idx); 9392 return; 9393 } 9394 } 9395 9396 /* Check that the user isn't editing the .add file somewhere. */ 9397 buf = buflist_findname_exp(fnamebuf); 9398 if (buf != NULL && buf->b_ml.ml_mfp == NULL) 9399 buf = NULL; 9400 if (buf != NULL && bufIsChanged(buf)) 9401 { 9402 EMSG(_(e_bufloaded)); 9403 return; 9404 } 9405 9406 fname = fnamebuf; 9407 } 9408 9409 if (bad || undo) 9410 { 9411 /* When the word appears as good word we need to remove that one, 9412 * since its flags sort before the one with WF_BANNED. */ 9413 fd = mch_fopen((char *)fname, "r"); 9414 if (fd != NULL) 9415 { 9416 while (!vim_fgets(line, MAXWLEN * 2, fd)) 9417 { 9418 fpos = fpos_next; 9419 fpos_next = ftell(fd); 9420 if (STRNCMP(word, line, len) == 0 9421 && (line[len] == '/' || line[len] < ' ')) 9422 { 9423 /* Found duplicate word. Remove it by writing a '#' at 9424 * the start of the line. Mixing reading and writing 9425 * doesn't work for all systems, close the file first. */ 9426 fclose(fd); 9427 fd = mch_fopen((char *)fname, "r+"); 9428 if (fd == NULL) 9429 break; 9430 if (fseek(fd, fpos, SEEK_SET) == 0) 9431 { 9432 fputc('#', fd); 9433 if (undo) 9434 { 9435 home_replace(NULL, fname, NameBuff, MAXPATHL, TRUE); 9436 smsg((char_u *)_("Word removed from %s"), NameBuff); 9437 } 9438 } 9439 fseek(fd, fpos_next, SEEK_SET); 9440 } 9441 } 9442 if (fd != NULL) 9443 fclose(fd); 9444 } 9445 } 9446 9447 if (!undo) 9448 { 9449 fd = mch_fopen((char *)fname, "a"); 9450 if (fd == NULL && new_spf) 9451 { 9452 char_u *p; 9453 9454 /* We just initialized the 'spellfile' option and can't open the 9455 * file. We may need to create the "spell" directory first. We 9456 * already checked the runtime directory is writable in 9457 * init_spellfile(). */ 9458 if (!dir_of_file_exists(fname) && (p = gettail_sep(fname)) != fname) 9459 { 9460 int c = *p; 9461 9462 /* The directory doesn't exist. Try creating it and opening 9463 * the file again. */ 9464 *p = NUL; 9465 vim_mkdir(fname, 0755); 9466 *p = c; 9467 fd = mch_fopen((char *)fname, "a"); 9468 } 9469 } 9470 9471 if (fd == NULL) 9472 EMSG2(_(e_notopen), fname); 9473 else 9474 { 9475 if (bad) 9476 fprintf(fd, "%.*s/!\n", len, word); 9477 else 9478 fprintf(fd, "%.*s\n", len, word); 9479 fclose(fd); 9480 9481 home_replace(NULL, fname, NameBuff, MAXPATHL, TRUE); 9482 smsg((char_u *)_("Word added to %s"), NameBuff); 9483 } 9484 } 9485 9486 if (fd != NULL) 9487 { 9488 /* Update the .add.spl file. */ 9489 mkspell(1, &fname, FALSE, TRUE, TRUE); 9490 9491 /* If the .add file is edited somewhere, reload it. */ 9492 if (buf != NULL) 9493 buf_reload(buf, buf->b_orig_mode); 9494 9495 redraw_all_later(SOME_VALID); 9496 } 9497} 9498 9499/* 9500 * Initialize 'spellfile' for the current buffer. 9501 */ 9502 static void 9503init_spellfile() 9504{ 9505 char_u buf[MAXPATHL]; 9506 int l; 9507 char_u *fname; 9508 char_u *rtp; 9509 char_u *lend; 9510 int aspath = FALSE; 9511 char_u *lstart = curbuf->b_s.b_p_spl; 9512 9513 if (*curwin->w_s->b_p_spl != NUL && curwin->w_s->b_langp.ga_len > 0) 9514 { 9515 /* Find the end of the language name. Exclude the region. If there 9516 * is a path separator remember the start of the tail. */ 9517 for (lend = curwin->w_s->b_p_spl; *lend != NUL 9518 && vim_strchr((char_u *)",._", *lend) == NULL; ++lend) 9519 if (vim_ispathsep(*lend)) 9520 { 9521 aspath = TRUE; 9522 lstart = lend + 1; 9523 } 9524 9525 /* Loop over all entries in 'runtimepath'. Use the first one where we 9526 * are allowed to write. */ 9527 rtp = p_rtp; 9528 while (*rtp != NUL) 9529 { 9530 if (aspath) 9531 /* Use directory of an entry with path, e.g., for 9532 * "/dir/lg.utf-8.spl" use "/dir". */ 9533 vim_strncpy(buf, curbuf->b_s.b_p_spl, lstart - curbuf->b_s.b_p_spl - 1); 9534 else 9535 /* Copy the path from 'runtimepath' to buf[]. */ 9536 copy_option_part(&rtp, buf, MAXPATHL, ","); 9537 if (filewritable(buf) == 2) 9538 { 9539 /* Use the first language name from 'spelllang' and the 9540 * encoding used in the first loaded .spl file. */ 9541 if (aspath) 9542 vim_strncpy(buf, curbuf->b_s.b_p_spl, lend - curbuf->b_s.b_p_spl); 9543 else 9544 { 9545 /* Create the "spell" directory if it doesn't exist yet. */ 9546 l = (int)STRLEN(buf); 9547 vim_snprintf((char *)buf + l, MAXPATHL - l, "/spell"); 9548 if (!filewritable(buf) != 2) 9549 vim_mkdir(buf, 0755); 9550 9551 l = (int)STRLEN(buf); 9552 vim_snprintf((char *)buf + l, MAXPATHL - l, 9553 "/%.*s", (int)(lend - lstart), lstart); 9554 } 9555 l = (int)STRLEN(buf); 9556 fname = LANGP_ENTRY(curwin->w_s->b_langp, 0)->lp_slang->sl_fname; 9557 vim_snprintf((char *)buf + l, MAXPATHL - l, ".%s.add", 9558 fname != NULL 9559 && strstr((char *)gettail(fname), ".ascii.") != NULL 9560 ? (char_u *)"ascii" : spell_enc()); 9561 set_option_value((char_u *)"spellfile", 0L, buf, OPT_LOCAL); 9562 break; 9563 } 9564 aspath = FALSE; 9565 } 9566 } 9567} 9568 9569 9570/* 9571 * Init the chartab used for spelling for ASCII. 9572 * EBCDIC is not supported! 9573 */ 9574 static void 9575clear_spell_chartab(sp) 9576 spelltab_T *sp; 9577{ 9578 int i; 9579 9580 /* Init everything to FALSE. */ 9581 vim_memset(sp->st_isw, FALSE, sizeof(sp->st_isw)); 9582 vim_memset(sp->st_isu, FALSE, sizeof(sp->st_isu)); 9583 for (i = 0; i < 256; ++i) 9584 { 9585 sp->st_fold[i] = i; 9586 sp->st_upper[i] = i; 9587 } 9588 9589 /* We include digits. A word shouldn't start with a digit, but handling 9590 * that is done separately. */ 9591 for (i = '0'; i <= '9'; ++i) 9592 sp->st_isw[i] = TRUE; 9593 for (i = 'A'; i <= 'Z'; ++i) 9594 { 9595 sp->st_isw[i] = TRUE; 9596 sp->st_isu[i] = TRUE; 9597 sp->st_fold[i] = i + 0x20; 9598 } 9599 for (i = 'a'; i <= 'z'; ++i) 9600 { 9601 sp->st_isw[i] = TRUE; 9602 sp->st_upper[i] = i - 0x20; 9603 } 9604} 9605 9606/* 9607 * Init the chartab used for spelling. Only depends on 'encoding'. 9608 * Called once while starting up and when 'encoding' changes. 9609 * The default is to use isalpha(), but the spell file should define the word 9610 * characters to make it possible that 'encoding' differs from the current 9611 * locale. For utf-8 we don't use isalpha() but our own functions. 9612 */ 9613 void 9614init_spell_chartab() 9615{ 9616 int i; 9617 9618 did_set_spelltab = FALSE; 9619 clear_spell_chartab(&spelltab); 9620#ifdef FEAT_MBYTE 9621 if (enc_dbcs) 9622 { 9623 /* DBCS: assume double-wide characters are word characters. */ 9624 for (i = 128; i <= 255; ++i) 9625 if (MB_BYTE2LEN(i) == 2) 9626 spelltab.st_isw[i] = TRUE; 9627 } 9628 else if (enc_utf8) 9629 { 9630 for (i = 128; i < 256; ++i) 9631 { 9632 int f = utf_fold(i); 9633 int u = utf_toupper(i); 9634 9635 spelltab.st_isu[i] = utf_isupper(i); 9636 spelltab.st_isw[i] = spelltab.st_isu[i] || utf_islower(i); 9637 /* The folded/upper-cased value is different between latin1 and 9638 * utf8 for 0xb5, causing E763 for no good reason. Use the latin1 9639 * value for utf-8 to avoid this. */ 9640 spelltab.st_fold[i] = (f < 256) ? f : i; 9641 spelltab.st_upper[i] = (u < 256) ? u : i; 9642 } 9643 } 9644 else 9645#endif 9646 { 9647 /* Rough guess: use locale-dependent library functions. */ 9648 for (i = 128; i < 256; ++i) 9649 { 9650 if (MB_ISUPPER(i)) 9651 { 9652 spelltab.st_isw[i] = TRUE; 9653 spelltab.st_isu[i] = TRUE; 9654 spelltab.st_fold[i] = MB_TOLOWER(i); 9655 } 9656 else if (MB_ISLOWER(i)) 9657 { 9658 spelltab.st_isw[i] = TRUE; 9659 spelltab.st_upper[i] = MB_TOUPPER(i); 9660 } 9661 } 9662 } 9663} 9664 9665/* 9666 * Set the spell character tables from strings in the affix file. 9667 */ 9668 static int 9669set_spell_chartab(fol, low, upp) 9670 char_u *fol; 9671 char_u *low; 9672 char_u *upp; 9673{ 9674 /* We build the new tables here first, so that we can compare with the 9675 * previous one. */ 9676 spelltab_T new_st; 9677 char_u *pf = fol, *pl = low, *pu = upp; 9678 int f, l, u; 9679 9680 clear_spell_chartab(&new_st); 9681 9682 while (*pf != NUL) 9683 { 9684 if (*pl == NUL || *pu == NUL) 9685 { 9686 EMSG(_(e_affform)); 9687 return FAIL; 9688 } 9689#ifdef FEAT_MBYTE 9690 f = mb_ptr2char_adv(&pf); 9691 l = mb_ptr2char_adv(&pl); 9692 u = mb_ptr2char_adv(&pu); 9693#else 9694 f = *pf++; 9695 l = *pl++; 9696 u = *pu++; 9697#endif 9698 /* Every character that appears is a word character. */ 9699 if (f < 256) 9700 new_st.st_isw[f] = TRUE; 9701 if (l < 256) 9702 new_st.st_isw[l] = TRUE; 9703 if (u < 256) 9704 new_st.st_isw[u] = TRUE; 9705 9706 /* if "LOW" and "FOL" are not the same the "LOW" char needs 9707 * case-folding */ 9708 if (l < 256 && l != f) 9709 { 9710 if (f >= 256) 9711 { 9712 EMSG(_(e_affrange)); 9713 return FAIL; 9714 } 9715 new_st.st_fold[l] = f; 9716 } 9717 9718 /* if "UPP" and "FOL" are not the same the "UPP" char needs 9719 * case-folding, it's upper case and the "UPP" is the upper case of 9720 * "FOL" . */ 9721 if (u < 256 && u != f) 9722 { 9723 if (f >= 256) 9724 { 9725 EMSG(_(e_affrange)); 9726 return FAIL; 9727 } 9728 new_st.st_fold[u] = f; 9729 new_st.st_isu[u] = TRUE; 9730 new_st.st_upper[f] = u; 9731 } 9732 } 9733 9734 if (*pl != NUL || *pu != NUL) 9735 { 9736 EMSG(_(e_affform)); 9737 return FAIL; 9738 } 9739 9740 return set_spell_finish(&new_st); 9741} 9742 9743/* 9744 * Set the spell character tables from strings in the .spl file. 9745 */ 9746 static void 9747set_spell_charflags(flags, cnt, fol) 9748 char_u *flags; 9749 int cnt; /* length of "flags" */ 9750 char_u *fol; 9751{ 9752 /* We build the new tables here first, so that we can compare with the 9753 * previous one. */ 9754 spelltab_T new_st; 9755 int i; 9756 char_u *p = fol; 9757 int c; 9758 9759 clear_spell_chartab(&new_st); 9760 9761 for (i = 0; i < 128; ++i) 9762 { 9763 if (i < cnt) 9764 { 9765 new_st.st_isw[i + 128] = (flags[i] & CF_WORD) != 0; 9766 new_st.st_isu[i + 128] = (flags[i] & CF_UPPER) != 0; 9767 } 9768 9769 if (*p != NUL) 9770 { 9771#ifdef FEAT_MBYTE 9772 c = mb_ptr2char_adv(&p); 9773#else 9774 c = *p++; 9775#endif 9776 new_st.st_fold[i + 128] = c; 9777 if (i + 128 != c && new_st.st_isu[i + 128] && c < 256) 9778 new_st.st_upper[c] = i + 128; 9779 } 9780 } 9781 9782 (void)set_spell_finish(&new_st); 9783} 9784 9785 static int 9786set_spell_finish(new_st) 9787 spelltab_T *new_st; 9788{ 9789 int i; 9790 9791 if (did_set_spelltab) 9792 { 9793 /* check that it's the same table */ 9794 for (i = 0; i < 256; ++i) 9795 { 9796 if (spelltab.st_isw[i] != new_st->st_isw[i] 9797 || spelltab.st_isu[i] != new_st->st_isu[i] 9798 || spelltab.st_fold[i] != new_st->st_fold[i] 9799 || spelltab.st_upper[i] != new_st->st_upper[i]) 9800 { 9801 EMSG(_("E763: Word characters differ between spell files")); 9802 return FAIL; 9803 } 9804 } 9805 } 9806 else 9807 { 9808 /* copy the new spelltab into the one being used */ 9809 spelltab = *new_st; 9810 did_set_spelltab = TRUE; 9811 } 9812 9813 return OK; 9814} 9815 9816/* 9817 * Return TRUE if "p" points to a word character. 9818 * As a special case we see "midword" characters as word character when it is 9819 * followed by a word character. This finds they'there but not 'they there'. 9820 * Thus this only works properly when past the first character of the word. 9821 */ 9822 static int 9823spell_iswordp(p, wp) 9824 char_u *p; 9825 win_T *wp; /* buffer used */ 9826{ 9827#ifdef FEAT_MBYTE 9828 char_u *s; 9829 int l; 9830 int c; 9831 9832 if (has_mbyte) 9833 { 9834 l = MB_BYTE2LEN(*p); 9835 s = p; 9836 if (l == 1) 9837 { 9838 /* be quick for ASCII */ 9839 if (wp->w_s->b_spell_ismw[*p]) 9840 { 9841 s = p + 1; /* skip a mid-word character */ 9842 l = MB_BYTE2LEN(*s); 9843 } 9844 } 9845 else 9846 { 9847 c = mb_ptr2char(p); 9848 if (c < 256 ? wp->w_s->b_spell_ismw[c] 9849 : (wp->w_s->b_spell_ismw_mb != NULL 9850 && vim_strchr(wp->w_s->b_spell_ismw_mb, c) != NULL)) 9851 { 9852 s = p + l; 9853 l = MB_BYTE2LEN(*s); 9854 } 9855 } 9856 9857 c = mb_ptr2char(s); 9858 if (c > 255) 9859 return spell_mb_isword_class(mb_get_class(s)); 9860 return spelltab.st_isw[c]; 9861 } 9862#endif 9863 9864 return spelltab.st_isw[wp->w_s->b_spell_ismw[*p] ? p[1] : p[0]]; 9865} 9866 9867/* 9868 * Return TRUE if "p" points to a word character. 9869 * Unlike spell_iswordp() this doesn't check for "midword" characters. 9870 */ 9871 static int 9872spell_iswordp_nmw(p) 9873 char_u *p; 9874{ 9875#ifdef FEAT_MBYTE 9876 int c; 9877 9878 if (has_mbyte) 9879 { 9880 c = mb_ptr2char(p); 9881 if (c > 255) 9882 return spell_mb_isword_class(mb_get_class(p)); 9883 return spelltab.st_isw[c]; 9884 } 9885#endif 9886 return spelltab.st_isw[*p]; 9887} 9888 9889#ifdef FEAT_MBYTE 9890/* 9891 * Return TRUE if word class indicates a word character. 9892 * Only for characters above 255. 9893 * Unicode subscript and superscript are not considered word characters. 9894 */ 9895 static int 9896spell_mb_isword_class(cl) 9897 int cl; 9898{ 9899 return cl >= 2 && cl != 0x2070 && cl != 0x2080; 9900} 9901 9902/* 9903 * Return TRUE if "p" points to a word character. 9904 * Wide version of spell_iswordp(). 9905 */ 9906 static int 9907spell_iswordp_w(p, wp) 9908 int *p; 9909 win_T *wp; 9910{ 9911 int *s; 9912 9913 if (*p < 256 ? wp->w_s->b_spell_ismw[*p] 9914 : (wp->w_s->b_spell_ismw_mb != NULL 9915 && vim_strchr(wp->w_s->b_spell_ismw_mb, *p) != NULL)) 9916 s = p + 1; 9917 else 9918 s = p; 9919 9920 if (*s > 255) 9921 { 9922 if (enc_utf8) 9923 return spell_mb_isword_class(utf_class(*s)); 9924 if (enc_dbcs) 9925 return dbcs_class((unsigned)*s >> 8, *s & 0xff) >= 2; 9926 return 0; 9927 } 9928 return spelltab.st_isw[*s]; 9929} 9930#endif 9931 9932/* 9933 * Write the table with prefix conditions to the .spl file. 9934 * When "fd" is NULL only count the length of what is written. 9935 */ 9936 static int 9937write_spell_prefcond(fd, gap) 9938 FILE *fd; 9939 garray_T *gap; 9940{ 9941 int i; 9942 char_u *p; 9943 int len; 9944 int totlen; 9945 size_t x = 1; /* collect return value of fwrite() */ 9946 9947 if (fd != NULL) 9948 put_bytes(fd, (long_u)gap->ga_len, 2); /* <prefcondcnt> */ 9949 9950 totlen = 2 + gap->ga_len; /* length of <prefcondcnt> and <condlen> bytes */ 9951 9952 for (i = 0; i < gap->ga_len; ++i) 9953 { 9954 /* <prefcond> : <condlen> <condstr> */ 9955 p = ((char_u **)gap->ga_data)[i]; 9956 if (p != NULL) 9957 { 9958 len = (int)STRLEN(p); 9959 if (fd != NULL) 9960 { 9961 fputc(len, fd); 9962 x &= fwrite(p, (size_t)len, (size_t)1, fd); 9963 } 9964 totlen += len; 9965 } 9966 else if (fd != NULL) 9967 fputc(0, fd); 9968 } 9969 9970 return totlen; 9971} 9972 9973/* 9974 * Case-fold "str[len]" into "buf[buflen]". The result is NUL terminated. 9975 * Uses the character definitions from the .spl file. 9976 * When using a multi-byte 'encoding' the length may change! 9977 * Returns FAIL when something wrong. 9978 */ 9979 static int 9980spell_casefold(str, len, buf, buflen) 9981 char_u *str; 9982 int len; 9983 char_u *buf; 9984 int buflen; 9985{ 9986 int i; 9987 9988 if (len >= buflen) 9989 { 9990 buf[0] = NUL; 9991 return FAIL; /* result will not fit */ 9992 } 9993 9994#ifdef FEAT_MBYTE 9995 if (has_mbyte) 9996 { 9997 int outi = 0; 9998 char_u *p; 9999 int c; 10000 10001 /* Fold one character at a time. */ 10002 for (p = str; p < str + len; ) 10003 { 10004 if (outi + MB_MAXBYTES > buflen) 10005 { 10006 buf[outi] = NUL; 10007 return FAIL; 10008 } 10009 c = mb_cptr2char_adv(&p); 10010 outi += mb_char2bytes(SPELL_TOFOLD(c), buf + outi); 10011 } 10012 buf[outi] = NUL; 10013 } 10014 else 10015#endif 10016 { 10017 /* Be quick for non-multibyte encodings. */ 10018 for (i = 0; i < len; ++i) 10019 buf[i] = spelltab.st_fold[str[i]]; 10020 buf[i] = NUL; 10021 } 10022 10023 return OK; 10024} 10025 10026/* values for sps_flags */ 10027#define SPS_BEST 1 10028#define SPS_FAST 2 10029#define SPS_DOUBLE 4 10030 10031static int sps_flags = SPS_BEST; /* flags from 'spellsuggest' */ 10032static int sps_limit = 9999; /* max nr of suggestions given */ 10033 10034/* 10035 * Check the 'spellsuggest' option. Return FAIL if it's wrong. 10036 * Sets "sps_flags" and "sps_limit". 10037 */ 10038 int 10039spell_check_sps() 10040{ 10041 char_u *p; 10042 char_u *s; 10043 char_u buf[MAXPATHL]; 10044 int f; 10045 10046 sps_flags = 0; 10047 sps_limit = 9999; 10048 10049 for (p = p_sps; *p != NUL; ) 10050 { 10051 copy_option_part(&p, buf, MAXPATHL, ","); 10052 10053 f = 0; 10054 if (VIM_ISDIGIT(*buf)) 10055 { 10056 s = buf; 10057 sps_limit = getdigits(&s); 10058 if (*s != NUL && !VIM_ISDIGIT(*s)) 10059 f = -1; 10060 } 10061 else if (STRCMP(buf, "best") == 0) 10062 f = SPS_BEST; 10063 else if (STRCMP(buf, "fast") == 0) 10064 f = SPS_FAST; 10065 else if (STRCMP(buf, "double") == 0) 10066 f = SPS_DOUBLE; 10067 else if (STRNCMP(buf, "expr:", 5) != 0 10068 && STRNCMP(buf, "file:", 5) != 0) 10069 f = -1; 10070 10071 if (f == -1 || (sps_flags != 0 && f != 0)) 10072 { 10073 sps_flags = SPS_BEST; 10074 sps_limit = 9999; 10075 return FAIL; 10076 } 10077 if (f != 0) 10078 sps_flags = f; 10079 } 10080 10081 if (sps_flags == 0) 10082 sps_flags = SPS_BEST; 10083 10084 return OK; 10085} 10086 10087/* 10088 * "z?": Find badly spelled word under or after the cursor. 10089 * Give suggestions for the properly spelled word. 10090 * In Visual mode use the highlighted word as the bad word. 10091 * When "count" is non-zero use that suggestion. 10092 */ 10093 void 10094spell_suggest(count) 10095 int count; 10096{ 10097 char_u *line; 10098 pos_T prev_cursor = curwin->w_cursor; 10099 char_u wcopy[MAXWLEN + 2]; 10100 char_u *p; 10101 int i; 10102 int c; 10103 suginfo_T sug; 10104 suggest_T *stp; 10105 int mouse_used; 10106 int need_cap; 10107 int limit; 10108 int selected = count; 10109 int badlen = 0; 10110 int msg_scroll_save = msg_scroll; 10111 10112 if (no_spell_checking(curwin)) 10113 return; 10114 10115#ifdef FEAT_VISUAL 10116 if (VIsual_active) 10117 { 10118 /* Use the Visually selected text as the bad word. But reject 10119 * a multi-line selection. */ 10120 if (curwin->w_cursor.lnum != VIsual.lnum) 10121 { 10122 vim_beep(); 10123 return; 10124 } 10125 badlen = (int)curwin->w_cursor.col - (int)VIsual.col; 10126 if (badlen < 0) 10127 badlen = -badlen; 10128 else 10129 curwin->w_cursor.col = VIsual.col; 10130 ++badlen; 10131 end_visual_mode(); 10132 } 10133 else 10134#endif 10135 /* Find the start of the badly spelled word. */ 10136 if (spell_move_to(curwin, FORWARD, TRUE, TRUE, NULL) == 0 10137 || curwin->w_cursor.col > prev_cursor.col) 10138 { 10139 /* No bad word or it starts after the cursor: use the word under the 10140 * cursor. */ 10141 curwin->w_cursor = prev_cursor; 10142 line = ml_get_curline(); 10143 p = line + curwin->w_cursor.col; 10144 /* Backup to before start of word. */ 10145 while (p > line && spell_iswordp_nmw(p)) 10146 mb_ptr_back(line, p); 10147 /* Forward to start of word. */ 10148 while (*p != NUL && !spell_iswordp_nmw(p)) 10149 mb_ptr_adv(p); 10150 10151 if (!spell_iswordp_nmw(p)) /* No word found. */ 10152 { 10153 beep_flush(); 10154 return; 10155 } 10156 curwin->w_cursor.col = (colnr_T)(p - line); 10157 } 10158 10159 /* Get the word and its length. */ 10160 10161 /* Figure out if the word should be capitalised. */ 10162 need_cap = check_need_cap(curwin->w_cursor.lnum, curwin->w_cursor.col); 10163 10164 /* Make a copy of current line since autocommands may free the line. */ 10165 line = vim_strsave(ml_get_curline()); 10166 if (line == NULL) 10167 goto skip; 10168 10169 /* Get the list of suggestions. Limit to 'lines' - 2 or the number in 10170 * 'spellsuggest', whatever is smaller. */ 10171 if (sps_limit > (int)Rows - 2) 10172 limit = (int)Rows - 2; 10173 else 10174 limit = sps_limit; 10175 spell_find_suggest(line + curwin->w_cursor.col, badlen, &sug, limit, 10176 TRUE, need_cap, TRUE); 10177 10178 if (sug.su_ga.ga_len == 0) 10179 MSG(_("Sorry, no suggestions")); 10180 else if (count > 0) 10181 { 10182 if (count > sug.su_ga.ga_len) 10183 smsg((char_u *)_("Sorry, only %ld suggestions"), 10184 (long)sug.su_ga.ga_len); 10185 } 10186 else 10187 { 10188 vim_free(repl_from); 10189 repl_from = NULL; 10190 vim_free(repl_to); 10191 repl_to = NULL; 10192 10193#ifdef FEAT_RIGHTLEFT 10194 /* When 'rightleft' is set the list is drawn right-left. */ 10195 cmdmsg_rl = curwin->w_p_rl; 10196 if (cmdmsg_rl) 10197 msg_col = Columns - 1; 10198#endif 10199 10200 /* List the suggestions. */ 10201 msg_start(); 10202 msg_row = Rows - 1; /* for when 'cmdheight' > 1 */ 10203 lines_left = Rows; /* avoid more prompt */ 10204 vim_snprintf((char *)IObuff, IOSIZE, _("Change \"%.*s\" to:"), 10205 sug.su_badlen, sug.su_badptr); 10206#ifdef FEAT_RIGHTLEFT 10207 if (cmdmsg_rl && STRNCMP(IObuff, "Change", 6) == 0) 10208 { 10209 /* And now the rabbit from the high hat: Avoid showing the 10210 * untranslated message rightleft. */ 10211 vim_snprintf((char *)IObuff, IOSIZE, ":ot \"%.*s\" egnahC", 10212 sug.su_badlen, sug.su_badptr); 10213 } 10214#endif 10215 msg_puts(IObuff); 10216 msg_clr_eos(); 10217 msg_putchar('\n'); 10218 10219 msg_scroll = TRUE; 10220 for (i = 0; i < sug.su_ga.ga_len; ++i) 10221 { 10222 stp = &SUG(sug.su_ga, i); 10223 10224 /* The suggested word may replace only part of the bad word, add 10225 * the not replaced part. */ 10226 STRCPY(wcopy, stp->st_word); 10227 if (sug.su_badlen > stp->st_orglen) 10228 vim_strncpy(wcopy + stp->st_wordlen, 10229 sug.su_badptr + stp->st_orglen, 10230 sug.su_badlen - stp->st_orglen); 10231 vim_snprintf((char *)IObuff, IOSIZE, "%2d", i + 1); 10232#ifdef FEAT_RIGHTLEFT 10233 if (cmdmsg_rl) 10234 rl_mirror(IObuff); 10235#endif 10236 msg_puts(IObuff); 10237 10238 vim_snprintf((char *)IObuff, IOSIZE, " \"%s\"", wcopy); 10239 msg_puts(IObuff); 10240 10241 /* The word may replace more than "su_badlen". */ 10242 if (sug.su_badlen < stp->st_orglen) 10243 { 10244 vim_snprintf((char *)IObuff, IOSIZE, _(" < \"%.*s\""), 10245 stp->st_orglen, sug.su_badptr); 10246 msg_puts(IObuff); 10247 } 10248 10249 if (p_verbose > 0) 10250 { 10251 /* Add the score. */ 10252 if (sps_flags & (SPS_DOUBLE | SPS_BEST)) 10253 vim_snprintf((char *)IObuff, IOSIZE, " (%s%d - %d)", 10254 stp->st_salscore ? "s " : "", 10255 stp->st_score, stp->st_altscore); 10256 else 10257 vim_snprintf((char *)IObuff, IOSIZE, " (%d)", 10258 stp->st_score); 10259#ifdef FEAT_RIGHTLEFT 10260 if (cmdmsg_rl) 10261 /* Mirror the numbers, but keep the leading space. */ 10262 rl_mirror(IObuff + 1); 10263#endif 10264 msg_advance(30); 10265 msg_puts(IObuff); 10266 } 10267 msg_putchar('\n'); 10268 } 10269 10270#ifdef FEAT_RIGHTLEFT 10271 cmdmsg_rl = FALSE; 10272 msg_col = 0; 10273#endif 10274 /* Ask for choice. */ 10275 selected = prompt_for_number(&mouse_used); 10276 if (mouse_used) 10277 selected -= lines_left; 10278 lines_left = Rows; /* avoid more prompt */ 10279 /* don't delay for 'smd' in normal_cmd() */ 10280 msg_scroll = msg_scroll_save; 10281 } 10282 10283 if (selected > 0 && selected <= sug.su_ga.ga_len && u_save_cursor() == OK) 10284 { 10285 /* Save the from and to text for :spellrepall. */ 10286 stp = &SUG(sug.su_ga, selected - 1); 10287 if (sug.su_badlen > stp->st_orglen) 10288 { 10289 /* Replacing less than "su_badlen", append the remainder to 10290 * repl_to. */ 10291 repl_from = vim_strnsave(sug.su_badptr, sug.su_badlen); 10292 vim_snprintf((char *)IObuff, IOSIZE, "%s%.*s", stp->st_word, 10293 sug.su_badlen - stp->st_orglen, 10294 sug.su_badptr + stp->st_orglen); 10295 repl_to = vim_strsave(IObuff); 10296 } 10297 else 10298 { 10299 /* Replacing su_badlen or more, use the whole word. */ 10300 repl_from = vim_strnsave(sug.su_badptr, stp->st_orglen); 10301 repl_to = vim_strsave(stp->st_word); 10302 } 10303 10304 /* Replace the word. */ 10305 p = alloc((unsigned)STRLEN(line) - stp->st_orglen 10306 + stp->st_wordlen + 1); 10307 if (p != NULL) 10308 { 10309 c = (int)(sug.su_badptr - line); 10310 mch_memmove(p, line, c); 10311 STRCPY(p + c, stp->st_word); 10312 STRCAT(p, sug.su_badptr + stp->st_orglen); 10313 ml_replace(curwin->w_cursor.lnum, p, FALSE); 10314 curwin->w_cursor.col = c; 10315 10316 /* For redo we use a change-word command. */ 10317 ResetRedobuff(); 10318 AppendToRedobuff((char_u *)"ciw"); 10319 AppendToRedobuffLit(p + c, 10320 stp->st_wordlen + sug.su_badlen - stp->st_orglen); 10321 AppendCharToRedobuff(ESC); 10322 10323 /* After this "p" may be invalid. */ 10324 changed_bytes(curwin->w_cursor.lnum, c); 10325 } 10326 } 10327 else 10328 curwin->w_cursor = prev_cursor; 10329 10330 spell_find_cleanup(&sug); 10331skip: 10332 vim_free(line); 10333} 10334 10335/* 10336 * Check if the word at line "lnum" column "col" is required to start with a 10337 * capital. This uses 'spellcapcheck' of the current buffer. 10338 */ 10339 static int 10340check_need_cap(lnum, col) 10341 linenr_T lnum; 10342 colnr_T col; 10343{ 10344 int need_cap = FALSE; 10345 char_u *line; 10346 char_u *line_copy = NULL; 10347 char_u *p; 10348 colnr_T endcol; 10349 regmatch_T regmatch; 10350 10351 if (curwin->w_s->b_cap_prog == NULL) 10352 return FALSE; 10353 10354 line = ml_get_curline(); 10355 endcol = 0; 10356 if ((int)(skipwhite(line) - line) >= (int)col) 10357 { 10358 /* At start of line, check if previous line is empty or sentence 10359 * ends there. */ 10360 if (lnum == 1) 10361 need_cap = TRUE; 10362 else 10363 { 10364 line = ml_get(lnum - 1); 10365 if (*skipwhite(line) == NUL) 10366 need_cap = TRUE; 10367 else 10368 { 10369 /* Append a space in place of the line break. */ 10370 line_copy = concat_str(line, (char_u *)" "); 10371 line = line_copy; 10372 endcol = (colnr_T)STRLEN(line); 10373 } 10374 } 10375 } 10376 else 10377 endcol = col; 10378 10379 if (endcol > 0) 10380 { 10381 /* Check if sentence ends before the bad word. */ 10382 regmatch.regprog = curwin->w_s->b_cap_prog; 10383 regmatch.rm_ic = FALSE; 10384 p = line + endcol; 10385 for (;;) 10386 { 10387 mb_ptr_back(line, p); 10388 if (p == line || spell_iswordp_nmw(p)) 10389 break; 10390 if (vim_regexec(®match, p, 0) 10391 && regmatch.endp[0] == line + endcol) 10392 { 10393 need_cap = TRUE; 10394 break; 10395 } 10396 } 10397 } 10398 10399 vim_free(line_copy); 10400 10401 return need_cap; 10402} 10403 10404 10405/* 10406 * ":spellrepall" 10407 */ 10408 void 10409ex_spellrepall(eap) 10410 exarg_T *eap UNUSED; 10411{ 10412 pos_T pos = curwin->w_cursor; 10413 char_u *frompat; 10414 int addlen; 10415 char_u *line; 10416 char_u *p; 10417 int save_ws = p_ws; 10418 linenr_T prev_lnum = 0; 10419 10420 if (repl_from == NULL || repl_to == NULL) 10421 { 10422 EMSG(_("E752: No previous spell replacement")); 10423 return; 10424 } 10425 addlen = (int)(STRLEN(repl_to) - STRLEN(repl_from)); 10426 10427 frompat = alloc((unsigned)STRLEN(repl_from) + 7); 10428 if (frompat == NULL) 10429 return; 10430 sprintf((char *)frompat, "\\V\\<%s\\>", repl_from); 10431 p_ws = FALSE; 10432 10433 sub_nsubs = 0; 10434 sub_nlines = 0; 10435 curwin->w_cursor.lnum = 0; 10436 while (!got_int) 10437 { 10438 if (do_search(NULL, '/', frompat, 1L, SEARCH_KEEP, NULL) == 0 10439 || u_save_cursor() == FAIL) 10440 break; 10441 10442 /* Only replace when the right word isn't there yet. This happens 10443 * when changing "etc" to "etc.". */ 10444 line = ml_get_curline(); 10445 if (addlen <= 0 || STRNCMP(line + curwin->w_cursor.col, 10446 repl_to, STRLEN(repl_to)) != 0) 10447 { 10448 p = alloc((unsigned)STRLEN(line) + addlen + 1); 10449 if (p == NULL) 10450 break; 10451 mch_memmove(p, line, curwin->w_cursor.col); 10452 STRCPY(p + curwin->w_cursor.col, repl_to); 10453 STRCAT(p, line + curwin->w_cursor.col + STRLEN(repl_from)); 10454 ml_replace(curwin->w_cursor.lnum, p, FALSE); 10455 changed_bytes(curwin->w_cursor.lnum, curwin->w_cursor.col); 10456 10457 if (curwin->w_cursor.lnum != prev_lnum) 10458 { 10459 ++sub_nlines; 10460 prev_lnum = curwin->w_cursor.lnum; 10461 } 10462 ++sub_nsubs; 10463 } 10464 curwin->w_cursor.col += (colnr_T)STRLEN(repl_to); 10465 } 10466 10467 p_ws = save_ws; 10468 curwin->w_cursor = pos; 10469 vim_free(frompat); 10470 10471 if (sub_nsubs == 0) 10472 EMSG2(_("E753: Not found: %s"), repl_from); 10473 else 10474 do_sub_msg(FALSE); 10475} 10476 10477/* 10478 * Find spell suggestions for "word". Return them in the growarray "*gap" as 10479 * a list of allocated strings. 10480 */ 10481 void 10482spell_suggest_list(gap, word, maxcount, need_cap, interactive) 10483 garray_T *gap; 10484 char_u *word; 10485 int maxcount; /* maximum nr of suggestions */ 10486 int need_cap; /* 'spellcapcheck' matched */ 10487 int interactive; 10488{ 10489 suginfo_T sug; 10490 int i; 10491 suggest_T *stp; 10492 char_u *wcopy; 10493 10494 spell_find_suggest(word, 0, &sug, maxcount, FALSE, need_cap, interactive); 10495 10496 /* Make room in "gap". */ 10497 ga_init2(gap, sizeof(char_u *), sug.su_ga.ga_len + 1); 10498 if (ga_grow(gap, sug.su_ga.ga_len) == OK) 10499 { 10500 for (i = 0; i < sug.su_ga.ga_len; ++i) 10501 { 10502 stp = &SUG(sug.su_ga, i); 10503 10504 /* The suggested word may replace only part of "word", add the not 10505 * replaced part. */ 10506 wcopy = alloc(stp->st_wordlen 10507 + (unsigned)STRLEN(sug.su_badptr + stp->st_orglen) + 1); 10508 if (wcopy == NULL) 10509 break; 10510 STRCPY(wcopy, stp->st_word); 10511 STRCPY(wcopy + stp->st_wordlen, sug.su_badptr + stp->st_orglen); 10512 ((char_u **)gap->ga_data)[gap->ga_len++] = wcopy; 10513 } 10514 } 10515 10516 spell_find_cleanup(&sug); 10517} 10518 10519/* 10520 * Find spell suggestions for the word at the start of "badptr". 10521 * Return the suggestions in "su->su_ga". 10522 * The maximum number of suggestions is "maxcount". 10523 * Note: does use info for the current window. 10524 * This is based on the mechanisms of Aspell, but completely reimplemented. 10525 */ 10526 static void 10527spell_find_suggest(badptr, badlen, su, maxcount, banbadword, need_cap, interactive) 10528 char_u *badptr; 10529 int badlen; /* length of bad word or 0 if unknown */ 10530 suginfo_T *su; 10531 int maxcount; 10532 int banbadword; /* don't include badword in suggestions */ 10533 int need_cap; /* word should start with capital */ 10534 int interactive; 10535{ 10536 hlf_T attr = HLF_COUNT; 10537 char_u buf[MAXPATHL]; 10538 char_u *p; 10539 int do_combine = FALSE; 10540 char_u *sps_copy; 10541#ifdef FEAT_EVAL 10542 static int expr_busy = FALSE; 10543#endif 10544 int c; 10545 int i; 10546 langp_T *lp; 10547 10548 /* 10549 * Set the info in "*su". 10550 */ 10551 vim_memset(su, 0, sizeof(suginfo_T)); 10552 ga_init2(&su->su_ga, (int)sizeof(suggest_T), 10); 10553 ga_init2(&su->su_sga, (int)sizeof(suggest_T), 10); 10554 if (*badptr == NUL) 10555 return; 10556 hash_init(&su->su_banned); 10557 10558 su->su_badptr = badptr; 10559 if (badlen != 0) 10560 su->su_badlen = badlen; 10561 else 10562 su->su_badlen = spell_check(curwin, su->su_badptr, &attr, NULL, FALSE); 10563 su->su_maxcount = maxcount; 10564 su->su_maxscore = SCORE_MAXINIT; 10565 10566 if (su->su_badlen >= MAXWLEN) 10567 su->su_badlen = MAXWLEN - 1; /* just in case */ 10568 vim_strncpy(su->su_badword, su->su_badptr, su->su_badlen); 10569 (void)spell_casefold(su->su_badptr, su->su_badlen, 10570 su->su_fbadword, MAXWLEN); 10571 /* get caps flags for bad word */ 10572 su->su_badflags = badword_captype(su->su_badptr, 10573 su->su_badptr + su->su_badlen); 10574 if (need_cap) 10575 su->su_badflags |= WF_ONECAP; 10576 10577 /* Find the default language for sound folding. We simply use the first 10578 * one in 'spelllang' that supports sound folding. That's good for when 10579 * using multiple files for one language, it's not that bad when mixing 10580 * languages (e.g., "pl,en"). */ 10581 for (i = 0; i < curbuf->b_s.b_langp.ga_len; ++i) 10582 { 10583 lp = LANGP_ENTRY(curbuf->b_s.b_langp, i); 10584 if (lp->lp_sallang != NULL) 10585 { 10586 su->su_sallang = lp->lp_sallang; 10587 break; 10588 } 10589 } 10590 10591 /* Soundfold the bad word with the default sound folding, so that we don't 10592 * have to do this many times. */ 10593 if (su->su_sallang != NULL) 10594 spell_soundfold(su->su_sallang, su->su_fbadword, TRUE, 10595 su->su_sal_badword); 10596 10597 /* If the word is not capitalised and spell_check() doesn't consider the 10598 * word to be bad then it might need to be capitalised. Add a suggestion 10599 * for that. */ 10600 c = PTR2CHAR(su->su_badptr); 10601 if (!SPELL_ISUPPER(c) && attr == HLF_COUNT) 10602 { 10603 make_case_word(su->su_badword, buf, WF_ONECAP); 10604 add_suggestion(su, &su->su_ga, buf, su->su_badlen, SCORE_ICASE, 10605 0, TRUE, su->su_sallang, FALSE); 10606 } 10607 10608 /* Ban the bad word itself. It may appear in another region. */ 10609 if (banbadword) 10610 add_banned(su, su->su_badword); 10611 10612 /* Make a copy of 'spellsuggest', because the expression may change it. */ 10613 sps_copy = vim_strsave(p_sps); 10614 if (sps_copy == NULL) 10615 return; 10616 10617 /* Loop over the items in 'spellsuggest'. */ 10618 for (p = sps_copy; *p != NUL; ) 10619 { 10620 copy_option_part(&p, buf, MAXPATHL, ","); 10621 10622 if (STRNCMP(buf, "expr:", 5) == 0) 10623 { 10624#ifdef FEAT_EVAL 10625 /* Evaluate an expression. Skip this when called recursively, 10626 * when using spellsuggest() in the expression. */ 10627 if (!expr_busy) 10628 { 10629 expr_busy = TRUE; 10630 spell_suggest_expr(su, buf + 5); 10631 expr_busy = FALSE; 10632 } 10633#endif 10634 } 10635 else if (STRNCMP(buf, "file:", 5) == 0) 10636 /* Use list of suggestions in a file. */ 10637 spell_suggest_file(su, buf + 5); 10638 else 10639 { 10640 /* Use internal method. */ 10641 spell_suggest_intern(su, interactive); 10642 if (sps_flags & SPS_DOUBLE) 10643 do_combine = TRUE; 10644 } 10645 } 10646 10647 vim_free(sps_copy); 10648 10649 if (do_combine) 10650 /* Combine the two list of suggestions. This must be done last, 10651 * because sorting changes the order again. */ 10652 score_combine(su); 10653} 10654 10655#ifdef FEAT_EVAL 10656/* 10657 * Find suggestions by evaluating expression "expr". 10658 */ 10659 static void 10660spell_suggest_expr(su, expr) 10661 suginfo_T *su; 10662 char_u *expr; 10663{ 10664 list_T *list; 10665 listitem_T *li; 10666 int score; 10667 char_u *p; 10668 10669 /* The work is split up in a few parts to avoid having to export 10670 * suginfo_T. 10671 * First evaluate the expression and get the resulting list. */ 10672 list = eval_spell_expr(su->su_badword, expr); 10673 if (list != NULL) 10674 { 10675 /* Loop over the items in the list. */ 10676 for (li = list->lv_first; li != NULL; li = li->li_next) 10677 if (li->li_tv.v_type == VAR_LIST) 10678 { 10679 /* Get the word and the score from the items. */ 10680 score = get_spellword(li->li_tv.vval.v_list, &p); 10681 if (score >= 0 && score <= su->su_maxscore) 10682 add_suggestion(su, &su->su_ga, p, su->su_badlen, 10683 score, 0, TRUE, su->su_sallang, FALSE); 10684 } 10685 list_unref(list); 10686 } 10687 10688 /* Remove bogus suggestions, sort and truncate at "maxcount". */ 10689 check_suggestions(su, &su->su_ga); 10690 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount); 10691} 10692#endif 10693 10694/* 10695 * Find suggestions in file "fname". Used for "file:" in 'spellsuggest'. 10696 */ 10697 static void 10698spell_suggest_file(su, fname) 10699 suginfo_T *su; 10700 char_u *fname; 10701{ 10702 FILE *fd; 10703 char_u line[MAXWLEN * 2]; 10704 char_u *p; 10705 int len; 10706 char_u cword[MAXWLEN]; 10707 10708 /* Open the file. */ 10709 fd = mch_fopen((char *)fname, "r"); 10710 if (fd == NULL) 10711 { 10712 EMSG2(_(e_notopen), fname); 10713 return; 10714 } 10715 10716 /* Read it line by line. */ 10717 while (!vim_fgets(line, MAXWLEN * 2, fd) && !got_int) 10718 { 10719 line_breakcheck(); 10720 10721 p = vim_strchr(line, '/'); 10722 if (p == NULL) 10723 continue; /* No Tab found, just skip the line. */ 10724 *p++ = NUL; 10725 if (STRICMP(su->su_badword, line) == 0) 10726 { 10727 /* Match! Isolate the good word, until CR or NL. */ 10728 for (len = 0; p[len] >= ' '; ++len) 10729 ; 10730 p[len] = NUL; 10731 10732 /* If the suggestion doesn't have specific case duplicate the case 10733 * of the bad word. */ 10734 if (captype(p, NULL) == 0) 10735 { 10736 make_case_word(p, cword, su->su_badflags); 10737 p = cword; 10738 } 10739 10740 add_suggestion(su, &su->su_ga, p, su->su_badlen, 10741 SCORE_FILE, 0, TRUE, su->su_sallang, FALSE); 10742 } 10743 } 10744 10745 fclose(fd); 10746 10747 /* Remove bogus suggestions, sort and truncate at "maxcount". */ 10748 check_suggestions(su, &su->su_ga); 10749 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount); 10750} 10751 10752/* 10753 * Find suggestions for the internal method indicated by "sps_flags". 10754 */ 10755 static void 10756spell_suggest_intern(su, interactive) 10757 suginfo_T *su; 10758 int interactive; 10759{ 10760 /* 10761 * Load the .sug file(s) that are available and not done yet. 10762 */ 10763 suggest_load_files(); 10764 10765 /* 10766 * 1. Try special cases, such as repeating a word: "the the" -> "the". 10767 * 10768 * Set a maximum score to limit the combination of operations that is 10769 * tried. 10770 */ 10771 suggest_try_special(su); 10772 10773 /* 10774 * 2. Try inserting/deleting/swapping/changing a letter, use REP entries 10775 * from the .aff file and inserting a space (split the word). 10776 */ 10777 suggest_try_change(su); 10778 10779 /* For the resulting top-scorers compute the sound-a-like score. */ 10780 if (sps_flags & SPS_DOUBLE) 10781 score_comp_sal(su); 10782 10783 /* 10784 * 3. Try finding sound-a-like words. 10785 */ 10786 if ((sps_flags & SPS_FAST) == 0) 10787 { 10788 if (sps_flags & SPS_BEST) 10789 /* Adjust the word score for the suggestions found so far for how 10790 * they sounds like. */ 10791 rescore_suggestions(su); 10792 10793 /* 10794 * While going through the soundfold tree "su_maxscore" is the score 10795 * for the soundfold word, limits the changes that are being tried, 10796 * and "su_sfmaxscore" the rescored score, which is set by 10797 * cleanup_suggestions(). 10798 * First find words with a small edit distance, because this is much 10799 * faster and often already finds the top-N suggestions. If we didn't 10800 * find many suggestions try again with a higher edit distance. 10801 * "sl_sounddone" is used to avoid doing the same word twice. 10802 */ 10803 suggest_try_soundalike_prep(); 10804 su->su_maxscore = SCORE_SFMAX1; 10805 su->su_sfmaxscore = SCORE_MAXINIT * 3; 10806 suggest_try_soundalike(su); 10807 if (su->su_ga.ga_len < SUG_CLEAN_COUNT(su)) 10808 { 10809 /* We didn't find enough matches, try again, allowing more 10810 * changes to the soundfold word. */ 10811 su->su_maxscore = SCORE_SFMAX2; 10812 suggest_try_soundalike(su); 10813 if (su->su_ga.ga_len < SUG_CLEAN_COUNT(su)) 10814 { 10815 /* Still didn't find enough matches, try again, allowing even 10816 * more changes to the soundfold word. */ 10817 su->su_maxscore = SCORE_SFMAX3; 10818 suggest_try_soundalike(su); 10819 } 10820 } 10821 su->su_maxscore = su->su_sfmaxscore; 10822 suggest_try_soundalike_finish(); 10823 } 10824 10825 /* When CTRL-C was hit while searching do show the results. Only clear 10826 * got_int when using a command, not for spellsuggest(). */ 10827 ui_breakcheck(); 10828 if (interactive && got_int) 10829 { 10830 (void)vgetc(); 10831 got_int = FALSE; 10832 } 10833 10834 if ((sps_flags & SPS_DOUBLE) == 0 && su->su_ga.ga_len != 0) 10835 { 10836 if (sps_flags & SPS_BEST) 10837 /* Adjust the word score for how it sounds like. */ 10838 rescore_suggestions(su); 10839 10840 /* Remove bogus suggestions, sort and truncate at "maxcount". */ 10841 check_suggestions(su, &su->su_ga); 10842 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount); 10843 } 10844} 10845 10846/* 10847 * Load the .sug files for languages that have one and weren't loaded yet. 10848 */ 10849 static void 10850suggest_load_files() 10851{ 10852 langp_T *lp; 10853 int lpi; 10854 slang_T *slang; 10855 char_u *dotp; 10856 FILE *fd; 10857 char_u buf[MAXWLEN]; 10858 int i; 10859 time_t timestamp; 10860 int wcount; 10861 int wordnr; 10862 garray_T ga; 10863 int c; 10864 10865 /* Do this for all languages that support sound folding. */ 10866 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi) 10867 { 10868 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi); 10869 slang = lp->lp_slang; 10870 if (slang->sl_sugtime != 0 && !slang->sl_sugloaded) 10871 { 10872 /* Change ".spl" to ".sug" and open the file. When the file isn't 10873 * found silently skip it. Do set "sl_sugloaded" so that we 10874 * don't try again and again. */ 10875 slang->sl_sugloaded = TRUE; 10876 10877 dotp = vim_strrchr(slang->sl_fname, '.'); 10878 if (dotp == NULL || fnamecmp(dotp, ".spl") != 0) 10879 continue; 10880 STRCPY(dotp, ".sug"); 10881 fd = mch_fopen((char *)slang->sl_fname, "r"); 10882 if (fd == NULL) 10883 goto nextone; 10884 10885 /* 10886 * <SUGHEADER>: <fileID> <versionnr> <timestamp> 10887 */ 10888 for (i = 0; i < VIMSUGMAGICL; ++i) 10889 buf[i] = getc(fd); /* <fileID> */ 10890 if (STRNCMP(buf, VIMSUGMAGIC, VIMSUGMAGICL) != 0) 10891 { 10892 EMSG2(_("E778: This does not look like a .sug file: %s"), 10893 slang->sl_fname); 10894 goto nextone; 10895 } 10896 c = getc(fd); /* <versionnr> */ 10897 if (c < VIMSUGVERSION) 10898 { 10899 EMSG2(_("E779: Old .sug file, needs to be updated: %s"), 10900 slang->sl_fname); 10901 goto nextone; 10902 } 10903 else if (c > VIMSUGVERSION) 10904 { 10905 EMSG2(_("E780: .sug file is for newer version of Vim: %s"), 10906 slang->sl_fname); 10907 goto nextone; 10908 } 10909 10910 /* Check the timestamp, it must be exactly the same as the one in 10911 * the .spl file. Otherwise the word numbers won't match. */ 10912 timestamp = get8ctime(fd); /* <timestamp> */ 10913 if (timestamp != slang->sl_sugtime) 10914 { 10915 EMSG2(_("E781: .sug file doesn't match .spl file: %s"), 10916 slang->sl_fname); 10917 goto nextone; 10918 } 10919 10920 /* 10921 * <SUGWORDTREE>: <wordtree> 10922 * Read the trie with the soundfolded words. 10923 */ 10924 if (spell_read_tree(fd, &slang->sl_sbyts, &slang->sl_sidxs, 10925 FALSE, 0) != 0) 10926 { 10927someerror: 10928 EMSG2(_("E782: error while reading .sug file: %s"), 10929 slang->sl_fname); 10930 slang_clear_sug(slang); 10931 goto nextone; 10932 } 10933 10934 /* 10935 * <SUGTABLE>: <sugwcount> <sugline> ... 10936 * 10937 * Read the table with word numbers. We use a file buffer for 10938 * this, because it's so much like a file with lines. Makes it 10939 * possible to swap the info and save on memory use. 10940 */ 10941 slang->sl_sugbuf = open_spellbuf(); 10942 if (slang->sl_sugbuf == NULL) 10943 goto someerror; 10944 /* <sugwcount> */ 10945 wcount = get4c(fd); 10946 if (wcount < 0) 10947 goto someerror; 10948 10949 /* Read all the wordnr lists into the buffer, one NUL terminated 10950 * list per line. */ 10951 ga_init2(&ga, 1, 100); 10952 for (wordnr = 0; wordnr < wcount; ++wordnr) 10953 { 10954 ga.ga_len = 0; 10955 for (;;) 10956 { 10957 c = getc(fd); /* <sugline> */ 10958 if (c < 0 || ga_grow(&ga, 1) == FAIL) 10959 goto someerror; 10960 ((char_u *)ga.ga_data)[ga.ga_len++] = c; 10961 if (c == NUL) 10962 break; 10963 } 10964 if (ml_append_buf(slang->sl_sugbuf, (linenr_T)wordnr, 10965 ga.ga_data, ga.ga_len, TRUE) == FAIL) 10966 goto someerror; 10967 } 10968 ga_clear(&ga); 10969 10970 /* 10971 * Need to put word counts in the word tries, so that we can find 10972 * a word by its number. 10973 */ 10974 tree_count_words(slang->sl_fbyts, slang->sl_fidxs); 10975 tree_count_words(slang->sl_sbyts, slang->sl_sidxs); 10976 10977nextone: 10978 if (fd != NULL) 10979 fclose(fd); 10980 STRCPY(dotp, ".spl"); 10981 } 10982 } 10983} 10984 10985 10986/* 10987 * Fill in the wordcount fields for a trie. 10988 * Returns the total number of words. 10989 */ 10990 static void 10991tree_count_words(byts, idxs) 10992 char_u *byts; 10993 idx_T *idxs; 10994{ 10995 int depth; 10996 idx_T arridx[MAXWLEN]; 10997 int curi[MAXWLEN]; 10998 int c; 10999 idx_T n; 11000 int wordcount[MAXWLEN]; 11001 11002 arridx[0] = 0; 11003 curi[0] = 1; 11004 wordcount[0] = 0; 11005 depth = 0; 11006 while (depth >= 0 && !got_int) 11007 { 11008 if (curi[depth] > byts[arridx[depth]]) 11009 { 11010 /* Done all bytes at this node, go up one level. */ 11011 idxs[arridx[depth]] = wordcount[depth]; 11012 if (depth > 0) 11013 wordcount[depth - 1] += wordcount[depth]; 11014 11015 --depth; 11016 fast_breakcheck(); 11017 } 11018 else 11019 { 11020 /* Do one more byte at this node. */ 11021 n = arridx[depth] + curi[depth]; 11022 ++curi[depth]; 11023 11024 c = byts[n]; 11025 if (c == 0) 11026 { 11027 /* End of word, count it. */ 11028 ++wordcount[depth]; 11029 11030 /* Skip over any other NUL bytes (same word with different 11031 * flags). */ 11032 while (byts[n + 1] == 0) 11033 { 11034 ++n; 11035 ++curi[depth]; 11036 } 11037 } 11038 else 11039 { 11040 /* Normal char, go one level deeper to count the words. */ 11041 ++depth; 11042 arridx[depth] = idxs[n]; 11043 curi[depth] = 1; 11044 wordcount[depth] = 0; 11045 } 11046 } 11047 } 11048} 11049 11050/* 11051 * Free the info put in "*su" by spell_find_suggest(). 11052 */ 11053 static void 11054spell_find_cleanup(su) 11055 suginfo_T *su; 11056{ 11057 int i; 11058 11059 /* Free the suggestions. */ 11060 for (i = 0; i < su->su_ga.ga_len; ++i) 11061 vim_free(SUG(su->su_ga, i).st_word); 11062 ga_clear(&su->su_ga); 11063 for (i = 0; i < su->su_sga.ga_len; ++i) 11064 vim_free(SUG(su->su_sga, i).st_word); 11065 ga_clear(&su->su_sga); 11066 11067 /* Free the banned words. */ 11068 hash_clear_all(&su->su_banned, 0); 11069} 11070 11071/* 11072 * Make a copy of "word", with the first letter upper or lower cased, to 11073 * "wcopy[MAXWLEN]". "word" must not be empty. 11074 * The result is NUL terminated. 11075 */ 11076 static void 11077onecap_copy(word, wcopy, upper) 11078 char_u *word; 11079 char_u *wcopy; 11080 int upper; /* TRUE: first letter made upper case */ 11081{ 11082 char_u *p; 11083 int c; 11084 int l; 11085 11086 p = word; 11087#ifdef FEAT_MBYTE 11088 if (has_mbyte) 11089 c = mb_cptr2char_adv(&p); 11090 else 11091#endif 11092 c = *p++; 11093 if (upper) 11094 c = SPELL_TOUPPER(c); 11095 else 11096 c = SPELL_TOFOLD(c); 11097#ifdef FEAT_MBYTE 11098 if (has_mbyte) 11099 l = mb_char2bytes(c, wcopy); 11100 else 11101#endif 11102 { 11103 l = 1; 11104 wcopy[0] = c; 11105 } 11106 vim_strncpy(wcopy + l, p, MAXWLEN - l - 1); 11107} 11108 11109/* 11110 * Make a copy of "word" with all the letters upper cased into 11111 * "wcopy[MAXWLEN]". The result is NUL terminated. 11112 */ 11113 static void 11114allcap_copy(word, wcopy) 11115 char_u *word; 11116 char_u *wcopy; 11117{ 11118 char_u *s; 11119 char_u *d; 11120 int c; 11121 11122 d = wcopy; 11123 for (s = word; *s != NUL; ) 11124 { 11125#ifdef FEAT_MBYTE 11126 if (has_mbyte) 11127 c = mb_cptr2char_adv(&s); 11128 else 11129#endif 11130 c = *s++; 11131 11132#ifdef FEAT_MBYTE 11133 /* We only change � to SS when we are certain latin1 is used. It 11134 * would cause weird errors in other 8-bit encodings. */ 11135 if (enc_latin1like && c == 0xdf) 11136 { 11137 c = 'S'; 11138 if (d - wcopy >= MAXWLEN - 1) 11139 break; 11140 *d++ = c; 11141 } 11142 else 11143#endif 11144 c = SPELL_TOUPPER(c); 11145 11146#ifdef FEAT_MBYTE 11147 if (has_mbyte) 11148 { 11149 if (d - wcopy >= MAXWLEN - MB_MAXBYTES) 11150 break; 11151 d += mb_char2bytes(c, d); 11152 } 11153 else 11154#endif 11155 { 11156 if (d - wcopy >= MAXWLEN - 1) 11157 break; 11158 *d++ = c; 11159 } 11160 } 11161 *d = NUL; 11162} 11163 11164/* 11165 * Try finding suggestions by recognizing specific situations. 11166 */ 11167 static void 11168suggest_try_special(su) 11169 suginfo_T *su; 11170{ 11171 char_u *p; 11172 size_t len; 11173 int c; 11174 char_u word[MAXWLEN]; 11175 11176 /* 11177 * Recognize a word that is repeated: "the the". 11178 */ 11179 p = skiptowhite(su->su_fbadword); 11180 len = p - su->su_fbadword; 11181 p = skipwhite(p); 11182 if (STRLEN(p) == len && STRNCMP(su->su_fbadword, p, len) == 0) 11183 { 11184 /* Include badflags: if the badword is onecap or allcap 11185 * use that for the goodword too: "The the" -> "The". */ 11186 c = su->su_fbadword[len]; 11187 su->su_fbadword[len] = NUL; 11188 make_case_word(su->su_fbadword, word, su->su_badflags); 11189 su->su_fbadword[len] = c; 11190 11191 /* Give a soundalike score of 0, compute the score as if deleting one 11192 * character. */ 11193 add_suggestion(su, &su->su_ga, word, su->su_badlen, 11194 RESCORE(SCORE_REP, 0), 0, TRUE, su->su_sallang, FALSE); 11195 } 11196} 11197 11198/* 11199 * Try finding suggestions by adding/removing/swapping letters. 11200 */ 11201 static void 11202suggest_try_change(su) 11203 suginfo_T *su; 11204{ 11205 char_u fword[MAXWLEN]; /* copy of the bad word, case-folded */ 11206 int n; 11207 char_u *p; 11208 int lpi; 11209 langp_T *lp; 11210 11211 /* We make a copy of the case-folded bad word, so that we can modify it 11212 * to find matches (esp. REP items). Append some more text, changing 11213 * chars after the bad word may help. */ 11214 STRCPY(fword, su->su_fbadword); 11215 n = (int)STRLEN(fword); 11216 p = su->su_badptr + su->su_badlen; 11217 (void)spell_casefold(p, (int)STRLEN(p), fword + n, MAXWLEN - n); 11218 11219 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi) 11220 { 11221 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi); 11222 11223 /* If reloading a spell file fails it's still in the list but 11224 * everything has been cleared. */ 11225 if (lp->lp_slang->sl_fbyts == NULL) 11226 continue; 11227 11228 /* Try it for this language. Will add possible suggestions. */ 11229 suggest_trie_walk(su, lp, fword, FALSE); 11230 } 11231} 11232 11233/* Check the maximum score, if we go over it we won't try this change. */ 11234#define TRY_DEEPER(su, stack, depth, add) \ 11235 (stack[depth].ts_score + (add) < su->su_maxscore) 11236 11237/* 11238 * Try finding suggestions by adding/removing/swapping letters. 11239 * 11240 * This uses a state machine. At each node in the tree we try various 11241 * operations. When trying if an operation works "depth" is increased and the 11242 * stack[] is used to store info. This allows combinations, thus insert one 11243 * character, replace one and delete another. The number of changes is 11244 * limited by su->su_maxscore. 11245 * 11246 * After implementing this I noticed an article by Kemal Oflazer that 11247 * describes something similar: "Error-tolerant Finite State Recognition with 11248 * Applications to Morphological Analysis and Spelling Correction" (1996). 11249 * The implementation in the article is simplified and requires a stack of 11250 * unknown depth. The implementation here only needs a stack depth equal to 11251 * the length of the word. 11252 * 11253 * This is also used for the sound-folded word, "soundfold" is TRUE then. 11254 * The mechanism is the same, but we find a match with a sound-folded word 11255 * that comes from one or more original words. Each of these words may be 11256 * added, this is done by add_sound_suggest(). 11257 * Don't use: 11258 * the prefix tree or the keep-case tree 11259 * "su->su_badlen" 11260 * anything to do with upper and lower case 11261 * anything to do with word or non-word characters ("spell_iswordp()") 11262 * banned words 11263 * word flags (rare, region, compounding) 11264 * word splitting for now 11265 * "similar_chars()" 11266 * use "slang->sl_repsal" instead of "lp->lp_replang->sl_rep" 11267 */ 11268 static void 11269suggest_trie_walk(su, lp, fword, soundfold) 11270 suginfo_T *su; 11271 langp_T *lp; 11272 char_u *fword; 11273 int soundfold; 11274{ 11275 char_u tword[MAXWLEN]; /* good word collected so far */ 11276 trystate_T stack[MAXWLEN]; 11277 char_u preword[MAXWLEN * 3]; /* word found with proper case; 11278 * concatenation of prefix compound 11279 * words and split word. NUL terminated 11280 * when going deeper but not when coming 11281 * back. */ 11282 char_u compflags[MAXWLEN]; /* compound flags, one for each word */ 11283 trystate_T *sp; 11284 int newscore; 11285 int score; 11286 char_u *byts, *fbyts, *pbyts; 11287 idx_T *idxs, *fidxs, *pidxs; 11288 int depth; 11289 int c, c2, c3; 11290 int n = 0; 11291 int flags; 11292 garray_T *gap; 11293 idx_T arridx; 11294 int len; 11295 char_u *p; 11296 fromto_T *ftp; 11297 int fl = 0, tl; 11298 int repextra = 0; /* extra bytes in fword[] from REP item */ 11299 slang_T *slang = lp->lp_slang; 11300 int fword_ends; 11301 int goodword_ends; 11302#ifdef DEBUG_TRIEWALK 11303 /* Stores the name of the change made at each level. */ 11304 char_u changename[MAXWLEN][80]; 11305#endif 11306 int breakcheckcount = 1000; 11307 int compound_ok; 11308 11309 /* 11310 * Go through the whole case-fold tree, try changes at each node. 11311 * "tword[]" contains the word collected from nodes in the tree. 11312 * "fword[]" the word we are trying to match with (initially the bad 11313 * word). 11314 */ 11315 depth = 0; 11316 sp = &stack[0]; 11317 vim_memset(sp, 0, sizeof(trystate_T)); 11318 sp->ts_curi = 1; 11319 11320 if (soundfold) 11321 { 11322 /* Going through the soundfold tree. */ 11323 byts = fbyts = slang->sl_sbyts; 11324 idxs = fidxs = slang->sl_sidxs; 11325 pbyts = NULL; 11326 pidxs = NULL; 11327 sp->ts_prefixdepth = PFD_NOPREFIX; 11328 sp->ts_state = STATE_START; 11329 } 11330 else 11331 { 11332 /* 11333 * When there are postponed prefixes we need to use these first. At 11334 * the end of the prefix we continue in the case-fold tree. 11335 */ 11336 fbyts = slang->sl_fbyts; 11337 fidxs = slang->sl_fidxs; 11338 pbyts = slang->sl_pbyts; 11339 pidxs = slang->sl_pidxs; 11340 if (pbyts != NULL) 11341 { 11342 byts = pbyts; 11343 idxs = pidxs; 11344 sp->ts_prefixdepth = PFD_PREFIXTREE; 11345 sp->ts_state = STATE_NOPREFIX; /* try without prefix first */ 11346 } 11347 else 11348 { 11349 byts = fbyts; 11350 idxs = fidxs; 11351 sp->ts_prefixdepth = PFD_NOPREFIX; 11352 sp->ts_state = STATE_START; 11353 } 11354 } 11355 11356 /* 11357 * Loop to find all suggestions. At each round we either: 11358 * - For the current state try one operation, advance "ts_curi", 11359 * increase "depth". 11360 * - When a state is done go to the next, set "ts_state". 11361 * - When all states are tried decrease "depth". 11362 */ 11363 while (depth >= 0 && !got_int) 11364 { 11365 sp = &stack[depth]; 11366 switch (sp->ts_state) 11367 { 11368 case STATE_START: 11369 case STATE_NOPREFIX: 11370 /* 11371 * Start of node: Deal with NUL bytes, which means 11372 * tword[] may end here. 11373 */ 11374 arridx = sp->ts_arridx; /* current node in the tree */ 11375 len = byts[arridx]; /* bytes in this node */ 11376 arridx += sp->ts_curi; /* index of current byte */ 11377 11378 if (sp->ts_prefixdepth == PFD_PREFIXTREE) 11379 { 11380 /* Skip over the NUL bytes, we use them later. */ 11381 for (n = 0; n < len && byts[arridx + n] == 0; ++n) 11382 ; 11383 sp->ts_curi += n; 11384 11385 /* Always past NUL bytes now. */ 11386 n = (int)sp->ts_state; 11387 sp->ts_state = STATE_ENDNUL; 11388 sp->ts_save_badflags = su->su_badflags; 11389 11390 /* At end of a prefix or at start of prefixtree: check for 11391 * following word. */ 11392 if (byts[arridx] == 0 || n == (int)STATE_NOPREFIX) 11393 { 11394 /* Set su->su_badflags to the caps type at this position. 11395 * Use the caps type until here for the prefix itself. */ 11396#ifdef FEAT_MBYTE 11397 if (has_mbyte) 11398 n = nofold_len(fword, sp->ts_fidx, su->su_badptr); 11399 else 11400#endif 11401 n = sp->ts_fidx; 11402 flags = badword_captype(su->su_badptr, su->su_badptr + n); 11403 su->su_badflags = badword_captype(su->su_badptr + n, 11404 su->su_badptr + su->su_badlen); 11405#ifdef DEBUG_TRIEWALK 11406 sprintf(changename[depth], "prefix"); 11407#endif 11408 go_deeper(stack, depth, 0); 11409 ++depth; 11410 sp = &stack[depth]; 11411 sp->ts_prefixdepth = depth - 1; 11412 byts = fbyts; 11413 idxs = fidxs; 11414 sp->ts_arridx = 0; 11415 11416 /* Move the prefix to preword[] with the right case 11417 * and make find_keepcap_word() works. */ 11418 tword[sp->ts_twordlen] = NUL; 11419 make_case_word(tword + sp->ts_splitoff, 11420 preword + sp->ts_prewordlen, flags); 11421 sp->ts_prewordlen = (char_u)STRLEN(preword); 11422 sp->ts_splitoff = sp->ts_twordlen; 11423 } 11424 break; 11425 } 11426 11427 if (sp->ts_curi > len || byts[arridx] != 0) 11428 { 11429 /* Past bytes in node and/or past NUL bytes. */ 11430 sp->ts_state = STATE_ENDNUL; 11431 sp->ts_save_badflags = su->su_badflags; 11432 break; 11433 } 11434 11435 /* 11436 * End of word in tree. 11437 */ 11438 ++sp->ts_curi; /* eat one NUL byte */ 11439 11440 flags = (int)idxs[arridx]; 11441 11442 /* Skip words with the NOSUGGEST flag. */ 11443 if (flags & WF_NOSUGGEST) 11444 break; 11445 11446 fword_ends = (fword[sp->ts_fidx] == NUL 11447 || (soundfold 11448 ? vim_iswhite(fword[sp->ts_fidx]) 11449 : !spell_iswordp(fword + sp->ts_fidx, curwin))); 11450 tword[sp->ts_twordlen] = NUL; 11451 11452 if (sp->ts_prefixdepth <= PFD_NOTSPECIAL 11453 && (sp->ts_flags & TSF_PREFIXOK) == 0) 11454 { 11455 /* There was a prefix before the word. Check that the prefix 11456 * can be used with this word. */ 11457 /* Count the length of the NULs in the prefix. If there are 11458 * none this must be the first try without a prefix. */ 11459 n = stack[sp->ts_prefixdepth].ts_arridx; 11460 len = pbyts[n++]; 11461 for (c = 0; c < len && pbyts[n + c] == 0; ++c) 11462 ; 11463 if (c > 0) 11464 { 11465 c = valid_word_prefix(c, n, flags, 11466 tword + sp->ts_splitoff, slang, FALSE); 11467 if (c == 0) 11468 break; 11469 11470 /* Use the WF_RARE flag for a rare prefix. */ 11471 if (c & WF_RAREPFX) 11472 flags |= WF_RARE; 11473 11474 /* Tricky: when checking for both prefix and compounding 11475 * we run into the prefix flag first. 11476 * Remember that it's OK, so that we accept the prefix 11477 * when arriving at a compound flag. */ 11478 sp->ts_flags |= TSF_PREFIXOK; 11479 } 11480 } 11481 11482 /* Check NEEDCOMPOUND: can't use word without compounding. Do try 11483 * appending another compound word below. */ 11484 if (sp->ts_complen == sp->ts_compsplit && fword_ends 11485 && (flags & WF_NEEDCOMP)) 11486 goodword_ends = FALSE; 11487 else 11488 goodword_ends = TRUE; 11489 11490 p = NULL; 11491 compound_ok = TRUE; 11492 if (sp->ts_complen > sp->ts_compsplit) 11493 { 11494 if (slang->sl_nobreak) 11495 { 11496 /* There was a word before this word. When there was no 11497 * change in this word (it was correct) add the first word 11498 * as a suggestion. If this word was corrected too, we 11499 * need to check if a correct word follows. */ 11500 if (sp->ts_fidx - sp->ts_splitfidx 11501 == sp->ts_twordlen - sp->ts_splitoff 11502 && STRNCMP(fword + sp->ts_splitfidx, 11503 tword + sp->ts_splitoff, 11504 sp->ts_fidx - sp->ts_splitfidx) == 0) 11505 { 11506 preword[sp->ts_prewordlen] = NUL; 11507 newscore = score_wordcount_adj(slang, sp->ts_score, 11508 preword + sp->ts_prewordlen, 11509 sp->ts_prewordlen > 0); 11510 /* Add the suggestion if the score isn't too bad. */ 11511 if (newscore <= su->su_maxscore) 11512 add_suggestion(su, &su->su_ga, preword, 11513 sp->ts_splitfidx - repextra, 11514 newscore, 0, FALSE, 11515 lp->lp_sallang, FALSE); 11516 break; 11517 } 11518 } 11519 else 11520 { 11521 /* There was a compound word before this word. If this 11522 * word does not support compounding then give up 11523 * (splitting is tried for the word without compound 11524 * flag). */ 11525 if (((unsigned)flags >> 24) == 0 11526 || sp->ts_twordlen - sp->ts_splitoff 11527 < slang->sl_compminlen) 11528 break; 11529#ifdef FEAT_MBYTE 11530 /* For multi-byte chars check character length against 11531 * COMPOUNDMIN. */ 11532 if (has_mbyte 11533 && slang->sl_compminlen > 0 11534 && mb_charlen(tword + sp->ts_splitoff) 11535 < slang->sl_compminlen) 11536 break; 11537#endif 11538 11539 compflags[sp->ts_complen] = ((unsigned)flags >> 24); 11540 compflags[sp->ts_complen + 1] = NUL; 11541 vim_strncpy(preword + sp->ts_prewordlen, 11542 tword + sp->ts_splitoff, 11543 sp->ts_twordlen - sp->ts_splitoff); 11544 11545 /* Verify CHECKCOMPOUNDPATTERN rules. */ 11546 if (match_checkcompoundpattern(preword, sp->ts_prewordlen, 11547 &slang->sl_comppat)) 11548 compound_ok = FALSE; 11549 11550 if (compound_ok) 11551 { 11552 p = preword; 11553 while (*skiptowhite(p) != NUL) 11554 p = skipwhite(skiptowhite(p)); 11555 if (fword_ends && !can_compound(slang, p, 11556 compflags + sp->ts_compsplit)) 11557 /* Compound is not allowed. But it may still be 11558 * possible if we add another (short) word. */ 11559 compound_ok = FALSE; 11560 } 11561 11562 /* Get pointer to last char of previous word. */ 11563 p = preword + sp->ts_prewordlen; 11564 mb_ptr_back(preword, p); 11565 } 11566 } 11567 11568 /* 11569 * Form the word with proper case in preword. 11570 * If there is a word from a previous split, append. 11571 * For the soundfold tree don't change the case, simply append. 11572 */ 11573 if (soundfold) 11574 STRCPY(preword + sp->ts_prewordlen, tword + sp->ts_splitoff); 11575 else if (flags & WF_KEEPCAP) 11576 /* Must find the word in the keep-case tree. */ 11577 find_keepcap_word(slang, tword + sp->ts_splitoff, 11578 preword + sp->ts_prewordlen); 11579 else 11580 { 11581 /* Include badflags: If the badword is onecap or allcap 11582 * use that for the goodword too. But if the badword is 11583 * allcap and it's only one char long use onecap. */ 11584 c = su->su_badflags; 11585 if ((c & WF_ALLCAP) 11586#ifdef FEAT_MBYTE 11587 && su->su_badlen == (*mb_ptr2len)(su->su_badptr) 11588#else 11589 && su->su_badlen == 1 11590#endif 11591 ) 11592 c = WF_ONECAP; 11593 c |= flags; 11594 11595 /* When appending a compound word after a word character don't 11596 * use Onecap. */ 11597 if (p != NULL && spell_iswordp_nmw(p)) 11598 c &= ~WF_ONECAP; 11599 make_case_word(tword + sp->ts_splitoff, 11600 preword + sp->ts_prewordlen, c); 11601 } 11602 11603 if (!soundfold) 11604 { 11605 /* Don't use a banned word. It may appear again as a good 11606 * word, thus remember it. */ 11607 if (flags & WF_BANNED) 11608 { 11609 add_banned(su, preword + sp->ts_prewordlen); 11610 break; 11611 } 11612 if ((sp->ts_complen == sp->ts_compsplit 11613 && WAS_BANNED(su, preword + sp->ts_prewordlen)) 11614 || WAS_BANNED(su, preword)) 11615 { 11616 if (slang->sl_compprog == NULL) 11617 break; 11618 /* the word so far was banned but we may try compounding */ 11619 goodword_ends = FALSE; 11620 } 11621 } 11622 11623 newscore = 0; 11624 if (!soundfold) /* soundfold words don't have flags */ 11625 { 11626 if ((flags & WF_REGION) 11627 && (((unsigned)flags >> 16) & lp->lp_region) == 0) 11628 newscore += SCORE_REGION; 11629 if (flags & WF_RARE) 11630 newscore += SCORE_RARE; 11631 11632 if (!spell_valid_case(su->su_badflags, 11633 captype(preword + sp->ts_prewordlen, NULL))) 11634 newscore += SCORE_ICASE; 11635 } 11636 11637 /* TODO: how about splitting in the soundfold tree? */ 11638 if (fword_ends 11639 && goodword_ends 11640 && sp->ts_fidx >= sp->ts_fidxtry 11641 && compound_ok) 11642 { 11643 /* The badword also ends: add suggestions. */ 11644#ifdef DEBUG_TRIEWALK 11645 if (soundfold && STRCMP(preword, "smwrd") == 0) 11646 { 11647 int j; 11648 11649 /* print the stack of changes that brought us here */ 11650 smsg("------ %s -------", fword); 11651 for (j = 0; j < depth; ++j) 11652 smsg("%s", changename[j]); 11653 } 11654#endif 11655 if (soundfold) 11656 { 11657 /* For soundfolded words we need to find the original 11658 * words, the edit distance and then add them. */ 11659 add_sound_suggest(su, preword, sp->ts_score, lp); 11660 } 11661 else if (sp->ts_fidx > 0) 11662 { 11663 /* Give a penalty when changing non-word char to word 11664 * char, e.g., "thes," -> "these". */ 11665 p = fword + sp->ts_fidx; 11666 mb_ptr_back(fword, p); 11667 if (!spell_iswordp(p, curwin)) 11668 { 11669 p = preword + STRLEN(preword); 11670 mb_ptr_back(preword, p); 11671 if (spell_iswordp(p, curwin)) 11672 newscore += SCORE_NONWORD; 11673 } 11674 11675 /* Give a bonus to words seen before. */ 11676 score = score_wordcount_adj(slang, 11677 sp->ts_score + newscore, 11678 preword + sp->ts_prewordlen, 11679 sp->ts_prewordlen > 0); 11680 11681 /* Add the suggestion if the score isn't too bad. */ 11682 if (score <= su->su_maxscore) 11683 { 11684 add_suggestion(su, &su->su_ga, preword, 11685 sp->ts_fidx - repextra, 11686 score, 0, FALSE, lp->lp_sallang, FALSE); 11687 11688 if (su->su_badflags & WF_MIXCAP) 11689 { 11690 /* We really don't know if the word should be 11691 * upper or lower case, add both. */ 11692 c = captype(preword, NULL); 11693 if (c == 0 || c == WF_ALLCAP) 11694 { 11695 make_case_word(tword + sp->ts_splitoff, 11696 preword + sp->ts_prewordlen, 11697 c == 0 ? WF_ALLCAP : 0); 11698 11699 add_suggestion(su, &su->su_ga, preword, 11700 sp->ts_fidx - repextra, 11701 score + SCORE_ICASE, 0, FALSE, 11702 lp->lp_sallang, FALSE); 11703 } 11704 } 11705 } 11706 } 11707 } 11708 11709 /* 11710 * Try word split and/or compounding. 11711 */ 11712 if ((sp->ts_fidx >= sp->ts_fidxtry || fword_ends) 11713#ifdef FEAT_MBYTE 11714 /* Don't split halfway a character. */ 11715 && (!has_mbyte || sp->ts_tcharlen == 0) 11716#endif 11717 ) 11718 { 11719 int try_compound; 11720 int try_split; 11721 11722 /* If past the end of the bad word don't try a split. 11723 * Otherwise try changing the next word. E.g., find 11724 * suggestions for "the the" where the second "the" is 11725 * different. It's done like a split. 11726 * TODO: word split for soundfold words */ 11727 try_split = (sp->ts_fidx - repextra < su->su_badlen) 11728 && !soundfold; 11729 11730 /* Get here in several situations: 11731 * 1. The word in the tree ends: 11732 * If the word allows compounding try that. Otherwise try 11733 * a split by inserting a space. For both check that a 11734 * valid words starts at fword[sp->ts_fidx]. 11735 * For NOBREAK do like compounding to be able to check if 11736 * the next word is valid. 11737 * 2. The badword does end, but it was due to a change (e.g., 11738 * a swap). No need to split, but do check that the 11739 * following word is valid. 11740 * 3. The badword and the word in the tree end. It may still 11741 * be possible to compound another (short) word. 11742 */ 11743 try_compound = FALSE; 11744 if (!soundfold 11745 && slang->sl_compprog != NULL 11746 && ((unsigned)flags >> 24) != 0 11747 && sp->ts_twordlen - sp->ts_splitoff 11748 >= slang->sl_compminlen 11749#ifdef FEAT_MBYTE 11750 && (!has_mbyte 11751 || slang->sl_compminlen == 0 11752 || mb_charlen(tword + sp->ts_splitoff) 11753 >= slang->sl_compminlen) 11754#endif 11755 && (slang->sl_compsylmax < MAXWLEN 11756 || sp->ts_complen + 1 - sp->ts_compsplit 11757 < slang->sl_compmax) 11758 && (can_be_compound(sp, slang, 11759 compflags, ((unsigned)flags >> 24)))) 11760 11761 { 11762 try_compound = TRUE; 11763 compflags[sp->ts_complen] = ((unsigned)flags >> 24); 11764 compflags[sp->ts_complen + 1] = NUL; 11765 } 11766 11767 /* For NOBREAK we never try splitting, it won't make any word 11768 * valid. */ 11769 if (slang->sl_nobreak) 11770 try_compound = TRUE; 11771 11772 /* If we could add a compound word, and it's also possible to 11773 * split at this point, do the split first and set 11774 * TSF_DIDSPLIT to avoid doing it again. */ 11775 else if (!fword_ends 11776 && try_compound 11777 && (sp->ts_flags & TSF_DIDSPLIT) == 0) 11778 { 11779 try_compound = FALSE; 11780 sp->ts_flags |= TSF_DIDSPLIT; 11781 --sp->ts_curi; /* do the same NUL again */ 11782 compflags[sp->ts_complen] = NUL; 11783 } 11784 else 11785 sp->ts_flags &= ~TSF_DIDSPLIT; 11786 11787 if (try_split || try_compound) 11788 { 11789 if (!try_compound && (!fword_ends || !goodword_ends)) 11790 { 11791 /* If we're going to split need to check that the 11792 * words so far are valid for compounding. If there 11793 * is only one word it must not have the NEEDCOMPOUND 11794 * flag. */ 11795 if (sp->ts_complen == sp->ts_compsplit 11796 && (flags & WF_NEEDCOMP)) 11797 break; 11798 p = preword; 11799 while (*skiptowhite(p) != NUL) 11800 p = skipwhite(skiptowhite(p)); 11801 if (sp->ts_complen > sp->ts_compsplit 11802 && !can_compound(slang, p, 11803 compflags + sp->ts_compsplit)) 11804 break; 11805 11806 if (slang->sl_nosplitsugs) 11807 newscore += SCORE_SPLIT_NO; 11808 else 11809 newscore += SCORE_SPLIT; 11810 11811 /* Give a bonus to words seen before. */ 11812 newscore = score_wordcount_adj(slang, newscore, 11813 preword + sp->ts_prewordlen, TRUE); 11814 } 11815 11816 if (TRY_DEEPER(su, stack, depth, newscore)) 11817 { 11818 go_deeper(stack, depth, newscore); 11819#ifdef DEBUG_TRIEWALK 11820 if (!try_compound && !fword_ends) 11821 sprintf(changename[depth], "%.*s-%s: split", 11822 sp->ts_twordlen, tword, fword + sp->ts_fidx); 11823 else 11824 sprintf(changename[depth], "%.*s-%s: compound", 11825 sp->ts_twordlen, tword, fword + sp->ts_fidx); 11826#endif 11827 /* Save things to be restored at STATE_SPLITUNDO. */ 11828 sp->ts_save_badflags = su->su_badflags; 11829 sp->ts_state = STATE_SPLITUNDO; 11830 11831 ++depth; 11832 sp = &stack[depth]; 11833 11834 /* Append a space to preword when splitting. */ 11835 if (!try_compound && !fword_ends) 11836 STRCAT(preword, " "); 11837 sp->ts_prewordlen = (char_u)STRLEN(preword); 11838 sp->ts_splitoff = sp->ts_twordlen; 11839 sp->ts_splitfidx = sp->ts_fidx; 11840 11841 /* If the badword has a non-word character at this 11842 * position skip it. That means replacing the 11843 * non-word character with a space. Always skip a 11844 * character when the word ends. But only when the 11845 * good word can end. */ 11846 if (((!try_compound && !spell_iswordp_nmw(fword 11847 + sp->ts_fidx)) 11848 || fword_ends) 11849 && fword[sp->ts_fidx] != NUL 11850 && goodword_ends) 11851 { 11852 int l; 11853 11854#ifdef FEAT_MBYTE 11855 if (has_mbyte) 11856 l = MB_BYTE2LEN(fword[sp->ts_fidx]); 11857 else 11858#endif 11859 l = 1; 11860 if (fword_ends) 11861 { 11862 /* Copy the skipped character to preword. */ 11863 mch_memmove(preword + sp->ts_prewordlen, 11864 fword + sp->ts_fidx, l); 11865 sp->ts_prewordlen += l; 11866 preword[sp->ts_prewordlen] = NUL; 11867 } 11868 else 11869 sp->ts_score -= SCORE_SPLIT - SCORE_SUBST; 11870 sp->ts_fidx += l; 11871 } 11872 11873 /* When compounding include compound flag in 11874 * compflags[] (already set above). When splitting we 11875 * may start compounding over again. */ 11876 if (try_compound) 11877 ++sp->ts_complen; 11878 else 11879 sp->ts_compsplit = sp->ts_complen; 11880 sp->ts_prefixdepth = PFD_NOPREFIX; 11881 11882 /* set su->su_badflags to the caps type at this 11883 * position */ 11884#ifdef FEAT_MBYTE 11885 if (has_mbyte) 11886 n = nofold_len(fword, sp->ts_fidx, su->su_badptr); 11887 else 11888#endif 11889 n = sp->ts_fidx; 11890 su->su_badflags = badword_captype(su->su_badptr + n, 11891 su->su_badptr + su->su_badlen); 11892 11893 /* Restart at top of the tree. */ 11894 sp->ts_arridx = 0; 11895 11896 /* If there are postponed prefixes, try these too. */ 11897 if (pbyts != NULL) 11898 { 11899 byts = pbyts; 11900 idxs = pidxs; 11901 sp->ts_prefixdepth = PFD_PREFIXTREE; 11902 sp->ts_state = STATE_NOPREFIX; 11903 } 11904 } 11905 } 11906 } 11907 break; 11908 11909 case STATE_SPLITUNDO: 11910 /* Undo the changes done for word split or compound word. */ 11911 su->su_badflags = sp->ts_save_badflags; 11912 11913 /* Continue looking for NUL bytes. */ 11914 sp->ts_state = STATE_START; 11915 11916 /* In case we went into the prefix tree. */ 11917 byts = fbyts; 11918 idxs = fidxs; 11919 break; 11920 11921 case STATE_ENDNUL: 11922 /* Past the NUL bytes in the node. */ 11923 su->su_badflags = sp->ts_save_badflags; 11924 if (fword[sp->ts_fidx] == NUL 11925#ifdef FEAT_MBYTE 11926 && sp->ts_tcharlen == 0 11927#endif 11928 ) 11929 { 11930 /* The badword ends, can't use STATE_PLAIN. */ 11931 sp->ts_state = STATE_DEL; 11932 break; 11933 } 11934 sp->ts_state = STATE_PLAIN; 11935 /*FALLTHROUGH*/ 11936 11937 case STATE_PLAIN: 11938 /* 11939 * Go over all possible bytes at this node, add each to tword[] 11940 * and use child node. "ts_curi" is the index. 11941 */ 11942 arridx = sp->ts_arridx; 11943 if (sp->ts_curi > byts[arridx]) 11944 { 11945 /* Done all bytes at this node, do next state. When still at 11946 * already changed bytes skip the other tricks. */ 11947 if (sp->ts_fidx >= sp->ts_fidxtry) 11948 sp->ts_state = STATE_DEL; 11949 else 11950 sp->ts_state = STATE_FINAL; 11951 } 11952 else 11953 { 11954 arridx += sp->ts_curi++; 11955 c = byts[arridx]; 11956 11957 /* Normal byte, go one level deeper. If it's not equal to the 11958 * byte in the bad word adjust the score. But don't even try 11959 * when the byte was already changed. And don't try when we 11960 * just deleted this byte, accepting it is always cheaper then 11961 * delete + substitute. */ 11962 if (c == fword[sp->ts_fidx] 11963#ifdef FEAT_MBYTE 11964 || (sp->ts_tcharlen > 0 && sp->ts_isdiff != DIFF_NONE) 11965#endif 11966 ) 11967 newscore = 0; 11968 else 11969 newscore = SCORE_SUBST; 11970 if ((newscore == 0 11971 || (sp->ts_fidx >= sp->ts_fidxtry 11972 && ((sp->ts_flags & TSF_DIDDEL) == 0 11973 || c != fword[sp->ts_delidx]))) 11974 && TRY_DEEPER(su, stack, depth, newscore)) 11975 { 11976 go_deeper(stack, depth, newscore); 11977#ifdef DEBUG_TRIEWALK 11978 if (newscore > 0) 11979 sprintf(changename[depth], "%.*s-%s: subst %c to %c", 11980 sp->ts_twordlen, tword, fword + sp->ts_fidx, 11981 fword[sp->ts_fidx], c); 11982 else 11983 sprintf(changename[depth], "%.*s-%s: accept %c", 11984 sp->ts_twordlen, tword, fword + sp->ts_fidx, 11985 fword[sp->ts_fidx]); 11986#endif 11987 ++depth; 11988 sp = &stack[depth]; 11989 ++sp->ts_fidx; 11990 tword[sp->ts_twordlen++] = c; 11991 sp->ts_arridx = idxs[arridx]; 11992#ifdef FEAT_MBYTE 11993 if (newscore == SCORE_SUBST) 11994 sp->ts_isdiff = DIFF_YES; 11995 if (has_mbyte) 11996 { 11997 /* Multi-byte characters are a bit complicated to 11998 * handle: They differ when any of the bytes differ 11999 * and then their length may also differ. */ 12000 if (sp->ts_tcharlen == 0) 12001 { 12002 /* First byte. */ 12003 sp->ts_tcharidx = 0; 12004 sp->ts_tcharlen = MB_BYTE2LEN(c); 12005 sp->ts_fcharstart = sp->ts_fidx - 1; 12006 sp->ts_isdiff = (newscore != 0) 12007 ? DIFF_YES : DIFF_NONE; 12008 } 12009 else if (sp->ts_isdiff == DIFF_INSERT) 12010 /* When inserting trail bytes don't advance in the 12011 * bad word. */ 12012 --sp->ts_fidx; 12013 if (++sp->ts_tcharidx == sp->ts_tcharlen) 12014 { 12015 /* Last byte of character. */ 12016 if (sp->ts_isdiff == DIFF_YES) 12017 { 12018 /* Correct ts_fidx for the byte length of the 12019 * character (we didn't check that before). */ 12020 sp->ts_fidx = sp->ts_fcharstart 12021 + MB_BYTE2LEN( 12022 fword[sp->ts_fcharstart]); 12023 12024 /* For changing a composing character adjust 12025 * the score from SCORE_SUBST to 12026 * SCORE_SUBCOMP. */ 12027 if (enc_utf8 12028 && utf_iscomposing( 12029 mb_ptr2char(tword 12030 + sp->ts_twordlen 12031 - sp->ts_tcharlen)) 12032 && utf_iscomposing( 12033 mb_ptr2char(fword 12034 + sp->ts_fcharstart))) 12035 sp->ts_score -= 12036 SCORE_SUBST - SCORE_SUBCOMP; 12037 12038 /* For a similar character adjust score from 12039 * SCORE_SUBST to SCORE_SIMILAR. */ 12040 else if (!soundfold 12041 && slang->sl_has_map 12042 && similar_chars(slang, 12043 mb_ptr2char(tword 12044 + sp->ts_twordlen 12045 - sp->ts_tcharlen), 12046 mb_ptr2char(fword 12047 + sp->ts_fcharstart))) 12048 sp->ts_score -= 12049 SCORE_SUBST - SCORE_SIMILAR; 12050 } 12051 else if (sp->ts_isdiff == DIFF_INSERT 12052 && sp->ts_twordlen > sp->ts_tcharlen) 12053 { 12054 p = tword + sp->ts_twordlen - sp->ts_tcharlen; 12055 c = mb_ptr2char(p); 12056 if (enc_utf8 && utf_iscomposing(c)) 12057 { 12058 /* Inserting a composing char doesn't 12059 * count that much. */ 12060 sp->ts_score -= SCORE_INS - SCORE_INSCOMP; 12061 } 12062 else 12063 { 12064 /* If the previous character was the same, 12065 * thus doubling a character, give a bonus 12066 * to the score. Also for the soundfold 12067 * tree (might seem illogical but does 12068 * give better scores). */ 12069 mb_ptr_back(tword, p); 12070 if (c == mb_ptr2char(p)) 12071 sp->ts_score -= SCORE_INS 12072 - SCORE_INSDUP; 12073 } 12074 } 12075 12076 /* Starting a new char, reset the length. */ 12077 sp->ts_tcharlen = 0; 12078 } 12079 } 12080 else 12081#endif 12082 { 12083 /* If we found a similar char adjust the score. 12084 * We do this after calling go_deeper() because 12085 * it's slow. */ 12086 if (newscore != 0 12087 && !soundfold 12088 && slang->sl_has_map 12089 && similar_chars(slang, 12090 c, fword[sp->ts_fidx - 1])) 12091 sp->ts_score -= SCORE_SUBST - SCORE_SIMILAR; 12092 } 12093 } 12094 } 12095 break; 12096 12097 case STATE_DEL: 12098#ifdef FEAT_MBYTE 12099 /* When past the first byte of a multi-byte char don't try 12100 * delete/insert/swap a character. */ 12101 if (has_mbyte && sp->ts_tcharlen > 0) 12102 { 12103 sp->ts_state = STATE_FINAL; 12104 break; 12105 } 12106#endif 12107 /* 12108 * Try skipping one character in the bad word (delete it). 12109 */ 12110 sp->ts_state = STATE_INS_PREP; 12111 sp->ts_curi = 1; 12112 if (soundfold && sp->ts_fidx == 0 && fword[sp->ts_fidx] == '*') 12113 /* Deleting a vowel at the start of a word counts less, see 12114 * soundalike_score(). */ 12115 newscore = 2 * SCORE_DEL / 3; 12116 else 12117 newscore = SCORE_DEL; 12118 if (fword[sp->ts_fidx] != NUL 12119 && TRY_DEEPER(su, stack, depth, newscore)) 12120 { 12121 go_deeper(stack, depth, newscore); 12122#ifdef DEBUG_TRIEWALK 12123 sprintf(changename[depth], "%.*s-%s: delete %c", 12124 sp->ts_twordlen, tword, fword + sp->ts_fidx, 12125 fword[sp->ts_fidx]); 12126#endif 12127 ++depth; 12128 12129 /* Remember what character we deleted, so that we can avoid 12130 * inserting it again. */ 12131 stack[depth].ts_flags |= TSF_DIDDEL; 12132 stack[depth].ts_delidx = sp->ts_fidx; 12133 12134 /* Advance over the character in fword[]. Give a bonus to the 12135 * score if the same character is following "nn" -> "n". It's 12136 * a bit illogical for soundfold tree but it does give better 12137 * results. */ 12138#ifdef FEAT_MBYTE 12139 if (has_mbyte) 12140 { 12141 c = mb_ptr2char(fword + sp->ts_fidx); 12142 stack[depth].ts_fidx += MB_BYTE2LEN(fword[sp->ts_fidx]); 12143 if (enc_utf8 && utf_iscomposing(c)) 12144 stack[depth].ts_score -= SCORE_DEL - SCORE_DELCOMP; 12145 else if (c == mb_ptr2char(fword + stack[depth].ts_fidx)) 12146 stack[depth].ts_score -= SCORE_DEL - SCORE_DELDUP; 12147 } 12148 else 12149#endif 12150 { 12151 ++stack[depth].ts_fidx; 12152 if (fword[sp->ts_fidx] == fword[sp->ts_fidx + 1]) 12153 stack[depth].ts_score -= SCORE_DEL - SCORE_DELDUP; 12154 } 12155 break; 12156 } 12157 /*FALLTHROUGH*/ 12158 12159 case STATE_INS_PREP: 12160 if (sp->ts_flags & TSF_DIDDEL) 12161 { 12162 /* If we just deleted a byte then inserting won't make sense, 12163 * a substitute is always cheaper. */ 12164 sp->ts_state = STATE_SWAP; 12165 break; 12166 } 12167 12168 /* skip over NUL bytes */ 12169 n = sp->ts_arridx; 12170 for (;;) 12171 { 12172 if (sp->ts_curi > byts[n]) 12173 { 12174 /* Only NUL bytes at this node, go to next state. */ 12175 sp->ts_state = STATE_SWAP; 12176 break; 12177 } 12178 if (byts[n + sp->ts_curi] != NUL) 12179 { 12180 /* Found a byte to insert. */ 12181 sp->ts_state = STATE_INS; 12182 break; 12183 } 12184 ++sp->ts_curi; 12185 } 12186 break; 12187 12188 /*FALLTHROUGH*/ 12189 12190 case STATE_INS: 12191 /* Insert one byte. Repeat this for each possible byte at this 12192 * node. */ 12193 n = sp->ts_arridx; 12194 if (sp->ts_curi > byts[n]) 12195 { 12196 /* Done all bytes at this node, go to next state. */ 12197 sp->ts_state = STATE_SWAP; 12198 break; 12199 } 12200 12201 /* Do one more byte at this node, but: 12202 * - Skip NUL bytes. 12203 * - Skip the byte if it's equal to the byte in the word, 12204 * accepting that byte is always better. 12205 */ 12206 n += sp->ts_curi++; 12207 c = byts[n]; 12208 if (soundfold && sp->ts_twordlen == 0 && c == '*') 12209 /* Inserting a vowel at the start of a word counts less, 12210 * see soundalike_score(). */ 12211 newscore = 2 * SCORE_INS / 3; 12212 else 12213 newscore = SCORE_INS; 12214 if (c != fword[sp->ts_fidx] 12215 && TRY_DEEPER(su, stack, depth, newscore)) 12216 { 12217 go_deeper(stack, depth, newscore); 12218#ifdef DEBUG_TRIEWALK 12219 sprintf(changename[depth], "%.*s-%s: insert %c", 12220 sp->ts_twordlen, tword, fword + sp->ts_fidx, 12221 c); 12222#endif 12223 ++depth; 12224 sp = &stack[depth]; 12225 tword[sp->ts_twordlen++] = c; 12226 sp->ts_arridx = idxs[n]; 12227#ifdef FEAT_MBYTE 12228 if (has_mbyte) 12229 { 12230 fl = MB_BYTE2LEN(c); 12231 if (fl > 1) 12232 { 12233 /* There are following bytes for the same character. 12234 * We must find all bytes before trying 12235 * delete/insert/swap/etc. */ 12236 sp->ts_tcharlen = fl; 12237 sp->ts_tcharidx = 1; 12238 sp->ts_isdiff = DIFF_INSERT; 12239 } 12240 } 12241 else 12242 fl = 1; 12243 if (fl == 1) 12244#endif 12245 { 12246 /* If the previous character was the same, thus doubling a 12247 * character, give a bonus to the score. Also for 12248 * soundfold words (illogical but does give a better 12249 * score). */ 12250 if (sp->ts_twordlen >= 2 12251 && tword[sp->ts_twordlen - 2] == c) 12252 sp->ts_score -= SCORE_INS - SCORE_INSDUP; 12253 } 12254 } 12255 break; 12256 12257 case STATE_SWAP: 12258 /* 12259 * Swap two bytes in the bad word: "12" -> "21". 12260 * We change "fword" here, it's changed back afterwards at 12261 * STATE_UNSWAP. 12262 */ 12263 p = fword + sp->ts_fidx; 12264 c = *p; 12265 if (c == NUL) 12266 { 12267 /* End of word, can't swap or replace. */ 12268 sp->ts_state = STATE_FINAL; 12269 break; 12270 } 12271 12272 /* Don't swap if the first character is not a word character. 12273 * SWAP3 etc. also don't make sense then. */ 12274 if (!soundfold && !spell_iswordp(p, curwin)) 12275 { 12276 sp->ts_state = STATE_REP_INI; 12277 break; 12278 } 12279 12280#ifdef FEAT_MBYTE 12281 if (has_mbyte) 12282 { 12283 n = mb_cptr2len(p); 12284 c = mb_ptr2char(p); 12285 if (p[n] == NUL) 12286 c2 = NUL; 12287 else if (!soundfold && !spell_iswordp(p + n, curwin)) 12288 c2 = c; /* don't swap non-word char */ 12289 else 12290 c2 = mb_ptr2char(p + n); 12291 } 12292 else 12293#endif 12294 { 12295 if (p[1] == NUL) 12296 c2 = NUL; 12297 else if (!soundfold && !spell_iswordp(p + 1, curwin)) 12298 c2 = c; /* don't swap non-word char */ 12299 else 12300 c2 = p[1]; 12301 } 12302 12303 /* When the second character is NUL we can't swap. */ 12304 if (c2 == NUL) 12305 { 12306 sp->ts_state = STATE_REP_INI; 12307 break; 12308 } 12309 12310 /* When characters are identical, swap won't do anything. 12311 * Also get here if the second char is not a word character. */ 12312 if (c == c2) 12313 { 12314 sp->ts_state = STATE_SWAP3; 12315 break; 12316 } 12317 if (c2 != NUL && TRY_DEEPER(su, stack, depth, SCORE_SWAP)) 12318 { 12319 go_deeper(stack, depth, SCORE_SWAP); 12320#ifdef DEBUG_TRIEWALK 12321 sprintf(changename[depth], "%.*s-%s: swap %c and %c", 12322 sp->ts_twordlen, tword, fword + sp->ts_fidx, 12323 c, c2); 12324#endif 12325 sp->ts_state = STATE_UNSWAP; 12326 ++depth; 12327#ifdef FEAT_MBYTE 12328 if (has_mbyte) 12329 { 12330 fl = mb_char2len(c2); 12331 mch_memmove(p, p + n, fl); 12332 mb_char2bytes(c, p + fl); 12333 stack[depth].ts_fidxtry = sp->ts_fidx + n + fl; 12334 } 12335 else 12336#endif 12337 { 12338 p[0] = c2; 12339 p[1] = c; 12340 stack[depth].ts_fidxtry = sp->ts_fidx + 2; 12341 } 12342 } 12343 else 12344 /* If this swap doesn't work then SWAP3 won't either. */ 12345 sp->ts_state = STATE_REP_INI; 12346 break; 12347 12348 case STATE_UNSWAP: 12349 /* Undo the STATE_SWAP swap: "21" -> "12". */ 12350 p = fword + sp->ts_fidx; 12351#ifdef FEAT_MBYTE 12352 if (has_mbyte) 12353 { 12354 n = MB_BYTE2LEN(*p); 12355 c = mb_ptr2char(p + n); 12356 mch_memmove(p + MB_BYTE2LEN(p[n]), p, n); 12357 mb_char2bytes(c, p); 12358 } 12359 else 12360#endif 12361 { 12362 c = *p; 12363 *p = p[1]; 12364 p[1] = c; 12365 } 12366 /*FALLTHROUGH*/ 12367 12368 case STATE_SWAP3: 12369 /* Swap two bytes, skipping one: "123" -> "321". We change 12370 * "fword" here, it's changed back afterwards at STATE_UNSWAP3. */ 12371 p = fword + sp->ts_fidx; 12372#ifdef FEAT_MBYTE 12373 if (has_mbyte) 12374 { 12375 n = mb_cptr2len(p); 12376 c = mb_ptr2char(p); 12377 fl = mb_cptr2len(p + n); 12378 c2 = mb_ptr2char(p + n); 12379 if (!soundfold && !spell_iswordp(p + n + fl, curwin)) 12380 c3 = c; /* don't swap non-word char */ 12381 else 12382 c3 = mb_ptr2char(p + n + fl); 12383 } 12384 else 12385#endif 12386 { 12387 c = *p; 12388 c2 = p[1]; 12389 if (!soundfold && !spell_iswordp(p + 2, curwin)) 12390 c3 = c; /* don't swap non-word char */ 12391 else 12392 c3 = p[2]; 12393 } 12394 12395 /* When characters are identical: "121" then SWAP3 result is 12396 * identical, ROT3L result is same as SWAP: "211", ROT3L result is 12397 * same as SWAP on next char: "112". Thus skip all swapping. 12398 * Also skip when c3 is NUL. 12399 * Also get here when the third character is not a word character. 12400 * Second character may any char: "a.b" -> "b.a" */ 12401 if (c == c3 || c3 == NUL) 12402 { 12403 sp->ts_state = STATE_REP_INI; 12404 break; 12405 } 12406 if (TRY_DEEPER(su, stack, depth, SCORE_SWAP3)) 12407 { 12408 go_deeper(stack, depth, SCORE_SWAP3); 12409#ifdef DEBUG_TRIEWALK 12410 sprintf(changename[depth], "%.*s-%s: swap3 %c and %c", 12411 sp->ts_twordlen, tword, fword + sp->ts_fidx, 12412 c, c3); 12413#endif 12414 sp->ts_state = STATE_UNSWAP3; 12415 ++depth; 12416#ifdef FEAT_MBYTE 12417 if (has_mbyte) 12418 { 12419 tl = mb_char2len(c3); 12420 mch_memmove(p, p + n + fl, tl); 12421 mb_char2bytes(c2, p + tl); 12422 mb_char2bytes(c, p + fl + tl); 12423 stack[depth].ts_fidxtry = sp->ts_fidx + n + fl + tl; 12424 } 12425 else 12426#endif 12427 { 12428 p[0] = p[2]; 12429 p[2] = c; 12430 stack[depth].ts_fidxtry = sp->ts_fidx + 3; 12431 } 12432 } 12433 else 12434 sp->ts_state = STATE_REP_INI; 12435 break; 12436 12437 case STATE_UNSWAP3: 12438 /* Undo STATE_SWAP3: "321" -> "123" */ 12439 p = fword + sp->ts_fidx; 12440#ifdef FEAT_MBYTE 12441 if (has_mbyte) 12442 { 12443 n = MB_BYTE2LEN(*p); 12444 c2 = mb_ptr2char(p + n); 12445 fl = MB_BYTE2LEN(p[n]); 12446 c = mb_ptr2char(p + n + fl); 12447 tl = MB_BYTE2LEN(p[n + fl]); 12448 mch_memmove(p + fl + tl, p, n); 12449 mb_char2bytes(c, p); 12450 mb_char2bytes(c2, p + tl); 12451 p = p + tl; 12452 } 12453 else 12454#endif 12455 { 12456 c = *p; 12457 *p = p[2]; 12458 p[2] = c; 12459 ++p; 12460 } 12461 12462 if (!soundfold && !spell_iswordp(p, curwin)) 12463 { 12464 /* Middle char is not a word char, skip the rotate. First and 12465 * third char were already checked at swap and swap3. */ 12466 sp->ts_state = STATE_REP_INI; 12467 break; 12468 } 12469 12470 /* Rotate three characters left: "123" -> "231". We change 12471 * "fword" here, it's changed back afterwards at STATE_UNROT3L. */ 12472 if (TRY_DEEPER(su, stack, depth, SCORE_SWAP3)) 12473 { 12474 go_deeper(stack, depth, SCORE_SWAP3); 12475#ifdef DEBUG_TRIEWALK 12476 p = fword + sp->ts_fidx; 12477 sprintf(changename[depth], "%.*s-%s: rotate left %c%c%c", 12478 sp->ts_twordlen, tword, fword + sp->ts_fidx, 12479 p[0], p[1], p[2]); 12480#endif 12481 sp->ts_state = STATE_UNROT3L; 12482 ++depth; 12483 p = fword + sp->ts_fidx; 12484#ifdef FEAT_MBYTE 12485 if (has_mbyte) 12486 { 12487 n = mb_cptr2len(p); 12488 c = mb_ptr2char(p); 12489 fl = mb_cptr2len(p + n); 12490 fl += mb_cptr2len(p + n + fl); 12491 mch_memmove(p, p + n, fl); 12492 mb_char2bytes(c, p + fl); 12493 stack[depth].ts_fidxtry = sp->ts_fidx + n + fl; 12494 } 12495 else 12496#endif 12497 { 12498 c = *p; 12499 *p = p[1]; 12500 p[1] = p[2]; 12501 p[2] = c; 12502 stack[depth].ts_fidxtry = sp->ts_fidx + 3; 12503 } 12504 } 12505 else 12506 sp->ts_state = STATE_REP_INI; 12507 break; 12508 12509 case STATE_UNROT3L: 12510 /* Undo ROT3L: "231" -> "123" */ 12511 p = fword + sp->ts_fidx; 12512#ifdef FEAT_MBYTE 12513 if (has_mbyte) 12514 { 12515 n = MB_BYTE2LEN(*p); 12516 n += MB_BYTE2LEN(p[n]); 12517 c = mb_ptr2char(p + n); 12518 tl = MB_BYTE2LEN(p[n]); 12519 mch_memmove(p + tl, p, n); 12520 mb_char2bytes(c, p); 12521 } 12522 else 12523#endif 12524 { 12525 c = p[2]; 12526 p[2] = p[1]; 12527 p[1] = *p; 12528 *p = c; 12529 } 12530 12531 /* Rotate three bytes right: "123" -> "312". We change "fword" 12532 * here, it's changed back afterwards at STATE_UNROT3R. */ 12533 if (TRY_DEEPER(su, stack, depth, SCORE_SWAP3)) 12534 { 12535 go_deeper(stack, depth, SCORE_SWAP3); 12536#ifdef DEBUG_TRIEWALK 12537 p = fword + sp->ts_fidx; 12538 sprintf(changename[depth], "%.*s-%s: rotate right %c%c%c", 12539 sp->ts_twordlen, tword, fword + sp->ts_fidx, 12540 p[0], p[1], p[2]); 12541#endif 12542 sp->ts_state = STATE_UNROT3R; 12543 ++depth; 12544 p = fword + sp->ts_fidx; 12545#ifdef FEAT_MBYTE 12546 if (has_mbyte) 12547 { 12548 n = mb_cptr2len(p); 12549 n += mb_cptr2len(p + n); 12550 c = mb_ptr2char(p + n); 12551 tl = mb_cptr2len(p + n); 12552 mch_memmove(p + tl, p, n); 12553 mb_char2bytes(c, p); 12554 stack[depth].ts_fidxtry = sp->ts_fidx + n + tl; 12555 } 12556 else 12557#endif 12558 { 12559 c = p[2]; 12560 p[2] = p[1]; 12561 p[1] = *p; 12562 *p = c; 12563 stack[depth].ts_fidxtry = sp->ts_fidx + 3; 12564 } 12565 } 12566 else 12567 sp->ts_state = STATE_REP_INI; 12568 break; 12569 12570 case STATE_UNROT3R: 12571 /* Undo ROT3R: "312" -> "123" */ 12572 p = fword + sp->ts_fidx; 12573#ifdef FEAT_MBYTE 12574 if (has_mbyte) 12575 { 12576 c = mb_ptr2char(p); 12577 tl = MB_BYTE2LEN(*p); 12578 n = MB_BYTE2LEN(p[tl]); 12579 n += MB_BYTE2LEN(p[tl + n]); 12580 mch_memmove(p, p + tl, n); 12581 mb_char2bytes(c, p + n); 12582 } 12583 else 12584#endif 12585 { 12586 c = *p; 12587 *p = p[1]; 12588 p[1] = p[2]; 12589 p[2] = c; 12590 } 12591 /*FALLTHROUGH*/ 12592 12593 case STATE_REP_INI: 12594 /* Check if matching with REP items from the .aff file would work. 12595 * Quickly skip if: 12596 * - there are no REP items and we are not in the soundfold trie 12597 * - the score is going to be too high anyway 12598 * - already applied a REP item or swapped here */ 12599 if ((lp->lp_replang == NULL && !soundfold) 12600 || sp->ts_score + SCORE_REP >= su->su_maxscore 12601 || sp->ts_fidx < sp->ts_fidxtry) 12602 { 12603 sp->ts_state = STATE_FINAL; 12604 break; 12605 } 12606 12607 /* Use the first byte to quickly find the first entry that may 12608 * match. If the index is -1 there is none. */ 12609 if (soundfold) 12610 sp->ts_curi = slang->sl_repsal_first[fword[sp->ts_fidx]]; 12611 else 12612 sp->ts_curi = lp->lp_replang->sl_rep_first[fword[sp->ts_fidx]]; 12613 12614 if (sp->ts_curi < 0) 12615 { 12616 sp->ts_state = STATE_FINAL; 12617 break; 12618 } 12619 12620 sp->ts_state = STATE_REP; 12621 /*FALLTHROUGH*/ 12622 12623 case STATE_REP: 12624 /* Try matching with REP items from the .aff file. For each match 12625 * replace the characters and check if the resulting word is 12626 * valid. */ 12627 p = fword + sp->ts_fidx; 12628 12629 if (soundfold) 12630 gap = &slang->sl_repsal; 12631 else 12632 gap = &lp->lp_replang->sl_rep; 12633 while (sp->ts_curi < gap->ga_len) 12634 { 12635 ftp = (fromto_T *)gap->ga_data + sp->ts_curi++; 12636 if (*ftp->ft_from != *p) 12637 { 12638 /* past possible matching entries */ 12639 sp->ts_curi = gap->ga_len; 12640 break; 12641 } 12642 if (STRNCMP(ftp->ft_from, p, STRLEN(ftp->ft_from)) == 0 12643 && TRY_DEEPER(su, stack, depth, SCORE_REP)) 12644 { 12645 go_deeper(stack, depth, SCORE_REP); 12646#ifdef DEBUG_TRIEWALK 12647 sprintf(changename[depth], "%.*s-%s: replace %s with %s", 12648 sp->ts_twordlen, tword, fword + sp->ts_fidx, 12649 ftp->ft_from, ftp->ft_to); 12650#endif 12651 /* Need to undo this afterwards. */ 12652 sp->ts_state = STATE_REP_UNDO; 12653 12654 /* Change the "from" to the "to" string. */ 12655 ++depth; 12656 fl = (int)STRLEN(ftp->ft_from); 12657 tl = (int)STRLEN(ftp->ft_to); 12658 if (fl != tl) 12659 { 12660 STRMOVE(p + tl, p + fl); 12661 repextra += tl - fl; 12662 } 12663 mch_memmove(p, ftp->ft_to, tl); 12664 stack[depth].ts_fidxtry = sp->ts_fidx + tl; 12665#ifdef FEAT_MBYTE 12666 stack[depth].ts_tcharlen = 0; 12667#endif 12668 break; 12669 } 12670 } 12671 12672 if (sp->ts_curi >= gap->ga_len && sp->ts_state == STATE_REP) 12673 /* No (more) matches. */ 12674 sp->ts_state = STATE_FINAL; 12675 12676 break; 12677 12678 case STATE_REP_UNDO: 12679 /* Undo a REP replacement and continue with the next one. */ 12680 if (soundfold) 12681 gap = &slang->sl_repsal; 12682 else 12683 gap = &lp->lp_replang->sl_rep; 12684 ftp = (fromto_T *)gap->ga_data + sp->ts_curi - 1; 12685 fl = (int)STRLEN(ftp->ft_from); 12686 tl = (int)STRLEN(ftp->ft_to); 12687 p = fword + sp->ts_fidx; 12688 if (fl != tl) 12689 { 12690 STRMOVE(p + fl, p + tl); 12691 repextra -= tl - fl; 12692 } 12693 mch_memmove(p, ftp->ft_from, fl); 12694 sp->ts_state = STATE_REP; 12695 break; 12696 12697 default: 12698 /* Did all possible states at this level, go up one level. */ 12699 --depth; 12700 12701 if (depth >= 0 && stack[depth].ts_prefixdepth == PFD_PREFIXTREE) 12702 { 12703 /* Continue in or go back to the prefix tree. */ 12704 byts = pbyts; 12705 idxs = pidxs; 12706 } 12707 12708 /* Don't check for CTRL-C too often, it takes time. */ 12709 if (--breakcheckcount == 0) 12710 { 12711 ui_breakcheck(); 12712 breakcheckcount = 1000; 12713 } 12714 } 12715 } 12716} 12717 12718 12719/* 12720 * Go one level deeper in the tree. 12721 */ 12722 static void 12723go_deeper(stack, depth, score_add) 12724 trystate_T *stack; 12725 int depth; 12726 int score_add; 12727{ 12728 stack[depth + 1] = stack[depth]; 12729 stack[depth + 1].ts_state = STATE_START; 12730 stack[depth + 1].ts_score = stack[depth].ts_score + score_add; 12731 stack[depth + 1].ts_curi = 1; /* start just after length byte */ 12732 stack[depth + 1].ts_flags = 0; 12733} 12734 12735#ifdef FEAT_MBYTE 12736/* 12737 * Case-folding may change the number of bytes: Count nr of chars in 12738 * fword[flen] and return the byte length of that many chars in "word". 12739 */ 12740 static int 12741nofold_len(fword, flen, word) 12742 char_u *fword; 12743 int flen; 12744 char_u *word; 12745{ 12746 char_u *p; 12747 int i = 0; 12748 12749 for (p = fword; p < fword + flen; mb_ptr_adv(p)) 12750 ++i; 12751 for (p = word; i > 0; mb_ptr_adv(p)) 12752 --i; 12753 return (int)(p - word); 12754} 12755#endif 12756 12757/* 12758 * "fword" is a good word with case folded. Find the matching keep-case 12759 * words and put it in "kword". 12760 * Theoretically there could be several keep-case words that result in the 12761 * same case-folded word, but we only find one... 12762 */ 12763 static void 12764find_keepcap_word(slang, fword, kword) 12765 slang_T *slang; 12766 char_u *fword; 12767 char_u *kword; 12768{ 12769 char_u uword[MAXWLEN]; /* "fword" in upper-case */ 12770 int depth; 12771 idx_T tryidx; 12772 12773 /* The following arrays are used at each depth in the tree. */ 12774 idx_T arridx[MAXWLEN]; 12775 int round[MAXWLEN]; 12776 int fwordidx[MAXWLEN]; 12777 int uwordidx[MAXWLEN]; 12778 int kwordlen[MAXWLEN]; 12779 12780 int flen, ulen; 12781 int l; 12782 int len; 12783 int c; 12784 idx_T lo, hi, m; 12785 char_u *p; 12786 char_u *byts = slang->sl_kbyts; /* array with bytes of the words */ 12787 idx_T *idxs = slang->sl_kidxs; /* array with indexes */ 12788 12789 if (byts == NULL) 12790 { 12791 /* array is empty: "cannot happen" */ 12792 *kword = NUL; 12793 return; 12794 } 12795 12796 /* Make an all-cap version of "fword". */ 12797 allcap_copy(fword, uword); 12798 12799 /* 12800 * Each character needs to be tried both case-folded and upper-case. 12801 * All this gets very complicated if we keep in mind that changing case 12802 * may change the byte length of a multi-byte character... 12803 */ 12804 depth = 0; 12805 arridx[0] = 0; 12806 round[0] = 0; 12807 fwordidx[0] = 0; 12808 uwordidx[0] = 0; 12809 kwordlen[0] = 0; 12810 while (depth >= 0) 12811 { 12812 if (fword[fwordidx[depth]] == NUL) 12813 { 12814 /* We are at the end of "fword". If the tree allows a word to end 12815 * here we have found a match. */ 12816 if (byts[arridx[depth] + 1] == 0) 12817 { 12818 kword[kwordlen[depth]] = NUL; 12819 return; 12820 } 12821 12822 /* kword is getting too long, continue one level up */ 12823 --depth; 12824 } 12825 else if (++round[depth] > 2) 12826 { 12827 /* tried both fold-case and upper-case character, continue one 12828 * level up */ 12829 --depth; 12830 } 12831 else 12832 { 12833 /* 12834 * round[depth] == 1: Try using the folded-case character. 12835 * round[depth] == 2: Try using the upper-case character. 12836 */ 12837#ifdef FEAT_MBYTE 12838 if (has_mbyte) 12839 { 12840 flen = mb_cptr2len(fword + fwordidx[depth]); 12841 ulen = mb_cptr2len(uword + uwordidx[depth]); 12842 } 12843 else 12844#endif 12845 ulen = flen = 1; 12846 if (round[depth] == 1) 12847 { 12848 p = fword + fwordidx[depth]; 12849 l = flen; 12850 } 12851 else 12852 { 12853 p = uword + uwordidx[depth]; 12854 l = ulen; 12855 } 12856 12857 for (tryidx = arridx[depth]; l > 0; --l) 12858 { 12859 /* Perform a binary search in the list of accepted bytes. */ 12860 len = byts[tryidx++]; 12861 c = *p++; 12862 lo = tryidx; 12863 hi = tryidx + len - 1; 12864 while (lo < hi) 12865 { 12866 m = (lo + hi) / 2; 12867 if (byts[m] > c) 12868 hi = m - 1; 12869 else if (byts[m] < c) 12870 lo = m + 1; 12871 else 12872 { 12873 lo = hi = m; 12874 break; 12875 } 12876 } 12877 12878 /* Stop if there is no matching byte. */ 12879 if (hi < lo || byts[lo] != c) 12880 break; 12881 12882 /* Continue at the child (if there is one). */ 12883 tryidx = idxs[lo]; 12884 } 12885 12886 if (l == 0) 12887 { 12888 /* 12889 * Found the matching char. Copy it to "kword" and go a 12890 * level deeper. 12891 */ 12892 if (round[depth] == 1) 12893 { 12894 STRNCPY(kword + kwordlen[depth], fword + fwordidx[depth], 12895 flen); 12896 kwordlen[depth + 1] = kwordlen[depth] + flen; 12897 } 12898 else 12899 { 12900 STRNCPY(kword + kwordlen[depth], uword + uwordidx[depth], 12901 ulen); 12902 kwordlen[depth + 1] = kwordlen[depth] + ulen; 12903 } 12904 fwordidx[depth + 1] = fwordidx[depth] + flen; 12905 uwordidx[depth + 1] = uwordidx[depth] + ulen; 12906 12907 ++depth; 12908 arridx[depth] = tryidx; 12909 round[depth] = 0; 12910 } 12911 } 12912 } 12913 12914 /* Didn't find it: "cannot happen". */ 12915 *kword = NUL; 12916} 12917 12918/* 12919 * Compute the sound-a-like score for suggestions in su->su_ga and add them to 12920 * su->su_sga. 12921 */ 12922 static void 12923score_comp_sal(su) 12924 suginfo_T *su; 12925{ 12926 langp_T *lp; 12927 char_u badsound[MAXWLEN]; 12928 int i; 12929 suggest_T *stp; 12930 suggest_T *sstp; 12931 int score; 12932 int lpi; 12933 12934 if (ga_grow(&su->su_sga, su->su_ga.ga_len) == FAIL) 12935 return; 12936 12937 /* Use the sound-folding of the first language that supports it. */ 12938 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi) 12939 { 12940 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi); 12941 if (lp->lp_slang->sl_sal.ga_len > 0) 12942 { 12943 /* soundfold the bad word */ 12944 spell_soundfold(lp->lp_slang, su->su_fbadword, TRUE, badsound); 12945 12946 for (i = 0; i < su->su_ga.ga_len; ++i) 12947 { 12948 stp = &SUG(su->su_ga, i); 12949 12950 /* Case-fold the suggested word, sound-fold it and compute the 12951 * sound-a-like score. */ 12952 score = stp_sal_score(stp, su, lp->lp_slang, badsound); 12953 if (score < SCORE_MAXMAX) 12954 { 12955 /* Add the suggestion. */ 12956 sstp = &SUG(su->su_sga, su->su_sga.ga_len); 12957 sstp->st_word = vim_strsave(stp->st_word); 12958 if (sstp->st_word != NULL) 12959 { 12960 sstp->st_wordlen = stp->st_wordlen; 12961 sstp->st_score = score; 12962 sstp->st_altscore = 0; 12963 sstp->st_orglen = stp->st_orglen; 12964 ++su->su_sga.ga_len; 12965 } 12966 } 12967 } 12968 break; 12969 } 12970 } 12971} 12972 12973/* 12974 * Combine the list of suggestions in su->su_ga and su->su_sga. 12975 * They are intwined. 12976 */ 12977 static void 12978score_combine(su) 12979 suginfo_T *su; 12980{ 12981 int i; 12982 int j; 12983 garray_T ga; 12984 garray_T *gap; 12985 langp_T *lp; 12986 suggest_T *stp; 12987 char_u *p; 12988 char_u badsound[MAXWLEN]; 12989 int round; 12990 int lpi; 12991 slang_T *slang = NULL; 12992 12993 /* Add the alternate score to su_ga. */ 12994 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi) 12995 { 12996 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi); 12997 if (lp->lp_slang->sl_sal.ga_len > 0) 12998 { 12999 /* soundfold the bad word */ 13000 slang = lp->lp_slang; 13001 spell_soundfold(slang, su->su_fbadword, TRUE, badsound); 13002 13003 for (i = 0; i < su->su_ga.ga_len; ++i) 13004 { 13005 stp = &SUG(su->su_ga, i); 13006 stp->st_altscore = stp_sal_score(stp, su, slang, badsound); 13007 if (stp->st_altscore == SCORE_MAXMAX) 13008 stp->st_score = (stp->st_score * 3 + SCORE_BIG) / 4; 13009 else 13010 stp->st_score = (stp->st_score * 3 13011 + stp->st_altscore) / 4; 13012 stp->st_salscore = FALSE; 13013 } 13014 break; 13015 } 13016 } 13017 13018 if (slang == NULL) /* Using "double" without sound folding. */ 13019 { 13020 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, 13021 su->su_maxcount); 13022 return; 13023 } 13024 13025 /* Add the alternate score to su_sga. */ 13026 for (i = 0; i < su->su_sga.ga_len; ++i) 13027 { 13028 stp = &SUG(su->su_sga, i); 13029 stp->st_altscore = spell_edit_score(slang, 13030 su->su_badword, stp->st_word); 13031 if (stp->st_score == SCORE_MAXMAX) 13032 stp->st_score = (SCORE_BIG * 7 + stp->st_altscore) / 8; 13033 else 13034 stp->st_score = (stp->st_score * 7 + stp->st_altscore) / 8; 13035 stp->st_salscore = TRUE; 13036 } 13037 13038 /* Remove bad suggestions, sort the suggestions and truncate at "maxcount" 13039 * for both lists. */ 13040 check_suggestions(su, &su->su_ga); 13041 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount); 13042 check_suggestions(su, &su->su_sga); 13043 (void)cleanup_suggestions(&su->su_sga, su->su_maxscore, su->su_maxcount); 13044 13045 ga_init2(&ga, (int)sizeof(suginfo_T), 1); 13046 if (ga_grow(&ga, su->su_ga.ga_len + su->su_sga.ga_len) == FAIL) 13047 return; 13048 13049 stp = &SUG(ga, 0); 13050 for (i = 0; i < su->su_ga.ga_len || i < su->su_sga.ga_len; ++i) 13051 { 13052 /* round 1: get a suggestion from su_ga 13053 * round 2: get a suggestion from su_sga */ 13054 for (round = 1; round <= 2; ++round) 13055 { 13056 gap = round == 1 ? &su->su_ga : &su->su_sga; 13057 if (i < gap->ga_len) 13058 { 13059 /* Don't add a word if it's already there. */ 13060 p = SUG(*gap, i).st_word; 13061 for (j = 0; j < ga.ga_len; ++j) 13062 if (STRCMP(stp[j].st_word, p) == 0) 13063 break; 13064 if (j == ga.ga_len) 13065 stp[ga.ga_len++] = SUG(*gap, i); 13066 else 13067 vim_free(p); 13068 } 13069 } 13070 } 13071 13072 ga_clear(&su->su_ga); 13073 ga_clear(&su->su_sga); 13074 13075 /* Truncate the list to the number of suggestions that will be displayed. */ 13076 if (ga.ga_len > su->su_maxcount) 13077 { 13078 for (i = su->su_maxcount; i < ga.ga_len; ++i) 13079 vim_free(stp[i].st_word); 13080 ga.ga_len = su->su_maxcount; 13081 } 13082 13083 su->su_ga = ga; 13084} 13085 13086/* 13087 * For the goodword in "stp" compute the soundalike score compared to the 13088 * badword. 13089 */ 13090 static int 13091stp_sal_score(stp, su, slang, badsound) 13092 suggest_T *stp; 13093 suginfo_T *su; 13094 slang_T *slang; 13095 char_u *badsound; /* sound-folded badword */ 13096{ 13097 char_u *p; 13098 char_u *pbad; 13099 char_u *pgood; 13100 char_u badsound2[MAXWLEN]; 13101 char_u fword[MAXWLEN]; 13102 char_u goodsound[MAXWLEN]; 13103 char_u goodword[MAXWLEN]; 13104 int lendiff; 13105 13106 lendiff = (int)(su->su_badlen - stp->st_orglen); 13107 if (lendiff >= 0) 13108 pbad = badsound; 13109 else 13110 { 13111 /* soundfold the bad word with more characters following */ 13112 (void)spell_casefold(su->su_badptr, stp->st_orglen, fword, MAXWLEN); 13113 13114 /* When joining two words the sound often changes a lot. E.g., "t he" 13115 * sounds like "t h" while "the" sounds like "@". Avoid that by 13116 * removing the space. Don't do it when the good word also contains a 13117 * space. */ 13118 if (vim_iswhite(su->su_badptr[su->su_badlen]) 13119 && *skiptowhite(stp->st_word) == NUL) 13120 for (p = fword; *(p = skiptowhite(p)) != NUL; ) 13121 STRMOVE(p, p + 1); 13122 13123 spell_soundfold(slang, fword, TRUE, badsound2); 13124 pbad = badsound2; 13125 } 13126 13127 if (lendiff > 0) 13128 { 13129 /* Add part of the bad word to the good word, so that we soundfold 13130 * what replaces the bad word. */ 13131 STRCPY(goodword, stp->st_word); 13132 vim_strncpy(goodword + stp->st_wordlen, 13133 su->su_badptr + su->su_badlen - lendiff, lendiff); 13134 pgood = goodword; 13135 } 13136 else 13137 pgood = stp->st_word; 13138 13139 /* Sound-fold the word and compute the score for the difference. */ 13140 spell_soundfold(slang, pgood, FALSE, goodsound); 13141 13142 return soundalike_score(goodsound, pbad); 13143} 13144 13145/* structure used to store soundfolded words that add_sound_suggest() has 13146 * handled already. */ 13147typedef struct 13148{ 13149 short sft_score; /* lowest score used */ 13150 char_u sft_word[1]; /* soundfolded word, actually longer */ 13151} sftword_T; 13152 13153static sftword_T dumsft; 13154#define HIKEY2SFT(p) ((sftword_T *)(p - (dumsft.sft_word - (char_u *)&dumsft))) 13155#define HI2SFT(hi) HIKEY2SFT((hi)->hi_key) 13156 13157/* 13158 * Prepare for calling suggest_try_soundalike(). 13159 */ 13160 static void 13161suggest_try_soundalike_prep() 13162{ 13163 langp_T *lp; 13164 int lpi; 13165 slang_T *slang; 13166 13167 /* Do this for all languages that support sound folding and for which a 13168 * .sug file has been loaded. */ 13169 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi) 13170 { 13171 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi); 13172 slang = lp->lp_slang; 13173 if (slang->sl_sal.ga_len > 0 && slang->sl_sbyts != NULL) 13174 /* prepare the hashtable used by add_sound_suggest() */ 13175 hash_init(&slang->sl_sounddone); 13176 } 13177} 13178 13179/* 13180 * Find suggestions by comparing the word in a sound-a-like form. 13181 * Note: This doesn't support postponed prefixes. 13182 */ 13183 static void 13184suggest_try_soundalike(su) 13185 suginfo_T *su; 13186{ 13187 char_u salword[MAXWLEN]; 13188 langp_T *lp; 13189 int lpi; 13190 slang_T *slang; 13191 13192 /* Do this for all languages that support sound folding and for which a 13193 * .sug file has been loaded. */ 13194 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi) 13195 { 13196 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi); 13197 slang = lp->lp_slang; 13198 if (slang->sl_sal.ga_len > 0 && slang->sl_sbyts != NULL) 13199 { 13200 /* soundfold the bad word */ 13201 spell_soundfold(slang, su->su_fbadword, TRUE, salword); 13202 13203 /* try all kinds of inserts/deletes/swaps/etc. */ 13204 /* TODO: also soundfold the next words, so that we can try joining 13205 * and splitting */ 13206 suggest_trie_walk(su, lp, salword, TRUE); 13207 } 13208 } 13209} 13210 13211/* 13212 * Finish up after calling suggest_try_soundalike(). 13213 */ 13214 static void 13215suggest_try_soundalike_finish() 13216{ 13217 langp_T *lp; 13218 int lpi; 13219 slang_T *slang; 13220 int todo; 13221 hashitem_T *hi; 13222 13223 /* Do this for all languages that support sound folding and for which a 13224 * .sug file has been loaded. */ 13225 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi) 13226 { 13227 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi); 13228 slang = lp->lp_slang; 13229 if (slang->sl_sal.ga_len > 0 && slang->sl_sbyts != NULL) 13230 { 13231 /* Free the info about handled words. */ 13232 todo = (int)slang->sl_sounddone.ht_used; 13233 for (hi = slang->sl_sounddone.ht_array; todo > 0; ++hi) 13234 if (!HASHITEM_EMPTY(hi)) 13235 { 13236 vim_free(HI2SFT(hi)); 13237 --todo; 13238 } 13239 13240 /* Clear the hashtable, it may also be used by another region. */ 13241 hash_clear(&slang->sl_sounddone); 13242 hash_init(&slang->sl_sounddone); 13243 } 13244 } 13245} 13246 13247/* 13248 * A match with a soundfolded word is found. Add the good word(s) that 13249 * produce this soundfolded word. 13250 */ 13251 static void 13252add_sound_suggest(su, goodword, score, lp) 13253 suginfo_T *su; 13254 char_u *goodword; 13255 int score; /* soundfold score */ 13256 langp_T *lp; 13257{ 13258 slang_T *slang = lp->lp_slang; /* language for sound folding */ 13259 int sfwordnr; 13260 char_u *nrline; 13261 int orgnr; 13262 char_u theword[MAXWLEN]; 13263 int i; 13264 int wlen; 13265 char_u *byts; 13266 idx_T *idxs; 13267 int n; 13268 int wordcount; 13269 int wc; 13270 int goodscore; 13271 hash_T hash; 13272 hashitem_T *hi; 13273 sftword_T *sft; 13274 int bc, gc; 13275 int limit; 13276 13277 /* 13278 * It's very well possible that the same soundfold word is found several 13279 * times with different scores. Since the following is quite slow only do 13280 * the words that have a better score than before. Use a hashtable to 13281 * remember the words that have been done. 13282 */ 13283 hash = hash_hash(goodword); 13284 hi = hash_lookup(&slang->sl_sounddone, goodword, hash); 13285 if (HASHITEM_EMPTY(hi)) 13286 { 13287 sft = (sftword_T *)alloc((unsigned)(sizeof(sftword_T) 13288 + STRLEN(goodword))); 13289 if (sft != NULL) 13290 { 13291 sft->sft_score = score; 13292 STRCPY(sft->sft_word, goodword); 13293 hash_add_item(&slang->sl_sounddone, hi, sft->sft_word, hash); 13294 } 13295 } 13296 else 13297 { 13298 sft = HI2SFT(hi); 13299 if (score >= sft->sft_score) 13300 return; 13301 sft->sft_score = score; 13302 } 13303 13304 /* 13305 * Find the word nr in the soundfold tree. 13306 */ 13307 sfwordnr = soundfold_find(slang, goodword); 13308 if (sfwordnr < 0) 13309 { 13310 EMSG2(_(e_intern2), "add_sound_suggest()"); 13311 return; 13312 } 13313 13314 /* 13315 * go over the list of good words that produce this soundfold word 13316 */ 13317 nrline = ml_get_buf(slang->sl_sugbuf, (linenr_T)(sfwordnr + 1), FALSE); 13318 orgnr = 0; 13319 while (*nrline != NUL) 13320 { 13321 /* The wordnr was stored in a minimal nr of bytes as an offset to the 13322 * previous wordnr. */ 13323 orgnr += bytes2offset(&nrline); 13324 13325 byts = slang->sl_fbyts; 13326 idxs = slang->sl_fidxs; 13327 13328 /* Lookup the word "orgnr" one of the two tries. */ 13329 n = 0; 13330 wlen = 0; 13331 wordcount = 0; 13332 for (;;) 13333 { 13334 i = 1; 13335 if (wordcount == orgnr && byts[n + 1] == NUL) 13336 break; /* found end of word */ 13337 13338 if (byts[n + 1] == NUL) 13339 ++wordcount; 13340 13341 /* skip over the NUL bytes */ 13342 for ( ; byts[n + i] == NUL; ++i) 13343 if (i > byts[n]) /* safety check */ 13344 { 13345 STRCPY(theword + wlen, "BAD"); 13346 goto badword; 13347 } 13348 13349 /* One of the siblings must have the word. */ 13350 for ( ; i < byts[n]; ++i) 13351 { 13352 wc = idxs[idxs[n + i]]; /* nr of words under this byte */ 13353 if (wordcount + wc > orgnr) 13354 break; 13355 wordcount += wc; 13356 } 13357 13358 theword[wlen++] = byts[n + i]; 13359 n = idxs[n + i]; 13360 } 13361badword: 13362 theword[wlen] = NUL; 13363 13364 /* Go over the possible flags and regions. */ 13365 for (; i <= byts[n] && byts[n + i] == NUL; ++i) 13366 { 13367 char_u cword[MAXWLEN]; 13368 char_u *p; 13369 int flags = (int)idxs[n + i]; 13370 13371 /* Skip words with the NOSUGGEST flag */ 13372 if (flags & WF_NOSUGGEST) 13373 continue; 13374 13375 if (flags & WF_KEEPCAP) 13376 { 13377 /* Must find the word in the keep-case tree. */ 13378 find_keepcap_word(slang, theword, cword); 13379 p = cword; 13380 } 13381 else 13382 { 13383 flags |= su->su_badflags; 13384 if ((flags & WF_CAPMASK) != 0) 13385 { 13386 /* Need to fix case according to "flags". */ 13387 make_case_word(theword, cword, flags); 13388 p = cword; 13389 } 13390 else 13391 p = theword; 13392 } 13393 13394 /* Add the suggestion. */ 13395 if (sps_flags & SPS_DOUBLE) 13396 { 13397 /* Add the suggestion if the score isn't too bad. */ 13398 if (score <= su->su_maxscore) 13399 add_suggestion(su, &su->su_sga, p, su->su_badlen, 13400 score, 0, FALSE, slang, FALSE); 13401 } 13402 else 13403 { 13404 /* Add a penalty for words in another region. */ 13405 if ((flags & WF_REGION) 13406 && (((unsigned)flags >> 16) & lp->lp_region) == 0) 13407 goodscore = SCORE_REGION; 13408 else 13409 goodscore = 0; 13410 13411 /* Add a small penalty for changing the first letter from 13412 * lower to upper case. Helps for "tath" -> "Kath", which is 13413 * less common thatn "tath" -> "path". Don't do it when the 13414 * letter is the same, that has already been counted. */ 13415 gc = PTR2CHAR(p); 13416 if (SPELL_ISUPPER(gc)) 13417 { 13418 bc = PTR2CHAR(su->su_badword); 13419 if (!SPELL_ISUPPER(bc) 13420 && SPELL_TOFOLD(bc) != SPELL_TOFOLD(gc)) 13421 goodscore += SCORE_ICASE / 2; 13422 } 13423 13424 /* Compute the score for the good word. This only does letter 13425 * insert/delete/swap/replace. REP items are not considered, 13426 * which may make the score a bit higher. 13427 * Use a limit for the score to make it work faster. Use 13428 * MAXSCORE(), because RESCORE() will change the score. 13429 * If the limit is very high then the iterative method is 13430 * inefficient, using an array is quicker. */ 13431 limit = MAXSCORE(su->su_sfmaxscore - goodscore, score); 13432 if (limit > SCORE_LIMITMAX) 13433 goodscore += spell_edit_score(slang, su->su_badword, p); 13434 else 13435 goodscore += spell_edit_score_limit(slang, su->su_badword, 13436 p, limit); 13437 13438 /* When going over the limit don't bother to do the rest. */ 13439 if (goodscore < SCORE_MAXMAX) 13440 { 13441 /* Give a bonus to words seen before. */ 13442 goodscore = score_wordcount_adj(slang, goodscore, p, FALSE); 13443 13444 /* Add the suggestion if the score isn't too bad. */ 13445 goodscore = RESCORE(goodscore, score); 13446 if (goodscore <= su->su_sfmaxscore) 13447 add_suggestion(su, &su->su_ga, p, su->su_badlen, 13448 goodscore, score, TRUE, slang, TRUE); 13449 } 13450 } 13451 } 13452 /* smsg("word %s (%d): %s (%d)", sftword, sftnr, theword, orgnr); */ 13453 } 13454} 13455 13456/* 13457 * Find word "word" in fold-case tree for "slang" and return the word number. 13458 */ 13459 static int 13460soundfold_find(slang, word) 13461 slang_T *slang; 13462 char_u *word; 13463{ 13464 idx_T arridx = 0; 13465 int len; 13466 int wlen = 0; 13467 int c; 13468 char_u *ptr = word; 13469 char_u *byts; 13470 idx_T *idxs; 13471 int wordnr = 0; 13472 13473 byts = slang->sl_sbyts; 13474 idxs = slang->sl_sidxs; 13475 13476 for (;;) 13477 { 13478 /* First byte is the number of possible bytes. */ 13479 len = byts[arridx++]; 13480 13481 /* If the first possible byte is a zero the word could end here. 13482 * If the word ends we found the word. If not skip the NUL bytes. */ 13483 c = ptr[wlen]; 13484 if (byts[arridx] == NUL) 13485 { 13486 if (c == NUL) 13487 break; 13488 13489 /* Skip over the zeros, there can be several. */ 13490 while (len > 0 && byts[arridx] == NUL) 13491 { 13492 ++arridx; 13493 --len; 13494 } 13495 if (len == 0) 13496 return -1; /* no children, word should have ended here */ 13497 ++wordnr; 13498 } 13499 13500 /* If the word ends we didn't find it. */ 13501 if (c == NUL) 13502 return -1; 13503 13504 /* Perform a binary search in the list of accepted bytes. */ 13505 if (c == TAB) /* <Tab> is handled like <Space> */ 13506 c = ' '; 13507 while (byts[arridx] < c) 13508 { 13509 /* The word count is in the first idxs[] entry of the child. */ 13510 wordnr += idxs[idxs[arridx]]; 13511 ++arridx; 13512 if (--len == 0) /* end of the bytes, didn't find it */ 13513 return -1; 13514 } 13515 if (byts[arridx] != c) /* didn't find the byte */ 13516 return -1; 13517 13518 /* Continue at the child (if there is one). */ 13519 arridx = idxs[arridx]; 13520 ++wlen; 13521 13522 /* One space in the good word may stand for several spaces in the 13523 * checked word. */ 13524 if (c == ' ') 13525 while (ptr[wlen] == ' ' || ptr[wlen] == TAB) 13526 ++wlen; 13527 } 13528 13529 return wordnr; 13530} 13531 13532/* 13533 * Copy "fword" to "cword", fixing case according to "flags". 13534 */ 13535 static void 13536make_case_word(fword, cword, flags) 13537 char_u *fword; 13538 char_u *cword; 13539 int flags; 13540{ 13541 if (flags & WF_ALLCAP) 13542 /* Make it all upper-case */ 13543 allcap_copy(fword, cword); 13544 else if (flags & WF_ONECAP) 13545 /* Make the first letter upper-case */ 13546 onecap_copy(fword, cword, TRUE); 13547 else 13548 /* Use goodword as-is. */ 13549 STRCPY(cword, fword); 13550} 13551 13552/* 13553 * Use map string "map" for languages "lp". 13554 */ 13555 static void 13556set_map_str(lp, map) 13557 slang_T *lp; 13558 char_u *map; 13559{ 13560 char_u *p; 13561 int headc = 0; 13562 int c; 13563 int i; 13564 13565 if (*map == NUL) 13566 { 13567 lp->sl_has_map = FALSE; 13568 return; 13569 } 13570 lp->sl_has_map = TRUE; 13571 13572 /* Init the array and hash tables empty. */ 13573 for (i = 0; i < 256; ++i) 13574 lp->sl_map_array[i] = 0; 13575#ifdef FEAT_MBYTE 13576 hash_init(&lp->sl_map_hash); 13577#endif 13578 13579 /* 13580 * The similar characters are stored separated with slashes: 13581 * "aaa/bbb/ccc/". Fill sl_map_array[c] with the character before c and 13582 * before the same slash. For characters above 255 sl_map_hash is used. 13583 */ 13584 for (p = map; *p != NUL; ) 13585 { 13586#ifdef FEAT_MBYTE 13587 c = mb_cptr2char_adv(&p); 13588#else 13589 c = *p++; 13590#endif 13591 if (c == '/') 13592 headc = 0; 13593 else 13594 { 13595 if (headc == 0) 13596 headc = c; 13597 13598#ifdef FEAT_MBYTE 13599 /* Characters above 255 don't fit in sl_map_array[], put them in 13600 * the hash table. Each entry is the char, a NUL the headchar and 13601 * a NUL. */ 13602 if (c >= 256) 13603 { 13604 int cl = mb_char2len(c); 13605 int headcl = mb_char2len(headc); 13606 char_u *b; 13607 hash_T hash; 13608 hashitem_T *hi; 13609 13610 b = alloc((unsigned)(cl + headcl + 2)); 13611 if (b == NULL) 13612 return; 13613 mb_char2bytes(c, b); 13614 b[cl] = NUL; 13615 mb_char2bytes(headc, b + cl + 1); 13616 b[cl + 1 + headcl] = NUL; 13617 hash = hash_hash(b); 13618 hi = hash_lookup(&lp->sl_map_hash, b, hash); 13619 if (HASHITEM_EMPTY(hi)) 13620 hash_add_item(&lp->sl_map_hash, hi, b, hash); 13621 else 13622 { 13623 /* This should have been checked when generating the .spl 13624 * file. */ 13625 EMSG(_("E783: duplicate char in MAP entry")); 13626 vim_free(b); 13627 } 13628 } 13629 else 13630#endif 13631 lp->sl_map_array[c] = headc; 13632 } 13633 } 13634} 13635 13636/* 13637 * Return TRUE if "c1" and "c2" are similar characters according to the MAP 13638 * lines in the .aff file. 13639 */ 13640 static int 13641similar_chars(slang, c1, c2) 13642 slang_T *slang; 13643 int c1; 13644 int c2; 13645{ 13646 int m1, m2; 13647#ifdef FEAT_MBYTE 13648 char_u buf[MB_MAXBYTES]; 13649 hashitem_T *hi; 13650 13651 if (c1 >= 256) 13652 { 13653 buf[mb_char2bytes(c1, buf)] = 0; 13654 hi = hash_find(&slang->sl_map_hash, buf); 13655 if (HASHITEM_EMPTY(hi)) 13656 m1 = 0; 13657 else 13658 m1 = mb_ptr2char(hi->hi_key + STRLEN(hi->hi_key) + 1); 13659 } 13660 else 13661#endif 13662 m1 = slang->sl_map_array[c1]; 13663 if (m1 == 0) 13664 return FALSE; 13665 13666 13667#ifdef FEAT_MBYTE 13668 if (c2 >= 256) 13669 { 13670 buf[mb_char2bytes(c2, buf)] = 0; 13671 hi = hash_find(&slang->sl_map_hash, buf); 13672 if (HASHITEM_EMPTY(hi)) 13673 m2 = 0; 13674 else 13675 m2 = mb_ptr2char(hi->hi_key + STRLEN(hi->hi_key) + 1); 13676 } 13677 else 13678#endif 13679 m2 = slang->sl_map_array[c2]; 13680 13681 return m1 == m2; 13682} 13683 13684/* 13685 * Add a suggestion to the list of suggestions. 13686 * For a suggestion that is already in the list the lowest score is remembered. 13687 */ 13688 static void 13689add_suggestion(su, gap, goodword, badlenarg, score, altscore, had_bonus, 13690 slang, maxsf) 13691 suginfo_T *su; 13692 garray_T *gap; /* either su_ga or su_sga */ 13693 char_u *goodword; 13694 int badlenarg; /* len of bad word replaced with "goodword" */ 13695 int score; 13696 int altscore; 13697 int had_bonus; /* value for st_had_bonus */ 13698 slang_T *slang; /* language for sound folding */ 13699 int maxsf; /* su_maxscore applies to soundfold score, 13700 su_sfmaxscore to the total score. */ 13701{ 13702 int goodlen; /* len of goodword changed */ 13703 int badlen; /* len of bad word changed */ 13704 suggest_T *stp; 13705 suggest_T new_sug; 13706 int i; 13707 char_u *pgood, *pbad; 13708 13709 /* Minimize "badlen" for consistency. Avoids that changing "the the" to 13710 * "thee the" is added next to changing the first "the" the "thee". */ 13711 pgood = goodword + STRLEN(goodword); 13712 pbad = su->su_badptr + badlenarg; 13713 for (;;) 13714 { 13715 goodlen = (int)(pgood - goodword); 13716 badlen = (int)(pbad - su->su_badptr); 13717 if (goodlen <= 0 || badlen <= 0) 13718 break; 13719 mb_ptr_back(goodword, pgood); 13720 mb_ptr_back(su->su_badptr, pbad); 13721#ifdef FEAT_MBYTE 13722 if (has_mbyte) 13723 { 13724 if (mb_ptr2char(pgood) != mb_ptr2char(pbad)) 13725 break; 13726 } 13727 else 13728#endif 13729 if (*pgood != *pbad) 13730 break; 13731 } 13732 13733 if (badlen == 0 && goodlen == 0) 13734 /* goodword doesn't change anything; may happen for "the the" changing 13735 * the first "the" to itself. */ 13736 return; 13737 13738 if (gap->ga_len == 0) 13739 i = -1; 13740 else 13741 { 13742 /* Check if the word is already there. Also check the length that is 13743 * being replaced "thes," -> "these" is a different suggestion from 13744 * "thes" -> "these". */ 13745 stp = &SUG(*gap, 0); 13746 for (i = gap->ga_len; --i >= 0; ++stp) 13747 if (stp->st_wordlen == goodlen 13748 && stp->st_orglen == badlen 13749 && STRNCMP(stp->st_word, goodword, goodlen) == 0) 13750 { 13751 /* 13752 * Found it. Remember the word with the lowest score. 13753 */ 13754 if (stp->st_slang == NULL) 13755 stp->st_slang = slang; 13756 13757 new_sug.st_score = score; 13758 new_sug.st_altscore = altscore; 13759 new_sug.st_had_bonus = had_bonus; 13760 13761 if (stp->st_had_bonus != had_bonus) 13762 { 13763 /* Only one of the two had the soundalike score computed. 13764 * Need to do that for the other one now, otherwise the 13765 * scores can't be compared. This happens because 13766 * suggest_try_change() doesn't compute the soundalike 13767 * word to keep it fast, while some special methods set 13768 * the soundalike score to zero. */ 13769 if (had_bonus) 13770 rescore_one(su, stp); 13771 else 13772 { 13773 new_sug.st_word = stp->st_word; 13774 new_sug.st_wordlen = stp->st_wordlen; 13775 new_sug.st_slang = stp->st_slang; 13776 new_sug.st_orglen = badlen; 13777 rescore_one(su, &new_sug); 13778 } 13779 } 13780 13781 if (stp->st_score > new_sug.st_score) 13782 { 13783 stp->st_score = new_sug.st_score; 13784 stp->st_altscore = new_sug.st_altscore; 13785 stp->st_had_bonus = new_sug.st_had_bonus; 13786 } 13787 break; 13788 } 13789 } 13790 13791 if (i < 0 && ga_grow(gap, 1) == OK) 13792 { 13793 /* Add a suggestion. */ 13794 stp = &SUG(*gap, gap->ga_len); 13795 stp->st_word = vim_strnsave(goodword, goodlen); 13796 if (stp->st_word != NULL) 13797 { 13798 stp->st_wordlen = goodlen; 13799 stp->st_score = score; 13800 stp->st_altscore = altscore; 13801 stp->st_had_bonus = had_bonus; 13802 stp->st_orglen = badlen; 13803 stp->st_slang = slang; 13804 ++gap->ga_len; 13805 13806 /* If we have too many suggestions now, sort the list and keep 13807 * the best suggestions. */ 13808 if (gap->ga_len > SUG_MAX_COUNT(su)) 13809 { 13810 if (maxsf) 13811 su->su_sfmaxscore = cleanup_suggestions(gap, 13812 su->su_sfmaxscore, SUG_CLEAN_COUNT(su)); 13813 else 13814 { 13815 i = su->su_maxscore; 13816 su->su_maxscore = cleanup_suggestions(gap, 13817 su->su_maxscore, SUG_CLEAN_COUNT(su)); 13818 } 13819 } 13820 } 13821 } 13822} 13823 13824/* 13825 * Suggestions may in fact be flagged as errors. Esp. for banned words and 13826 * for split words, such as "the the". Remove these from the list here. 13827 */ 13828 static void 13829check_suggestions(su, gap) 13830 suginfo_T *su; 13831 garray_T *gap; /* either su_ga or su_sga */ 13832{ 13833 suggest_T *stp; 13834 int i; 13835 char_u longword[MAXWLEN + 1]; 13836 int len; 13837 hlf_T attr; 13838 13839 stp = &SUG(*gap, 0); 13840 for (i = gap->ga_len - 1; i >= 0; --i) 13841 { 13842 /* Need to append what follows to check for "the the". */ 13843 STRCPY(longword, stp[i].st_word); 13844 len = stp[i].st_wordlen; 13845 vim_strncpy(longword + len, su->su_badptr + stp[i].st_orglen, 13846 MAXWLEN - len); 13847 attr = HLF_COUNT; 13848 (void)spell_check(curwin, longword, &attr, NULL, FALSE); 13849 if (attr != HLF_COUNT) 13850 { 13851 /* Remove this entry. */ 13852 vim_free(stp[i].st_word); 13853 --gap->ga_len; 13854 if (i < gap->ga_len) 13855 mch_memmove(stp + i, stp + i + 1, 13856 sizeof(suggest_T) * (gap->ga_len - i)); 13857 } 13858 } 13859} 13860 13861 13862/* 13863 * Add a word to be banned. 13864 */ 13865 static void 13866add_banned(su, word) 13867 suginfo_T *su; 13868 char_u *word; 13869{ 13870 char_u *s; 13871 hash_T hash; 13872 hashitem_T *hi; 13873 13874 hash = hash_hash(word); 13875 hi = hash_lookup(&su->su_banned, word, hash); 13876 if (HASHITEM_EMPTY(hi)) 13877 { 13878 s = vim_strsave(word); 13879 if (s != NULL) 13880 hash_add_item(&su->su_banned, hi, s, hash); 13881 } 13882} 13883 13884/* 13885 * Recompute the score for all suggestions if sound-folding is possible. This 13886 * is slow, thus only done for the final results. 13887 */ 13888 static void 13889rescore_suggestions(su) 13890 suginfo_T *su; 13891{ 13892 int i; 13893 13894 if (su->su_sallang != NULL) 13895 for (i = 0; i < su->su_ga.ga_len; ++i) 13896 rescore_one(su, &SUG(su->su_ga, i)); 13897} 13898 13899/* 13900 * Recompute the score for one suggestion if sound-folding is possible. 13901 */ 13902 static void 13903rescore_one(su, stp) 13904 suginfo_T *su; 13905 suggest_T *stp; 13906{ 13907 slang_T *slang = stp->st_slang; 13908 char_u sal_badword[MAXWLEN]; 13909 char_u *p; 13910 13911 /* Only rescore suggestions that have no sal score yet and do have a 13912 * language. */ 13913 if (slang != NULL && slang->sl_sal.ga_len > 0 && !stp->st_had_bonus) 13914 { 13915 if (slang == su->su_sallang) 13916 p = su->su_sal_badword; 13917 else 13918 { 13919 spell_soundfold(slang, su->su_fbadword, TRUE, sal_badword); 13920 p = sal_badword; 13921 } 13922 13923 stp->st_altscore = stp_sal_score(stp, su, slang, p); 13924 if (stp->st_altscore == SCORE_MAXMAX) 13925 stp->st_altscore = SCORE_BIG; 13926 stp->st_score = RESCORE(stp->st_score, stp->st_altscore); 13927 stp->st_had_bonus = TRUE; 13928 } 13929} 13930 13931static int 13932#ifdef __BORLANDC__ 13933_RTLENTRYF 13934#endif 13935sug_compare __ARGS((const void *s1, const void *s2)); 13936 13937/* 13938 * Function given to qsort() to sort the suggestions on st_score. 13939 * First on "st_score", then "st_altscore" then alphabetically. 13940 */ 13941 static int 13942#ifdef __BORLANDC__ 13943_RTLENTRYF 13944#endif 13945sug_compare(s1, s2) 13946 const void *s1; 13947 const void *s2; 13948{ 13949 suggest_T *p1 = (suggest_T *)s1; 13950 suggest_T *p2 = (suggest_T *)s2; 13951 int n = p1->st_score - p2->st_score; 13952 13953 if (n == 0) 13954 { 13955 n = p1->st_altscore - p2->st_altscore; 13956 if (n == 0) 13957 n = STRICMP(p1->st_word, p2->st_word); 13958 } 13959 return n; 13960} 13961 13962/* 13963 * Cleanup the suggestions: 13964 * - Sort on score. 13965 * - Remove words that won't be displayed. 13966 * Returns the maximum score in the list or "maxscore" unmodified. 13967 */ 13968 static int 13969cleanup_suggestions(gap, maxscore, keep) 13970 garray_T *gap; 13971 int maxscore; 13972 int keep; /* nr of suggestions to keep */ 13973{ 13974 suggest_T *stp = &SUG(*gap, 0); 13975 int i; 13976 13977 /* Sort the list. */ 13978 qsort(gap->ga_data, (size_t)gap->ga_len, sizeof(suggest_T), sug_compare); 13979 13980 /* Truncate the list to the number of suggestions that will be displayed. */ 13981 if (gap->ga_len > keep) 13982 { 13983 for (i = keep; i < gap->ga_len; ++i) 13984 vim_free(stp[i].st_word); 13985 gap->ga_len = keep; 13986 return stp[keep - 1].st_score; 13987 } 13988 return maxscore; 13989} 13990 13991#if defined(FEAT_EVAL) || defined(PROTO) 13992/* 13993 * Soundfold a string, for soundfold(). 13994 * Result is in allocated memory, NULL for an error. 13995 */ 13996 char_u * 13997eval_soundfold(word) 13998 char_u *word; 13999{ 14000 langp_T *lp; 14001 char_u sound[MAXWLEN]; 14002 int lpi; 14003 14004 if (curwin->w_p_spell && *curwin->w_s->b_p_spl != NUL) 14005 /* Use the sound-folding of the first language that supports it. */ 14006 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi) 14007 { 14008 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi); 14009 if (lp->lp_slang->sl_sal.ga_len > 0) 14010 { 14011 /* soundfold the word */ 14012 spell_soundfold(lp->lp_slang, word, FALSE, sound); 14013 return vim_strsave(sound); 14014 } 14015 } 14016 14017 /* No language with sound folding, return word as-is. */ 14018 return vim_strsave(word); 14019} 14020#endif 14021 14022/* 14023 * Turn "inword" into its sound-a-like equivalent in "res[MAXWLEN]". 14024 * 14025 * There are many ways to turn a word into a sound-a-like representation. The 14026 * oldest is Soundex (1918!). A nice overview can be found in "Approximate 14027 * swedish name matching - survey and test of different algorithms" by Klas 14028 * Erikson. 14029 * 14030 * We support two methods: 14031 * 1. SOFOFROM/SOFOTO do a simple character mapping. 14032 * 2. SAL items define a more advanced sound-folding (and much slower). 14033 */ 14034 static void 14035spell_soundfold(slang, inword, folded, res) 14036 slang_T *slang; 14037 char_u *inword; 14038 int folded; /* "inword" is already case-folded */ 14039 char_u *res; 14040{ 14041 char_u fword[MAXWLEN]; 14042 char_u *word; 14043 14044 if (slang->sl_sofo) 14045 /* SOFOFROM and SOFOTO used */ 14046 spell_soundfold_sofo(slang, inword, res); 14047 else 14048 { 14049 /* SAL items used. Requires the word to be case-folded. */ 14050 if (folded) 14051 word = inword; 14052 else 14053 { 14054 (void)spell_casefold(inword, (int)STRLEN(inword), fword, MAXWLEN); 14055 word = fword; 14056 } 14057 14058#ifdef FEAT_MBYTE 14059 if (has_mbyte) 14060 spell_soundfold_wsal(slang, word, res); 14061 else 14062#endif 14063 spell_soundfold_sal(slang, word, res); 14064 } 14065} 14066 14067/* 14068 * Perform sound folding of "inword" into "res" according to SOFOFROM and 14069 * SOFOTO lines. 14070 */ 14071 static void 14072spell_soundfold_sofo(slang, inword, res) 14073 slang_T *slang; 14074 char_u *inword; 14075 char_u *res; 14076{ 14077 char_u *s; 14078 int ri = 0; 14079 int c; 14080 14081#ifdef FEAT_MBYTE 14082 if (has_mbyte) 14083 { 14084 int prevc = 0; 14085 int *ip; 14086 14087 /* The sl_sal_first[] table contains the translation for chars up to 14088 * 255, sl_sal the rest. */ 14089 for (s = inword; *s != NUL; ) 14090 { 14091 c = mb_cptr2char_adv(&s); 14092 if (enc_utf8 ? utf_class(c) == 0 : vim_iswhite(c)) 14093 c = ' '; 14094 else if (c < 256) 14095 c = slang->sl_sal_first[c]; 14096 else 14097 { 14098 ip = ((int **)slang->sl_sal.ga_data)[c & 0xff]; 14099 if (ip == NULL) /* empty list, can't match */ 14100 c = NUL; 14101 else 14102 for (;;) /* find "c" in the list */ 14103 { 14104 if (*ip == 0) /* not found */ 14105 { 14106 c = NUL; 14107 break; 14108 } 14109 if (*ip == c) /* match! */ 14110 { 14111 c = ip[1]; 14112 break; 14113 } 14114 ip += 2; 14115 } 14116 } 14117 14118 if (c != NUL && c != prevc) 14119 { 14120 ri += mb_char2bytes(c, res + ri); 14121 if (ri + MB_MAXBYTES > MAXWLEN) 14122 break; 14123 prevc = c; 14124 } 14125 } 14126 } 14127 else 14128#endif 14129 { 14130 /* The sl_sal_first[] table contains the translation. */ 14131 for (s = inword; (c = *s) != NUL; ++s) 14132 { 14133 if (vim_iswhite(c)) 14134 c = ' '; 14135 else 14136 c = slang->sl_sal_first[c]; 14137 if (c != NUL && (ri == 0 || res[ri - 1] != c)) 14138 res[ri++] = c; 14139 } 14140 } 14141 14142 res[ri] = NUL; 14143} 14144 14145 static void 14146spell_soundfold_sal(slang, inword, res) 14147 slang_T *slang; 14148 char_u *inword; 14149 char_u *res; 14150{ 14151 salitem_T *smp; 14152 char_u word[MAXWLEN]; 14153 char_u *s = inword; 14154 char_u *t; 14155 char_u *pf; 14156 int i, j, z; 14157 int reslen; 14158 int n, k = 0; 14159 int z0; 14160 int k0; 14161 int n0; 14162 int c; 14163 int pri; 14164 int p0 = -333; 14165 int c0; 14166 14167 /* Remove accents, if wanted. We actually remove all non-word characters. 14168 * But keep white space. We need a copy, the word may be changed here. */ 14169 if (slang->sl_rem_accents) 14170 { 14171 t = word; 14172 while (*s != NUL) 14173 { 14174 if (vim_iswhite(*s)) 14175 { 14176 *t++ = ' '; 14177 s = skipwhite(s); 14178 } 14179 else 14180 { 14181 if (spell_iswordp_nmw(s)) 14182 *t++ = *s; 14183 ++s; 14184 } 14185 } 14186 *t = NUL; 14187 } 14188 else 14189 STRCPY(word, s); 14190 14191 smp = (salitem_T *)slang->sl_sal.ga_data; 14192 14193 /* 14194 * This comes from Aspell phonet.cpp. Converted from C++ to C. 14195 * Changed to keep spaces. 14196 */ 14197 i = reslen = z = 0; 14198 while ((c = word[i]) != NUL) 14199 { 14200 /* Start with the first rule that has the character in the word. */ 14201 n = slang->sl_sal_first[c]; 14202 z0 = 0; 14203 14204 if (n >= 0) 14205 { 14206 /* check all rules for the same letter */ 14207 for (; (s = smp[n].sm_lead)[0] == c; ++n) 14208 { 14209 /* Quickly skip entries that don't match the word. Most 14210 * entries are less then three chars, optimize for that. */ 14211 k = smp[n].sm_leadlen; 14212 if (k > 1) 14213 { 14214 if (word[i + 1] != s[1]) 14215 continue; 14216 if (k > 2) 14217 { 14218 for (j = 2; j < k; ++j) 14219 if (word[i + j] != s[j]) 14220 break; 14221 if (j < k) 14222 continue; 14223 } 14224 } 14225 14226 if ((pf = smp[n].sm_oneof) != NULL) 14227 { 14228 /* Check for match with one of the chars in "sm_oneof". */ 14229 while (*pf != NUL && *pf != word[i + k]) 14230 ++pf; 14231 if (*pf == NUL) 14232 continue; 14233 ++k; 14234 } 14235 s = smp[n].sm_rules; 14236 pri = 5; /* default priority */ 14237 14238 p0 = *s; 14239 k0 = k; 14240 while (*s == '-' && k > 1) 14241 { 14242 k--; 14243 s++; 14244 } 14245 if (*s == '<') 14246 s++; 14247 if (VIM_ISDIGIT(*s)) 14248 { 14249 /* determine priority */ 14250 pri = *s - '0'; 14251 s++; 14252 } 14253 if (*s == '^' && *(s + 1) == '^') 14254 s++; 14255 14256 if (*s == NUL 14257 || (*s == '^' 14258 && (i == 0 || !(word[i - 1] == ' ' 14259 || spell_iswordp(word + i - 1, curwin))) 14260 && (*(s + 1) != '$' 14261 || (!spell_iswordp(word + i + k0, curwin)))) 14262 || (*s == '$' && i > 0 14263 && spell_iswordp(word + i - 1, curwin) 14264 && (!spell_iswordp(word + i + k0, curwin)))) 14265 { 14266 /* search for followup rules, if: */ 14267 /* followup and k > 1 and NO '-' in searchstring */ 14268 c0 = word[i + k - 1]; 14269 n0 = slang->sl_sal_first[c0]; 14270 14271 if (slang->sl_followup && k > 1 && n0 >= 0 14272 && p0 != '-' && word[i + k] != NUL) 14273 { 14274 /* test follow-up rule for "word[i + k]" */ 14275 for ( ; (s = smp[n0].sm_lead)[0] == c0; ++n0) 14276 { 14277 /* Quickly skip entries that don't match the word. 14278 * */ 14279 k0 = smp[n0].sm_leadlen; 14280 if (k0 > 1) 14281 { 14282 if (word[i + k] != s[1]) 14283 continue; 14284 if (k0 > 2) 14285 { 14286 pf = word + i + k + 1; 14287 for (j = 2; j < k0; ++j) 14288 if (*pf++ != s[j]) 14289 break; 14290 if (j < k0) 14291 continue; 14292 } 14293 } 14294 k0 += k - 1; 14295 14296 if ((pf = smp[n0].sm_oneof) != NULL) 14297 { 14298 /* Check for match with one of the chars in 14299 * "sm_oneof". */ 14300 while (*pf != NUL && *pf != word[i + k0]) 14301 ++pf; 14302 if (*pf == NUL) 14303 continue; 14304 ++k0; 14305 } 14306 14307 p0 = 5; 14308 s = smp[n0].sm_rules; 14309 while (*s == '-') 14310 { 14311 /* "k0" gets NOT reduced because 14312 * "if (k0 == k)" */ 14313 s++; 14314 } 14315 if (*s == '<') 14316 s++; 14317 if (VIM_ISDIGIT(*s)) 14318 { 14319 p0 = *s - '0'; 14320 s++; 14321 } 14322 14323 if (*s == NUL 14324 /* *s == '^' cuts */ 14325 || (*s == '$' 14326 && !spell_iswordp(word + i + k0, 14327 curwin))) 14328 { 14329 if (k0 == k) 14330 /* this is just a piece of the string */ 14331 continue; 14332 14333 if (p0 < pri) 14334 /* priority too low */ 14335 continue; 14336 /* rule fits; stop search */ 14337 break; 14338 } 14339 } 14340 14341 if (p0 >= pri && smp[n0].sm_lead[0] == c0) 14342 continue; 14343 } 14344 14345 /* replace string */ 14346 s = smp[n].sm_to; 14347 if (s == NULL) 14348 s = (char_u *)""; 14349 pf = smp[n].sm_rules; 14350 p0 = (vim_strchr(pf, '<') != NULL) ? 1 : 0; 14351 if (p0 == 1 && z == 0) 14352 { 14353 /* rule with '<' is used */ 14354 if (reslen > 0 && *s != NUL && (res[reslen - 1] == c 14355 || res[reslen - 1] == *s)) 14356 reslen--; 14357 z0 = 1; 14358 z = 1; 14359 k0 = 0; 14360 while (*s != NUL && word[i + k0] != NUL) 14361 { 14362 word[i + k0] = *s; 14363 k0++; 14364 s++; 14365 } 14366 if (k > k0) 14367 STRMOVE(word + i + k0, word + i + k); 14368 14369 /* new "actual letter" */ 14370 c = word[i]; 14371 } 14372 else 14373 { 14374 /* no '<' rule used */ 14375 i += k - 1; 14376 z = 0; 14377 while (*s != NUL && s[1] != NUL && reslen < MAXWLEN) 14378 { 14379 if (reslen == 0 || res[reslen - 1] != *s) 14380 res[reslen++] = *s; 14381 s++; 14382 } 14383 /* new "actual letter" */ 14384 c = *s; 14385 if (strstr((char *)pf, "^^") != NULL) 14386 { 14387 if (c != NUL) 14388 res[reslen++] = c; 14389 STRMOVE(word, word + i + 1); 14390 i = 0; 14391 z0 = 1; 14392 } 14393 } 14394 break; 14395 } 14396 } 14397 } 14398 else if (vim_iswhite(c)) 14399 { 14400 c = ' '; 14401 k = 1; 14402 } 14403 14404 if (z0 == 0) 14405 { 14406 if (k && !p0 && reslen < MAXWLEN && c != NUL 14407 && (!slang->sl_collapse || reslen == 0 14408 || res[reslen - 1] != c)) 14409 /* condense only double letters */ 14410 res[reslen++] = c; 14411 14412 i++; 14413 z = 0; 14414 k = 0; 14415 } 14416 } 14417 14418 res[reslen] = NUL; 14419} 14420 14421#ifdef FEAT_MBYTE 14422/* 14423 * Turn "inword" into its sound-a-like equivalent in "res[MAXWLEN]". 14424 * Multi-byte version of spell_soundfold(). 14425 */ 14426 static void 14427spell_soundfold_wsal(slang, inword, res) 14428 slang_T *slang; 14429 char_u *inword; 14430 char_u *res; 14431{ 14432 salitem_T *smp = (salitem_T *)slang->sl_sal.ga_data; 14433 int word[MAXWLEN]; 14434 int wres[MAXWLEN]; 14435 int l; 14436 char_u *s; 14437 int *ws; 14438 char_u *t; 14439 int *pf; 14440 int i, j, z; 14441 int reslen; 14442 int n, k = 0; 14443 int z0; 14444 int k0; 14445 int n0; 14446 int c; 14447 int pri; 14448 int p0 = -333; 14449 int c0; 14450 int did_white = FALSE; 14451 14452 /* 14453 * Convert the multi-byte string to a wide-character string. 14454 * Remove accents, if wanted. We actually remove all non-word characters. 14455 * But keep white space. 14456 */ 14457 n = 0; 14458 for (s = inword; *s != NUL; ) 14459 { 14460 t = s; 14461 c = mb_cptr2char_adv(&s); 14462 if (slang->sl_rem_accents) 14463 { 14464 if (enc_utf8 ? utf_class(c) == 0 : vim_iswhite(c)) 14465 { 14466 if (did_white) 14467 continue; 14468 c = ' '; 14469 did_white = TRUE; 14470 } 14471 else 14472 { 14473 did_white = FALSE; 14474 if (!spell_iswordp_nmw(t)) 14475 continue; 14476 } 14477 } 14478 word[n++] = c; 14479 } 14480 word[n] = NUL; 14481 14482 /* 14483 * This comes from Aspell phonet.cpp. 14484 * Converted from C++ to C. Added support for multi-byte chars. 14485 * Changed to keep spaces. 14486 */ 14487 i = reslen = z = 0; 14488 while ((c = word[i]) != NUL) 14489 { 14490 /* Start with the first rule that has the character in the word. */ 14491 n = slang->sl_sal_first[c & 0xff]; 14492 z0 = 0; 14493 14494 if (n >= 0) 14495 { 14496 /* Check all rules for the same index byte. 14497 * If c is 0x300 need extra check for the end of the array, as 14498 * (c & 0xff) is NUL. */ 14499 for (; ((ws = smp[n].sm_lead_w)[0] & 0xff) == (c & 0xff) 14500 && ws[0] != NUL; ++n) 14501 { 14502 /* Quickly skip entries that don't match the word. Most 14503 * entries are less then three chars, optimize for that. */ 14504 if (c != ws[0]) 14505 continue; 14506 k = smp[n].sm_leadlen; 14507 if (k > 1) 14508 { 14509 if (word[i + 1] != ws[1]) 14510 continue; 14511 if (k > 2) 14512 { 14513 for (j = 2; j < k; ++j) 14514 if (word[i + j] != ws[j]) 14515 break; 14516 if (j < k) 14517 continue; 14518 } 14519 } 14520 14521 if ((pf = smp[n].sm_oneof_w) != NULL) 14522 { 14523 /* Check for match with one of the chars in "sm_oneof". */ 14524 while (*pf != NUL && *pf != word[i + k]) 14525 ++pf; 14526 if (*pf == NUL) 14527 continue; 14528 ++k; 14529 } 14530 s = smp[n].sm_rules; 14531 pri = 5; /* default priority */ 14532 14533 p0 = *s; 14534 k0 = k; 14535 while (*s == '-' && k > 1) 14536 { 14537 k--; 14538 s++; 14539 } 14540 if (*s == '<') 14541 s++; 14542 if (VIM_ISDIGIT(*s)) 14543 { 14544 /* determine priority */ 14545 pri = *s - '0'; 14546 s++; 14547 } 14548 if (*s == '^' && *(s + 1) == '^') 14549 s++; 14550 14551 if (*s == NUL 14552 || (*s == '^' 14553 && (i == 0 || !(word[i - 1] == ' ' 14554 || spell_iswordp_w(word + i - 1, curwin))) 14555 && (*(s + 1) != '$' 14556 || (!spell_iswordp_w(word + i + k0, curwin)))) 14557 || (*s == '$' && i > 0 14558 && spell_iswordp_w(word + i - 1, curwin) 14559 && (!spell_iswordp_w(word + i + k0, curwin)))) 14560 { 14561 /* search for followup rules, if: */ 14562 /* followup and k > 1 and NO '-' in searchstring */ 14563 c0 = word[i + k - 1]; 14564 n0 = slang->sl_sal_first[c0 & 0xff]; 14565 14566 if (slang->sl_followup && k > 1 && n0 >= 0 14567 && p0 != '-' && word[i + k] != NUL) 14568 { 14569 /* Test follow-up rule for "word[i + k]"; loop over 14570 * all entries with the same index byte. */ 14571 for ( ; ((ws = smp[n0].sm_lead_w)[0] & 0xff) 14572 == (c0 & 0xff); ++n0) 14573 { 14574 /* Quickly skip entries that don't match the word. 14575 */ 14576 if (c0 != ws[0]) 14577 continue; 14578 k0 = smp[n0].sm_leadlen; 14579 if (k0 > 1) 14580 { 14581 if (word[i + k] != ws[1]) 14582 continue; 14583 if (k0 > 2) 14584 { 14585 pf = word + i + k + 1; 14586 for (j = 2; j < k0; ++j) 14587 if (*pf++ != ws[j]) 14588 break; 14589 if (j < k0) 14590 continue; 14591 } 14592 } 14593 k0 += k - 1; 14594 14595 if ((pf = smp[n0].sm_oneof_w) != NULL) 14596 { 14597 /* Check for match with one of the chars in 14598 * "sm_oneof". */ 14599 while (*pf != NUL && *pf != word[i + k0]) 14600 ++pf; 14601 if (*pf == NUL) 14602 continue; 14603 ++k0; 14604 } 14605 14606 p0 = 5; 14607 s = smp[n0].sm_rules; 14608 while (*s == '-') 14609 { 14610 /* "k0" gets NOT reduced because 14611 * "if (k0 == k)" */ 14612 s++; 14613 } 14614 if (*s == '<') 14615 s++; 14616 if (VIM_ISDIGIT(*s)) 14617 { 14618 p0 = *s - '0'; 14619 s++; 14620 } 14621 14622 if (*s == NUL 14623 /* *s == '^' cuts */ 14624 || (*s == '$' 14625 && !spell_iswordp_w(word + i + k0, 14626 curwin))) 14627 { 14628 if (k0 == k) 14629 /* this is just a piece of the string */ 14630 continue; 14631 14632 if (p0 < pri) 14633 /* priority too low */ 14634 continue; 14635 /* rule fits; stop search */ 14636 break; 14637 } 14638 } 14639 14640 if (p0 >= pri && (smp[n0].sm_lead_w[0] & 0xff) 14641 == (c0 & 0xff)) 14642 continue; 14643 } 14644 14645 /* replace string */ 14646 ws = smp[n].sm_to_w; 14647 s = smp[n].sm_rules; 14648 p0 = (vim_strchr(s, '<') != NULL) ? 1 : 0; 14649 if (p0 == 1 && z == 0) 14650 { 14651 /* rule with '<' is used */ 14652 if (reslen > 0 && ws != NULL && *ws != NUL 14653 && (wres[reslen - 1] == c 14654 || wres[reslen - 1] == *ws)) 14655 reslen--; 14656 z0 = 1; 14657 z = 1; 14658 k0 = 0; 14659 if (ws != NULL) 14660 while (*ws != NUL && word[i + k0] != NUL) 14661 { 14662 word[i + k0] = *ws; 14663 k0++; 14664 ws++; 14665 } 14666 if (k > k0) 14667 mch_memmove(word + i + k0, word + i + k, 14668 sizeof(int) * (STRLEN(word + i + k) + 1)); 14669 14670 /* new "actual letter" */ 14671 c = word[i]; 14672 } 14673 else 14674 { 14675 /* no '<' rule used */ 14676 i += k - 1; 14677 z = 0; 14678 if (ws != NULL) 14679 while (*ws != NUL && ws[1] != NUL 14680 && reslen < MAXWLEN) 14681 { 14682 if (reslen == 0 || wres[reslen - 1] != *ws) 14683 wres[reslen++] = *ws; 14684 ws++; 14685 } 14686 /* new "actual letter" */ 14687 if (ws == NULL) 14688 c = NUL; 14689 else 14690 c = *ws; 14691 if (strstr((char *)s, "^^") != NULL) 14692 { 14693 if (c != NUL) 14694 wres[reslen++] = c; 14695 mch_memmove(word, word + i + 1, 14696 sizeof(int) * (STRLEN(word + i + 1) + 1)); 14697 i = 0; 14698 z0 = 1; 14699 } 14700 } 14701 break; 14702 } 14703 } 14704 } 14705 else if (vim_iswhite(c)) 14706 { 14707 c = ' '; 14708 k = 1; 14709 } 14710 14711 if (z0 == 0) 14712 { 14713 if (k && !p0 && reslen < MAXWLEN && c != NUL 14714 && (!slang->sl_collapse || reslen == 0 14715 || wres[reslen - 1] != c)) 14716 /* condense only double letters */ 14717 wres[reslen++] = c; 14718 14719 i++; 14720 z = 0; 14721 k = 0; 14722 } 14723 } 14724 14725 /* Convert wide characters in "wres" to a multi-byte string in "res". */ 14726 l = 0; 14727 for (n = 0; n < reslen; ++n) 14728 { 14729 l += mb_char2bytes(wres[n], res + l); 14730 if (l + MB_MAXBYTES > MAXWLEN) 14731 break; 14732 } 14733 res[l] = NUL; 14734} 14735#endif 14736 14737/* 14738 * Compute a score for two sound-a-like words. 14739 * This permits up to two inserts/deletes/swaps/etc. to keep things fast. 14740 * Instead of a generic loop we write out the code. That keeps it fast by 14741 * avoiding checks that will not be possible. 14742 */ 14743 static int 14744soundalike_score(goodstart, badstart) 14745 char_u *goodstart; /* sound-folded good word */ 14746 char_u *badstart; /* sound-folded bad word */ 14747{ 14748 char_u *goodsound = goodstart; 14749 char_u *badsound = badstart; 14750 int goodlen; 14751 int badlen; 14752 int n; 14753 char_u *pl, *ps; 14754 char_u *pl2, *ps2; 14755 int score = 0; 14756 14757 /* Adding/inserting "*" at the start (word starts with vowel) shouldn't be 14758 * counted so much, vowels halfway the word aren't counted at all. */ 14759 if ((*badsound == '*' || *goodsound == '*') && *badsound != *goodsound) 14760 { 14761 if ((badsound[0] == NUL && goodsound[1] == NUL) 14762 || (goodsound[0] == NUL && badsound[1] == NUL)) 14763 /* changing word with vowel to word without a sound */ 14764 return SCORE_DEL; 14765 if (badsound[0] == NUL || goodsound[0] == NUL) 14766 /* more than two changes */ 14767 return SCORE_MAXMAX; 14768 14769 if (badsound[1] == goodsound[1] 14770 || (badsound[1] != NUL 14771 && goodsound[1] != NUL 14772 && badsound[2] == goodsound[2])) 14773 { 14774 /* handle like a substitute */ 14775 } 14776 else 14777 { 14778 score = 2 * SCORE_DEL / 3; 14779 if (*badsound == '*') 14780 ++badsound; 14781 else 14782 ++goodsound; 14783 } 14784 } 14785 14786 goodlen = (int)STRLEN(goodsound); 14787 badlen = (int)STRLEN(badsound); 14788 14789 /* Return quickly if the lengths are too different to be fixed by two 14790 * changes. */ 14791 n = goodlen - badlen; 14792 if (n < -2 || n > 2) 14793 return SCORE_MAXMAX; 14794 14795 if (n > 0) 14796 { 14797 pl = goodsound; /* goodsound is longest */ 14798 ps = badsound; 14799 } 14800 else 14801 { 14802 pl = badsound; /* badsound is longest */ 14803 ps = goodsound; 14804 } 14805 14806 /* Skip over the identical part. */ 14807 while (*pl == *ps && *pl != NUL) 14808 { 14809 ++pl; 14810 ++ps; 14811 } 14812 14813 switch (n) 14814 { 14815 case -2: 14816 case 2: 14817 /* 14818 * Must delete two characters from "pl". 14819 */ 14820 ++pl; /* first delete */ 14821 while (*pl == *ps) 14822 { 14823 ++pl; 14824 ++ps; 14825 } 14826 /* strings must be equal after second delete */ 14827 if (STRCMP(pl + 1, ps) == 0) 14828 return score + SCORE_DEL * 2; 14829 14830 /* Failed to compare. */ 14831 break; 14832 14833 case -1: 14834 case 1: 14835 /* 14836 * Minimal one delete from "pl" required. 14837 */ 14838 14839 /* 1: delete */ 14840 pl2 = pl + 1; 14841 ps2 = ps; 14842 while (*pl2 == *ps2) 14843 { 14844 if (*pl2 == NUL) /* reached the end */ 14845 return score + SCORE_DEL; 14846 ++pl2; 14847 ++ps2; 14848 } 14849 14850 /* 2: delete then swap, then rest must be equal */ 14851 if (pl2[0] == ps2[1] && pl2[1] == ps2[0] 14852 && STRCMP(pl2 + 2, ps2 + 2) == 0) 14853 return score + SCORE_DEL + SCORE_SWAP; 14854 14855 /* 3: delete then substitute, then the rest must be equal */ 14856 if (STRCMP(pl2 + 1, ps2 + 1) == 0) 14857 return score + SCORE_DEL + SCORE_SUBST; 14858 14859 /* 4: first swap then delete */ 14860 if (pl[0] == ps[1] && pl[1] == ps[0]) 14861 { 14862 pl2 = pl + 2; /* swap, skip two chars */ 14863 ps2 = ps + 2; 14864 while (*pl2 == *ps2) 14865 { 14866 ++pl2; 14867 ++ps2; 14868 } 14869 /* delete a char and then strings must be equal */ 14870 if (STRCMP(pl2 + 1, ps2) == 0) 14871 return score + SCORE_SWAP + SCORE_DEL; 14872 } 14873 14874 /* 5: first substitute then delete */ 14875 pl2 = pl + 1; /* substitute, skip one char */ 14876 ps2 = ps + 1; 14877 while (*pl2 == *ps2) 14878 { 14879 ++pl2; 14880 ++ps2; 14881 } 14882 /* delete a char and then strings must be equal */ 14883 if (STRCMP(pl2 + 1, ps2) == 0) 14884 return score + SCORE_SUBST + SCORE_DEL; 14885 14886 /* Failed to compare. */ 14887 break; 14888 14889 case 0: 14890 /* 14891 * Lengths are equal, thus changes must result in same length: An 14892 * insert is only possible in combination with a delete. 14893 * 1: check if for identical strings 14894 */ 14895 if (*pl == NUL) 14896 return score; 14897 14898 /* 2: swap */ 14899 if (pl[0] == ps[1] && pl[1] == ps[0]) 14900 { 14901 pl2 = pl + 2; /* swap, skip two chars */ 14902 ps2 = ps + 2; 14903 while (*pl2 == *ps2) 14904 { 14905 if (*pl2 == NUL) /* reached the end */ 14906 return score + SCORE_SWAP; 14907 ++pl2; 14908 ++ps2; 14909 } 14910 /* 3: swap and swap again */ 14911 if (pl2[0] == ps2[1] && pl2[1] == ps2[0] 14912 && STRCMP(pl2 + 2, ps2 + 2) == 0) 14913 return score + SCORE_SWAP + SCORE_SWAP; 14914 14915 /* 4: swap and substitute */ 14916 if (STRCMP(pl2 + 1, ps2 + 1) == 0) 14917 return score + SCORE_SWAP + SCORE_SUBST; 14918 } 14919 14920 /* 5: substitute */ 14921 pl2 = pl + 1; 14922 ps2 = ps + 1; 14923 while (*pl2 == *ps2) 14924 { 14925 if (*pl2 == NUL) /* reached the end */ 14926 return score + SCORE_SUBST; 14927 ++pl2; 14928 ++ps2; 14929 } 14930 14931 /* 6: substitute and swap */ 14932 if (pl2[0] == ps2[1] && pl2[1] == ps2[0] 14933 && STRCMP(pl2 + 2, ps2 + 2) == 0) 14934 return score + SCORE_SUBST + SCORE_SWAP; 14935 14936 /* 7: substitute and substitute */ 14937 if (STRCMP(pl2 + 1, ps2 + 1) == 0) 14938 return score + SCORE_SUBST + SCORE_SUBST; 14939 14940 /* 8: insert then delete */ 14941 pl2 = pl; 14942 ps2 = ps + 1; 14943 while (*pl2 == *ps2) 14944 { 14945 ++pl2; 14946 ++ps2; 14947 } 14948 if (STRCMP(pl2 + 1, ps2) == 0) 14949 return score + SCORE_INS + SCORE_DEL; 14950 14951 /* 9: delete then insert */ 14952 pl2 = pl + 1; 14953 ps2 = ps; 14954 while (*pl2 == *ps2) 14955 { 14956 ++pl2; 14957 ++ps2; 14958 } 14959 if (STRCMP(pl2, ps2 + 1) == 0) 14960 return score + SCORE_INS + SCORE_DEL; 14961 14962 /* Failed to compare. */ 14963 break; 14964 } 14965 14966 return SCORE_MAXMAX; 14967} 14968 14969/* 14970 * Compute the "edit distance" to turn "badword" into "goodword". The less 14971 * deletes/inserts/substitutes/swaps are required the lower the score. 14972 * 14973 * The algorithm is described by Du and Chang, 1992. 14974 * The implementation of the algorithm comes from Aspell editdist.cpp, 14975 * edit_distance(). It has been converted from C++ to C and modified to 14976 * support multi-byte characters. 14977 */ 14978 static int 14979spell_edit_score(slang, badword, goodword) 14980 slang_T *slang; 14981 char_u *badword; 14982 char_u *goodword; 14983{ 14984 int *cnt; 14985 int badlen, goodlen; /* lengths including NUL */ 14986 int j, i; 14987 int t; 14988 int bc, gc; 14989 int pbc, pgc; 14990#ifdef FEAT_MBYTE 14991 char_u *p; 14992 int wbadword[MAXWLEN]; 14993 int wgoodword[MAXWLEN]; 14994 14995 if (has_mbyte) 14996 { 14997 /* Get the characters from the multi-byte strings and put them in an 14998 * int array for easy access. */ 14999 for (p = badword, badlen = 0; *p != NUL; ) 15000 wbadword[badlen++] = mb_cptr2char_adv(&p); 15001 wbadword[badlen++] = 0; 15002 for (p = goodword, goodlen = 0; *p != NUL; ) 15003 wgoodword[goodlen++] = mb_cptr2char_adv(&p); 15004 wgoodword[goodlen++] = 0; 15005 } 15006 else 15007#endif 15008 { 15009 badlen = (int)STRLEN(badword) + 1; 15010 goodlen = (int)STRLEN(goodword) + 1; 15011 } 15012 15013 /* We use "cnt" as an array: CNT(badword_idx, goodword_idx). */ 15014#define CNT(a, b) cnt[(a) + (b) * (badlen + 1)] 15015 cnt = (int *)lalloc((long_u)(sizeof(int) * (badlen + 1) * (goodlen + 1)), 15016 TRUE); 15017 if (cnt == NULL) 15018 return 0; /* out of memory */ 15019 15020 CNT(0, 0) = 0; 15021 for (j = 1; j <= goodlen; ++j) 15022 CNT(0, j) = CNT(0, j - 1) + SCORE_INS; 15023 15024 for (i = 1; i <= badlen; ++i) 15025 { 15026 CNT(i, 0) = CNT(i - 1, 0) + SCORE_DEL; 15027 for (j = 1; j <= goodlen; ++j) 15028 { 15029#ifdef FEAT_MBYTE 15030 if (has_mbyte) 15031 { 15032 bc = wbadword[i - 1]; 15033 gc = wgoodword[j - 1]; 15034 } 15035 else 15036#endif 15037 { 15038 bc = badword[i - 1]; 15039 gc = goodword[j - 1]; 15040 } 15041 if (bc == gc) 15042 CNT(i, j) = CNT(i - 1, j - 1); 15043 else 15044 { 15045 /* Use a better score when there is only a case difference. */ 15046 if (SPELL_TOFOLD(bc) == SPELL_TOFOLD(gc)) 15047 CNT(i, j) = SCORE_ICASE + CNT(i - 1, j - 1); 15048 else 15049 { 15050 /* For a similar character use SCORE_SIMILAR. */ 15051 if (slang != NULL 15052 && slang->sl_has_map 15053 && similar_chars(slang, gc, bc)) 15054 CNT(i, j) = SCORE_SIMILAR + CNT(i - 1, j - 1); 15055 else 15056 CNT(i, j) = SCORE_SUBST + CNT(i - 1, j - 1); 15057 } 15058 15059 if (i > 1 && j > 1) 15060 { 15061#ifdef FEAT_MBYTE 15062 if (has_mbyte) 15063 { 15064 pbc = wbadword[i - 2]; 15065 pgc = wgoodword[j - 2]; 15066 } 15067 else 15068#endif 15069 { 15070 pbc = badword[i - 2]; 15071 pgc = goodword[j - 2]; 15072 } 15073 if (bc == pgc && pbc == gc) 15074 { 15075 t = SCORE_SWAP + CNT(i - 2, j - 2); 15076 if (t < CNT(i, j)) 15077 CNT(i, j) = t; 15078 } 15079 } 15080 t = SCORE_DEL + CNT(i - 1, j); 15081 if (t < CNT(i, j)) 15082 CNT(i, j) = t; 15083 t = SCORE_INS + CNT(i, j - 1); 15084 if (t < CNT(i, j)) 15085 CNT(i, j) = t; 15086 } 15087 } 15088 } 15089 15090 i = CNT(badlen - 1, goodlen - 1); 15091 vim_free(cnt); 15092 return i; 15093} 15094 15095typedef struct 15096{ 15097 int badi; 15098 int goodi; 15099 int score; 15100} limitscore_T; 15101 15102/* 15103 * Like spell_edit_score(), but with a limit on the score to make it faster. 15104 * May return SCORE_MAXMAX when the score is higher than "limit". 15105 * 15106 * This uses a stack for the edits still to be tried. 15107 * The idea comes from Aspell leditdist.cpp. Rewritten in C and added support 15108 * for multi-byte characters. 15109 */ 15110 static int 15111spell_edit_score_limit(slang, badword, goodword, limit) 15112 slang_T *slang; 15113 char_u *badword; 15114 char_u *goodword; 15115 int limit; 15116{ 15117 limitscore_T stack[10]; /* allow for over 3 * 2 edits */ 15118 int stackidx; 15119 int bi, gi; 15120 int bi2, gi2; 15121 int bc, gc; 15122 int score; 15123 int score_off; 15124 int minscore; 15125 int round; 15126 15127#ifdef FEAT_MBYTE 15128 /* Multi-byte characters require a bit more work, use a different function 15129 * to avoid testing "has_mbyte" quite often. */ 15130 if (has_mbyte) 15131 return spell_edit_score_limit_w(slang, badword, goodword, limit); 15132#endif 15133 15134 /* 15135 * The idea is to go from start to end over the words. So long as 15136 * characters are equal just continue, this always gives the lowest score. 15137 * When there is a difference try several alternatives. Each alternative 15138 * increases "score" for the edit distance. Some of the alternatives are 15139 * pushed unto a stack and tried later, some are tried right away. At the 15140 * end of the word the score for one alternative is known. The lowest 15141 * possible score is stored in "minscore". 15142 */ 15143 stackidx = 0; 15144 bi = 0; 15145 gi = 0; 15146 score = 0; 15147 minscore = limit + 1; 15148 15149 for (;;) 15150 { 15151 /* Skip over an equal part, score remains the same. */ 15152 for (;;) 15153 { 15154 bc = badword[bi]; 15155 gc = goodword[gi]; 15156 if (bc != gc) /* stop at a char that's different */ 15157 break; 15158 if (bc == NUL) /* both words end */ 15159 { 15160 if (score < minscore) 15161 minscore = score; 15162 goto pop; /* do next alternative */ 15163 } 15164 ++bi; 15165 ++gi; 15166 } 15167 15168 if (gc == NUL) /* goodword ends, delete badword chars */ 15169 { 15170 do 15171 { 15172 if ((score += SCORE_DEL) >= minscore) 15173 goto pop; /* do next alternative */ 15174 } while (badword[++bi] != NUL); 15175 minscore = score; 15176 } 15177 else if (bc == NUL) /* badword ends, insert badword chars */ 15178 { 15179 do 15180 { 15181 if ((score += SCORE_INS) >= minscore) 15182 goto pop; /* do next alternative */ 15183 } while (goodword[++gi] != NUL); 15184 minscore = score; 15185 } 15186 else /* both words continue */ 15187 { 15188 /* If not close to the limit, perform a change. Only try changes 15189 * that may lead to a lower score than "minscore". 15190 * round 0: try deleting a char from badword 15191 * round 1: try inserting a char in badword */ 15192 for (round = 0; round <= 1; ++round) 15193 { 15194 score_off = score + (round == 0 ? SCORE_DEL : SCORE_INS); 15195 if (score_off < minscore) 15196 { 15197 if (score_off + SCORE_EDIT_MIN >= minscore) 15198 { 15199 /* Near the limit, rest of the words must match. We 15200 * can check that right now, no need to push an item 15201 * onto the stack. */ 15202 bi2 = bi + 1 - round; 15203 gi2 = gi + round; 15204 while (goodword[gi2] == badword[bi2]) 15205 { 15206 if (goodword[gi2] == NUL) 15207 { 15208 minscore = score_off; 15209 break; 15210 } 15211 ++bi2; 15212 ++gi2; 15213 } 15214 } 15215 else 15216 { 15217 /* try deleting/inserting a character later */ 15218 stack[stackidx].badi = bi + 1 - round; 15219 stack[stackidx].goodi = gi + round; 15220 stack[stackidx].score = score_off; 15221 ++stackidx; 15222 } 15223 } 15224 } 15225 15226 if (score + SCORE_SWAP < minscore) 15227 { 15228 /* If swapping two characters makes a match then the 15229 * substitution is more expensive, thus there is no need to 15230 * try both. */ 15231 if (gc == badword[bi + 1] && bc == goodword[gi + 1]) 15232 { 15233 /* Swap two characters, that is: skip them. */ 15234 gi += 2; 15235 bi += 2; 15236 score += SCORE_SWAP; 15237 continue; 15238 } 15239 } 15240 15241 /* Substitute one character for another which is the same 15242 * thing as deleting a character from both goodword and badword. 15243 * Use a better score when there is only a case difference. */ 15244 if (SPELL_TOFOLD(bc) == SPELL_TOFOLD(gc)) 15245 score += SCORE_ICASE; 15246 else 15247 { 15248 /* For a similar character use SCORE_SIMILAR. */ 15249 if (slang != NULL 15250 && slang->sl_has_map 15251 && similar_chars(slang, gc, bc)) 15252 score += SCORE_SIMILAR; 15253 else 15254 score += SCORE_SUBST; 15255 } 15256 15257 if (score < minscore) 15258 { 15259 /* Do the substitution. */ 15260 ++gi; 15261 ++bi; 15262 continue; 15263 } 15264 } 15265pop: 15266 /* 15267 * Get here to try the next alternative, pop it from the stack. 15268 */ 15269 if (stackidx == 0) /* stack is empty, finished */ 15270 break; 15271 15272 /* pop an item from the stack */ 15273 --stackidx; 15274 gi = stack[stackidx].goodi; 15275 bi = stack[stackidx].badi; 15276 score = stack[stackidx].score; 15277 } 15278 15279 /* When the score goes over "limit" it may actually be much higher. 15280 * Return a very large number to avoid going below the limit when giving a 15281 * bonus. */ 15282 if (minscore > limit) 15283 return SCORE_MAXMAX; 15284 return minscore; 15285} 15286 15287#ifdef FEAT_MBYTE 15288/* 15289 * Multi-byte version of spell_edit_score_limit(). 15290 * Keep it in sync with the above! 15291 */ 15292 static int 15293spell_edit_score_limit_w(slang, badword, goodword, limit) 15294 slang_T *slang; 15295 char_u *badword; 15296 char_u *goodword; 15297 int limit; 15298{ 15299 limitscore_T stack[10]; /* allow for over 3 * 2 edits */ 15300 int stackidx; 15301 int bi, gi; 15302 int bi2, gi2; 15303 int bc, gc; 15304 int score; 15305 int score_off; 15306 int minscore; 15307 int round; 15308 char_u *p; 15309 int wbadword[MAXWLEN]; 15310 int wgoodword[MAXWLEN]; 15311 15312 /* Get the characters from the multi-byte strings and put them in an 15313 * int array for easy access. */ 15314 bi = 0; 15315 for (p = badword; *p != NUL; ) 15316 wbadword[bi++] = mb_cptr2char_adv(&p); 15317 wbadword[bi++] = 0; 15318 gi = 0; 15319 for (p = goodword; *p != NUL; ) 15320 wgoodword[gi++] = mb_cptr2char_adv(&p); 15321 wgoodword[gi++] = 0; 15322 15323 /* 15324 * The idea is to go from start to end over the words. So long as 15325 * characters are equal just continue, this always gives the lowest score. 15326 * When there is a difference try several alternatives. Each alternative 15327 * increases "score" for the edit distance. Some of the alternatives are 15328 * pushed unto a stack and tried later, some are tried right away. At the 15329 * end of the word the score for one alternative is known. The lowest 15330 * possible score is stored in "minscore". 15331 */ 15332 stackidx = 0; 15333 bi = 0; 15334 gi = 0; 15335 score = 0; 15336 minscore = limit + 1; 15337 15338 for (;;) 15339 { 15340 /* Skip over an equal part, score remains the same. */ 15341 for (;;) 15342 { 15343 bc = wbadword[bi]; 15344 gc = wgoodword[gi]; 15345 15346 if (bc != gc) /* stop at a char that's different */ 15347 break; 15348 if (bc == NUL) /* both words end */ 15349 { 15350 if (score < minscore) 15351 minscore = score; 15352 goto pop; /* do next alternative */ 15353 } 15354 ++bi; 15355 ++gi; 15356 } 15357 15358 if (gc == NUL) /* goodword ends, delete badword chars */ 15359 { 15360 do 15361 { 15362 if ((score += SCORE_DEL) >= minscore) 15363 goto pop; /* do next alternative */ 15364 } while (wbadword[++bi] != NUL); 15365 minscore = score; 15366 } 15367 else if (bc == NUL) /* badword ends, insert badword chars */ 15368 { 15369 do 15370 { 15371 if ((score += SCORE_INS) >= minscore) 15372 goto pop; /* do next alternative */ 15373 } while (wgoodword[++gi] != NUL); 15374 minscore = score; 15375 } 15376 else /* both words continue */ 15377 { 15378 /* If not close to the limit, perform a change. Only try changes 15379 * that may lead to a lower score than "minscore". 15380 * round 0: try deleting a char from badword 15381 * round 1: try inserting a char in badword */ 15382 for (round = 0; round <= 1; ++round) 15383 { 15384 score_off = score + (round == 0 ? SCORE_DEL : SCORE_INS); 15385 if (score_off < minscore) 15386 { 15387 if (score_off + SCORE_EDIT_MIN >= minscore) 15388 { 15389 /* Near the limit, rest of the words must match. We 15390 * can check that right now, no need to push an item 15391 * onto the stack. */ 15392 bi2 = bi + 1 - round; 15393 gi2 = gi + round; 15394 while (wgoodword[gi2] == wbadword[bi2]) 15395 { 15396 if (wgoodword[gi2] == NUL) 15397 { 15398 minscore = score_off; 15399 break; 15400 } 15401 ++bi2; 15402 ++gi2; 15403 } 15404 } 15405 else 15406 { 15407 /* try deleting a character from badword later */ 15408 stack[stackidx].badi = bi + 1 - round; 15409 stack[stackidx].goodi = gi + round; 15410 stack[stackidx].score = score_off; 15411 ++stackidx; 15412 } 15413 } 15414 } 15415 15416 if (score + SCORE_SWAP < minscore) 15417 { 15418 /* If swapping two characters makes a match then the 15419 * substitution is more expensive, thus there is no need to 15420 * try both. */ 15421 if (gc == wbadword[bi + 1] && bc == wgoodword[gi + 1]) 15422 { 15423 /* Swap two characters, that is: skip them. */ 15424 gi += 2; 15425 bi += 2; 15426 score += SCORE_SWAP; 15427 continue; 15428 } 15429 } 15430 15431 /* Substitute one character for another which is the same 15432 * thing as deleting a character from both goodword and badword. 15433 * Use a better score when there is only a case difference. */ 15434 if (SPELL_TOFOLD(bc) == SPELL_TOFOLD(gc)) 15435 score += SCORE_ICASE; 15436 else 15437 { 15438 /* For a similar character use SCORE_SIMILAR. */ 15439 if (slang != NULL 15440 && slang->sl_has_map 15441 && similar_chars(slang, gc, bc)) 15442 score += SCORE_SIMILAR; 15443 else 15444 score += SCORE_SUBST; 15445 } 15446 15447 if (score < minscore) 15448 { 15449 /* Do the substitution. */ 15450 ++gi; 15451 ++bi; 15452 continue; 15453 } 15454 } 15455pop: 15456 /* 15457 * Get here to try the next alternative, pop it from the stack. 15458 */ 15459 if (stackidx == 0) /* stack is empty, finished */ 15460 break; 15461 15462 /* pop an item from the stack */ 15463 --stackidx; 15464 gi = stack[stackidx].goodi; 15465 bi = stack[stackidx].badi; 15466 score = stack[stackidx].score; 15467 } 15468 15469 /* When the score goes over "limit" it may actually be much higher. 15470 * Return a very large number to avoid going below the limit when giving a 15471 * bonus. */ 15472 if (minscore > limit) 15473 return SCORE_MAXMAX; 15474 return minscore; 15475} 15476#endif 15477 15478/* 15479 * ":spellinfo" 15480 */ 15481 void 15482ex_spellinfo(eap) 15483 exarg_T *eap UNUSED; 15484{ 15485 int lpi; 15486 langp_T *lp; 15487 char_u *p; 15488 15489 if (no_spell_checking(curwin)) 15490 return; 15491 15492 msg_start(); 15493 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len && !got_int; ++lpi) 15494 { 15495 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi); 15496 msg_puts((char_u *)"file: "); 15497 msg_puts(lp->lp_slang->sl_fname); 15498 msg_putchar('\n'); 15499 p = lp->lp_slang->sl_info; 15500 if (p != NULL) 15501 { 15502 msg_puts(p); 15503 msg_putchar('\n'); 15504 } 15505 } 15506 msg_end(); 15507} 15508 15509#define DUMPFLAG_KEEPCASE 1 /* round 2: keep-case tree */ 15510#define DUMPFLAG_COUNT 2 /* include word count */ 15511#define DUMPFLAG_ICASE 4 /* ignore case when finding matches */ 15512#define DUMPFLAG_ONECAP 8 /* pattern starts with capital */ 15513#define DUMPFLAG_ALLCAP 16 /* pattern is all capitals */ 15514 15515/* 15516 * ":spelldump" 15517 */ 15518 void 15519ex_spelldump(eap) 15520 exarg_T *eap; 15521{ 15522 if (no_spell_checking(curwin)) 15523 return; 15524 15525 /* Create a new empty buffer by splitting the window. */ 15526 do_cmdline_cmd((char_u *)"new"); 15527 if (!bufempty() || !buf_valid(curbuf)) 15528 return; 15529 15530 spell_dump_compl(NULL, 0, NULL, eap->forceit ? DUMPFLAG_COUNT : 0); 15531 15532 /* Delete the empty line that we started with. */ 15533 if (curbuf->b_ml.ml_line_count > 1) 15534 ml_delete(curbuf->b_ml.ml_line_count, FALSE); 15535 15536 redraw_later(NOT_VALID); 15537} 15538 15539/* 15540 * Go through all possible words and: 15541 * 1. When "pat" is NULL: dump a list of all words in the current buffer. 15542 * "ic" and "dir" are not used. 15543 * 2. When "pat" is not NULL: add matching words to insert mode completion. 15544 */ 15545 void 15546spell_dump_compl(pat, ic, dir, dumpflags_arg) 15547 char_u *pat; /* leading part of the word */ 15548 int ic; /* ignore case */ 15549 int *dir; /* direction for adding matches */ 15550 int dumpflags_arg; /* DUMPFLAG_* */ 15551{ 15552 langp_T *lp; 15553 slang_T *slang; 15554 idx_T arridx[MAXWLEN]; 15555 int curi[MAXWLEN]; 15556 char_u word[MAXWLEN]; 15557 int c; 15558 char_u *byts; 15559 idx_T *idxs; 15560 linenr_T lnum = 0; 15561 int round; 15562 int depth; 15563 int n; 15564 int flags; 15565 char_u *region_names = NULL; /* region names being used */ 15566 int do_region = TRUE; /* dump region names and numbers */ 15567 char_u *p; 15568 int lpi; 15569 int dumpflags = dumpflags_arg; 15570 int patlen; 15571 15572 /* When ignoring case or when the pattern starts with capital pass this on 15573 * to dump_word(). */ 15574 if (pat != NULL) 15575 { 15576 if (ic) 15577 dumpflags |= DUMPFLAG_ICASE; 15578 else 15579 { 15580 n = captype(pat, NULL); 15581 if (n == WF_ONECAP) 15582 dumpflags |= DUMPFLAG_ONECAP; 15583 else if (n == WF_ALLCAP 15584#ifdef FEAT_MBYTE 15585 && (int)STRLEN(pat) > mb_ptr2len(pat) 15586#else 15587 && (int)STRLEN(pat) > 1 15588#endif 15589 ) 15590 dumpflags |= DUMPFLAG_ALLCAP; 15591 } 15592 } 15593 15594 /* Find out if we can support regions: All languages must support the same 15595 * regions or none at all. */ 15596 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi) 15597 { 15598 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi); 15599 p = lp->lp_slang->sl_regions; 15600 if (p[0] != 0) 15601 { 15602 if (region_names == NULL) /* first language with regions */ 15603 region_names = p; 15604 else if (STRCMP(region_names, p) != 0) 15605 { 15606 do_region = FALSE; /* region names are different */ 15607 break; 15608 } 15609 } 15610 } 15611 15612 if (do_region && region_names != NULL) 15613 { 15614 if (pat == NULL) 15615 { 15616 vim_snprintf((char *)IObuff, IOSIZE, "/regions=%s", region_names); 15617 ml_append(lnum++, IObuff, (colnr_T)0, FALSE); 15618 } 15619 } 15620 else 15621 do_region = FALSE; 15622 15623 /* 15624 * Loop over all files loaded for the entries in 'spelllang'. 15625 */ 15626 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi) 15627 { 15628 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi); 15629 slang = lp->lp_slang; 15630 if (slang->sl_fbyts == NULL) /* reloading failed */ 15631 continue; 15632 15633 if (pat == NULL) 15634 { 15635 vim_snprintf((char *)IObuff, IOSIZE, "# file: %s", slang->sl_fname); 15636 ml_append(lnum++, IObuff, (colnr_T)0, FALSE); 15637 } 15638 15639 /* When matching with a pattern and there are no prefixes only use 15640 * parts of the tree that match "pat". */ 15641 if (pat != NULL && slang->sl_pbyts == NULL) 15642 patlen = (int)STRLEN(pat); 15643 else 15644 patlen = -1; 15645 15646 /* round 1: case-folded tree 15647 * round 2: keep-case tree */ 15648 for (round = 1; round <= 2; ++round) 15649 { 15650 if (round == 1) 15651 { 15652 dumpflags &= ~DUMPFLAG_KEEPCASE; 15653 byts = slang->sl_fbyts; 15654 idxs = slang->sl_fidxs; 15655 } 15656 else 15657 { 15658 dumpflags |= DUMPFLAG_KEEPCASE; 15659 byts = slang->sl_kbyts; 15660 idxs = slang->sl_kidxs; 15661 } 15662 if (byts == NULL) 15663 continue; /* array is empty */ 15664 15665 depth = 0; 15666 arridx[0] = 0; 15667 curi[0] = 1; 15668 while (depth >= 0 && !got_int 15669 && (pat == NULL || !compl_interrupted)) 15670 { 15671 if (curi[depth] > byts[arridx[depth]]) 15672 { 15673 /* Done all bytes at this node, go up one level. */ 15674 --depth; 15675 line_breakcheck(); 15676 ins_compl_check_keys(50); 15677 } 15678 else 15679 { 15680 /* Do one more byte at this node. */ 15681 n = arridx[depth] + curi[depth]; 15682 ++curi[depth]; 15683 c = byts[n]; 15684 if (c == 0) 15685 { 15686 /* End of word, deal with the word. 15687 * Don't use keep-case words in the fold-case tree, 15688 * they will appear in the keep-case tree. 15689 * Only use the word when the region matches. */ 15690 flags = (int)idxs[n]; 15691 if ((round == 2 || (flags & WF_KEEPCAP) == 0) 15692 && (flags & WF_NEEDCOMP) == 0 15693 && (do_region 15694 || (flags & WF_REGION) == 0 15695 || (((unsigned)flags >> 16) 15696 & lp->lp_region) != 0)) 15697 { 15698 word[depth] = NUL; 15699 if (!do_region) 15700 flags &= ~WF_REGION; 15701 15702 /* Dump the basic word if there is no prefix or 15703 * when it's the first one. */ 15704 c = (unsigned)flags >> 24; 15705 if (c == 0 || curi[depth] == 2) 15706 { 15707 dump_word(slang, word, pat, dir, 15708 dumpflags, flags, lnum); 15709 if (pat == NULL) 15710 ++lnum; 15711 } 15712 15713 /* Apply the prefix, if there is one. */ 15714 if (c != 0) 15715 lnum = dump_prefixes(slang, word, pat, dir, 15716 dumpflags, flags, lnum); 15717 } 15718 } 15719 else 15720 { 15721 /* Normal char, go one level deeper. */ 15722 word[depth++] = c; 15723 arridx[depth] = idxs[n]; 15724 curi[depth] = 1; 15725 15726 /* Check if this characters matches with the pattern. 15727 * If not skip the whole tree below it. 15728 * Always ignore case here, dump_word() will check 15729 * proper case later. This isn't exactly right when 15730 * length changes for multi-byte characters with 15731 * ignore case... */ 15732 if (depth <= patlen 15733 && MB_STRNICMP(word, pat, depth) != 0) 15734 --depth; 15735 } 15736 } 15737 } 15738 } 15739 } 15740} 15741 15742/* 15743 * Dump one word: apply case modifications and append a line to the buffer. 15744 * When "lnum" is zero add insert mode completion. 15745 */ 15746 static void 15747dump_word(slang, word, pat, dir, dumpflags, wordflags, lnum) 15748 slang_T *slang; 15749 char_u *word; 15750 char_u *pat; 15751 int *dir; 15752 int dumpflags; 15753 int wordflags; 15754 linenr_T lnum; 15755{ 15756 int keepcap = FALSE; 15757 char_u *p; 15758 char_u *tw; 15759 char_u cword[MAXWLEN]; 15760 char_u badword[MAXWLEN + 10]; 15761 int i; 15762 int flags = wordflags; 15763 15764 if (dumpflags & DUMPFLAG_ONECAP) 15765 flags |= WF_ONECAP; 15766 if (dumpflags & DUMPFLAG_ALLCAP) 15767 flags |= WF_ALLCAP; 15768 15769 if ((dumpflags & DUMPFLAG_KEEPCASE) == 0 && (flags & WF_CAPMASK) != 0) 15770 { 15771 /* Need to fix case according to "flags". */ 15772 make_case_word(word, cword, flags); 15773 p = cword; 15774 } 15775 else 15776 { 15777 p = word; 15778 if ((dumpflags & DUMPFLAG_KEEPCASE) 15779 && ((captype(word, NULL) & WF_KEEPCAP) == 0 15780 || (flags & WF_FIXCAP) != 0)) 15781 keepcap = TRUE; 15782 } 15783 tw = p; 15784 15785 if (pat == NULL) 15786 { 15787 /* Add flags and regions after a slash. */ 15788 if ((flags & (WF_BANNED | WF_RARE | WF_REGION)) || keepcap) 15789 { 15790 STRCPY(badword, p); 15791 STRCAT(badword, "/"); 15792 if (keepcap) 15793 STRCAT(badword, "="); 15794 if (flags & WF_BANNED) 15795 STRCAT(badword, "!"); 15796 else if (flags & WF_RARE) 15797 STRCAT(badword, "?"); 15798 if (flags & WF_REGION) 15799 for (i = 0; i < 7; ++i) 15800 if (flags & (0x10000 << i)) 15801 sprintf((char *)badword + STRLEN(badword), "%d", i + 1); 15802 p = badword; 15803 } 15804 15805 if (dumpflags & DUMPFLAG_COUNT) 15806 { 15807 hashitem_T *hi; 15808 15809 /* Include the word count for ":spelldump!". */ 15810 hi = hash_find(&slang->sl_wordcount, tw); 15811 if (!HASHITEM_EMPTY(hi)) 15812 { 15813 vim_snprintf((char *)IObuff, IOSIZE, "%s\t%d", 15814 tw, HI2WC(hi)->wc_count); 15815 p = IObuff; 15816 } 15817 } 15818 15819 ml_append(lnum, p, (colnr_T)0, FALSE); 15820 } 15821 else if (((dumpflags & DUMPFLAG_ICASE) 15822 ? MB_STRNICMP(p, pat, STRLEN(pat)) == 0 15823 : STRNCMP(p, pat, STRLEN(pat)) == 0) 15824 && ins_compl_add_infercase(p, (int)STRLEN(p), 15825 p_ic, NULL, *dir, 0) == OK) 15826 /* if dir was BACKWARD then honor it just once */ 15827 *dir = FORWARD; 15828} 15829 15830/* 15831 * For ":spelldump": Find matching prefixes for "word". Prepend each to 15832 * "word" and append a line to the buffer. 15833 * When "lnum" is zero add insert mode completion. 15834 * Return the updated line number. 15835 */ 15836 static linenr_T 15837dump_prefixes(slang, word, pat, dir, dumpflags, flags, startlnum) 15838 slang_T *slang; 15839 char_u *word; /* case-folded word */ 15840 char_u *pat; 15841 int *dir; 15842 int dumpflags; 15843 int flags; /* flags with prefix ID */ 15844 linenr_T startlnum; 15845{ 15846 idx_T arridx[MAXWLEN]; 15847 int curi[MAXWLEN]; 15848 char_u prefix[MAXWLEN]; 15849 char_u word_up[MAXWLEN]; 15850 int has_word_up = FALSE; 15851 int c; 15852 char_u *byts; 15853 idx_T *idxs; 15854 linenr_T lnum = startlnum; 15855 int depth; 15856 int n; 15857 int len; 15858 int i; 15859 15860 /* If the word starts with a lower-case letter make the word with an 15861 * upper-case letter in word_up[]. */ 15862 c = PTR2CHAR(word); 15863 if (SPELL_TOUPPER(c) != c) 15864 { 15865 onecap_copy(word, word_up, TRUE); 15866 has_word_up = TRUE; 15867 } 15868 15869 byts = slang->sl_pbyts; 15870 idxs = slang->sl_pidxs; 15871 if (byts != NULL) /* array not is empty */ 15872 { 15873 /* 15874 * Loop over all prefixes, building them byte-by-byte in prefix[]. 15875 * When at the end of a prefix check that it supports "flags". 15876 */ 15877 depth = 0; 15878 arridx[0] = 0; 15879 curi[0] = 1; 15880 while (depth >= 0 && !got_int) 15881 { 15882 n = arridx[depth]; 15883 len = byts[n]; 15884 if (curi[depth] > len) 15885 { 15886 /* Done all bytes at this node, go up one level. */ 15887 --depth; 15888 line_breakcheck(); 15889 } 15890 else 15891 { 15892 /* Do one more byte at this node. */ 15893 n += curi[depth]; 15894 ++curi[depth]; 15895 c = byts[n]; 15896 if (c == 0) 15897 { 15898 /* End of prefix, find out how many IDs there are. */ 15899 for (i = 1; i < len; ++i) 15900 if (byts[n + i] != 0) 15901 break; 15902 curi[depth] += i - 1; 15903 15904 c = valid_word_prefix(i, n, flags, word, slang, FALSE); 15905 if (c != 0) 15906 { 15907 vim_strncpy(prefix + depth, word, MAXWLEN - depth - 1); 15908 dump_word(slang, prefix, pat, dir, dumpflags, 15909 (c & WF_RAREPFX) ? (flags | WF_RARE) 15910 : flags, lnum); 15911 if (lnum != 0) 15912 ++lnum; 15913 } 15914 15915 /* Check for prefix that matches the word when the 15916 * first letter is upper-case, but only if the prefix has 15917 * a condition. */ 15918 if (has_word_up) 15919 { 15920 c = valid_word_prefix(i, n, flags, word_up, slang, 15921 TRUE); 15922 if (c != 0) 15923 { 15924 vim_strncpy(prefix + depth, word_up, 15925 MAXWLEN - depth - 1); 15926 dump_word(slang, prefix, pat, dir, dumpflags, 15927 (c & WF_RAREPFX) ? (flags | WF_RARE) 15928 : flags, lnum); 15929 if (lnum != 0) 15930 ++lnum; 15931 } 15932 } 15933 } 15934 else 15935 { 15936 /* Normal char, go one level deeper. */ 15937 prefix[depth++] = c; 15938 arridx[depth] = idxs[n]; 15939 curi[depth] = 1; 15940 } 15941 } 15942 } 15943 } 15944 15945 return lnum; 15946} 15947 15948/* 15949 * Move "p" to the end of word "start". 15950 * Uses the spell-checking word characters. 15951 */ 15952 char_u * 15953spell_to_word_end(start, win) 15954 char_u *start; 15955 win_T *win; 15956{ 15957 char_u *p = start; 15958 15959 while (*p != NUL && spell_iswordp(p, win)) 15960 mb_ptr_adv(p); 15961 return p; 15962} 15963 15964#if defined(FEAT_INS_EXPAND) || defined(PROTO) 15965/* 15966 * For Insert mode completion CTRL-X s: 15967 * Find start of the word in front of column "startcol". 15968 * We don't check if it is badly spelled, with completion we can only change 15969 * the word in front of the cursor. 15970 * Returns the column number of the word. 15971 */ 15972 int 15973spell_word_start(startcol) 15974 int startcol; 15975{ 15976 char_u *line; 15977 char_u *p; 15978 int col = 0; 15979 15980 if (no_spell_checking(curwin)) 15981 return startcol; 15982 15983 /* Find a word character before "startcol". */ 15984 line = ml_get_curline(); 15985 for (p = line + startcol; p > line; ) 15986 { 15987 mb_ptr_back(line, p); 15988 if (spell_iswordp_nmw(p)) 15989 break; 15990 } 15991 15992 /* Go back to start of the word. */ 15993 while (p > line) 15994 { 15995 col = (int)(p - line); 15996 mb_ptr_back(line, p); 15997 if (!spell_iswordp(p, curwin)) 15998 break; 15999 col = 0; 16000 } 16001 16002 return col; 16003} 16004 16005/* 16006 * Need to check for 'spellcapcheck' now, the word is removed before 16007 * expand_spelling() is called. Therefore the ugly global variable. 16008 */ 16009static int spell_expand_need_cap; 16010 16011 void 16012spell_expand_check_cap(col) 16013 colnr_T col; 16014{ 16015 spell_expand_need_cap = check_need_cap(curwin->w_cursor.lnum, col); 16016} 16017 16018/* 16019 * Get list of spelling suggestions. 16020 * Used for Insert mode completion CTRL-X ?. 16021 * Returns the number of matches. The matches are in "matchp[]", array of 16022 * allocated strings. 16023 */ 16024 int 16025expand_spelling(lnum, pat, matchp) 16026 linenr_T lnum UNUSED; 16027 char_u *pat; 16028 char_u ***matchp; 16029{ 16030 garray_T ga; 16031 16032 spell_suggest_list(&ga, pat, 100, spell_expand_need_cap, TRUE); 16033 *matchp = ga.ga_data; 16034 return ga.ga_len; 16035} 16036#endif 16037 16038#endif /* FEAT_SPELL */ 16039