1/* 2 * NKF - Ruby extension for Network Kanji Filter 3 * 4 * original nkf2.x is maintained at http://sourceforge.jp/projects/nkf/ 5 * 6 * $Id: nkf.c 36966 2012-09-14 16:07:49Z naruse $ 7 * 8 */ 9 10#define RUBY_NKF_REVISION "$Revision: 36966 $" 11#define RUBY_NKF_VERSION NKF_VERSION " (" NKF_RELEASE_DATE ")" 12 13#include "ruby/ruby.h" 14#include "ruby/encoding.h" 15 16/* Replace nkf's getchar/putchar for variable modification */ 17/* we never use getc, ungetc */ 18 19#undef getc 20#undef ungetc 21#define getc(f) (input_ctr>=i_len?-1:input[input_ctr++]) 22#define ungetc(c,f) input_ctr-- 23 24#define INCSIZE 32 25#undef putchar 26#undef TRUE 27#undef FALSE 28#define putchar(c) rb_nkf_putchar(c) 29 30/* Input/Output pointers */ 31 32static unsigned char *output; 33static unsigned char *input; 34static int input_ctr; 35static int i_len; 36static int output_ctr; 37static int o_len; 38static int incsize; 39 40static VALUE result; 41 42static int 43rb_nkf_putchar(unsigned int c) 44{ 45 if (output_ctr >= o_len) { 46 o_len += incsize; 47 rb_str_resize(result, o_len); 48 incsize *= 2; 49 output = (unsigned char *)RSTRING_PTR(result); 50 } 51 output[output_ctr++] = c; 52 53 return c; 54} 55 56/* Include kanji filter main part */ 57/* getchar and putchar will be replaced during inclusion */ 58 59#define PERL_XS 1 60#include "nkf-utf8/config.h" 61#include "nkf-utf8/utf8tbl.c" 62#include "nkf-utf8/nkf.c" 63 64rb_encoding* rb_nkf_enc_get(const char *name) 65{ 66 int idx = rb_enc_find_index(name); 67 if (idx < 0) { 68 nkf_encoding *nkf_enc = nkf_enc_find(name); 69 idx = rb_enc_find_index(nkf_enc_name(nkf_enc_to_base_encoding(nkf_enc))); 70 if (idx < 0) { 71 idx = rb_define_dummy_encoding(name); 72 } 73 } 74 return rb_enc_from_index(idx); 75} 76 77int nkf_split_options(const char *arg) 78{ 79 int count = 0; 80 unsigned char option[256]; 81 int i = 0, j = 0; 82 int is_escaped = FALSE; 83 int is_single_quoted = FALSE; 84 int is_double_quoted = FALSE; 85 for(i = 0; arg[i]; i++){ 86 if(j == 255){ 87 return -1; 88 }else if(is_single_quoted){ 89 if(arg[i] == '\''){ 90 is_single_quoted = FALSE; 91 }else{ 92 option[j++] = arg[i]; 93 } 94 }else if(is_escaped){ 95 is_escaped = FALSE; 96 option[j++] = arg[i]; 97 }else if(arg[i] == '\\'){ 98 is_escaped = TRUE; 99 }else if(is_double_quoted){ 100 if(arg[i] == '"'){ 101 is_double_quoted = FALSE; 102 }else{ 103 option[j++] = arg[i]; 104 } 105 }else if(arg[i] == '\''){ 106 is_single_quoted = TRUE; 107 }else if(arg[i] == '"'){ 108 is_double_quoted = TRUE; 109 }else if(arg[i] == ' '){ 110 option[j] = '\0'; 111 options(option); 112 j = 0; 113 }else{ 114 option[j++] = arg[i]; 115 } 116 } 117 if(j){ 118 option[j] = '\0'; 119 options(option); 120 } 121 return count; 122} 123 124/* 125 * call-seq: 126 * NKF.nkf(opt, str) => string 127 * 128 * Convert _str_ and return converted result. 129 * Conversion details are specified by _opt_ as String. 130 * 131 * require 'nkf' 132 * output = NKF.nkf("-s", input) 133 */ 134 135static VALUE 136rb_nkf_convert(VALUE obj, VALUE opt, VALUE src) 137{ 138 VALUE tmp; 139 reinit(); 140 StringValue(opt); 141 nkf_split_options(RSTRING_PTR(opt)); 142 if (!output_encoding) rb_raise(rb_eArgError, "no output encoding given"); 143 144 switch (nkf_enc_to_index(output_encoding)) { 145 case UTF_8_BOM: output_encoding = nkf_enc_from_index(UTF_8); break; 146 case UTF_16BE_BOM: output_encoding = nkf_enc_from_index(UTF_16BE); break; 147 case UTF_16LE_BOM: output_encoding = nkf_enc_from_index(UTF_16LE); break; 148 case UTF_32BE_BOM: output_encoding = nkf_enc_from_index(UTF_32BE); break; 149 case UTF_32LE_BOM: output_encoding = nkf_enc_from_index(UTF_32LE); break; 150 } 151 output_bom_f = FALSE; 152 153 incsize = INCSIZE; 154 155 input_ctr = 0; 156 StringValue(src); 157 input = (unsigned char *)RSTRING_PTR(src); 158 i_len = RSTRING_LENINT(src); 159 tmp = rb_str_new(0, i_len*3 + 10); 160 161 output_ctr = 0; 162 output = (unsigned char *)RSTRING_PTR(tmp); 163 o_len = RSTRING_LENINT(tmp); 164 *output = '\0'; 165 166 /* use _result_ begin*/ 167 result = tmp; 168 kanji_convert(NULL); 169 result = Qnil; 170 /* use _result_ end */ 171 172 rb_str_set_len(tmp, output_ctr); 173 OBJ_INFECT(tmp, src); 174 175 if (mimeout_f) 176 rb_enc_associate(tmp, rb_usascii_encoding()); 177 else 178 rb_enc_associate(tmp, rb_nkf_enc_get(nkf_enc_name(output_encoding))); 179 180 return tmp; 181} 182 183 184/* 185 * call-seq: 186 * NKF.guess(str) => encoding 187 * 188 * Returns guessed encoding of _str_ by nkf routine. 189 * 190 */ 191 192static VALUE 193rb_nkf_guess(VALUE obj, VALUE src) 194{ 195 reinit(); 196 197 input_ctr = 0; 198 StringValue(src); 199 input = (unsigned char *)RSTRING_PTR(src); 200 i_len = RSTRING_LENINT(src); 201 202 guess_f = TRUE; 203 kanji_convert( NULL ); 204 guess_f = FALSE; 205 206 return rb_enc_from_encoding(rb_nkf_enc_get(get_guessed_code())); 207} 208 209 210/* 211 * NKF - Ruby extension for Network Kanji Filter 212 * 213 * == Description 214 * 215 * This is a Ruby Extension version of nkf (Network Kanji Filter). 216 * It converts the first argument and returns converted result. Conversion 217 * details are specified by flags as the first argument. 218 * 219 * *Nkf* is a yet another kanji code converter among networks, hosts and terminals. 220 * It converts input kanji code to designated kanji code 221 * such as ISO-2022-JP, Shift_JIS, EUC-JP, UTF-8 or UTF-16. 222 * 223 * One of the most unique faculty of *nkf* is the guess of the input kanji encodings. 224 * It currently recognizes ISO-2022-JP, Shift_JIS, EUC-JP, UTF-8 and UTF-16. 225 * So users needn't set the input kanji code explicitly. 226 * 227 * By default, X0201 kana is converted into X0208 kana. 228 * For X0201 kana, SO/SI, SSO and ESC-(-I methods are supported. 229 * For automatic code detection, nkf assumes no X0201 kana in Shift_JIS. 230 * To accept X0201 in Shift_JIS, use <b>-X</b>, <b>-x</b> or <b>-S</b>. 231 * 232 * == Flags 233 * 234 * === -b -u 235 * 236 * Output is buffered (DEFAULT), Output is unbuffered. 237 * 238 * === -j -s -e -w -w16 -w32 239 * 240 * Output code is ISO-2022-JP (7bit JIS), Shift_JIS, EUC-JP, 241 * UTF-8N, UTF-16BE, UTF-32BE. 242 * Without this option and compile option, ISO-2022-JP is assumed. 243 * 244 * === -J -S -E -W -W16 -W32 245 * 246 * Input assumption is JIS 7 bit, Shift_JIS, EUC-JP, 247 * UTF-8, UTF-16, UTF-32. 248 * 249 * ==== -J 250 * 251 * Assume JIS input. It also accepts EUC-JP. 252 * This is the default. This flag does not exclude Shift_JIS. 253 * 254 * ==== -S 255 * 256 * Assume Shift_JIS and X0201 kana input. It also accepts JIS. 257 * EUC-JP is recognized as X0201 kana. Without <b>-x</b> flag, 258 * X0201 kana (halfwidth kana) is converted into X0208. 259 * 260 * ==== -E 261 * 262 * Assume EUC-JP input. It also accepts JIS. 263 * Same as -J. 264 * 265 * === -t 266 * 267 * No conversion. 268 * 269 * === -i_ 270 * 271 * Output sequence to designate JIS-kanji. (DEFAULT B) 272 * 273 * === -o_ 274 * 275 * Output sequence to designate ASCII. (DEFAULT B) 276 * 277 * === -r 278 * 279 * {de/en}crypt ROT13/47 280 * 281 * === -h[123] --hiragana --katakana --katakana-hiragana 282 * 283 * [-h1 --hiragana] Katakana to Hiragana conversion. 284 * 285 * [-h2 --katakana] Hiragana to Katakana conversion. 286 * 287 * [-h3 --katakana-hiragana] Katakana to Hiragana and Hiragana to Katakana conversion. 288 * 289 * === -T 290 * 291 * Text mode output (MS-DOS) 292 * 293 * === -l 294 * 295 * ISO8859-1 (Latin-1) support 296 * 297 * === -f[<code>m</code> [- <code>n</code>]] 298 * 299 * Folding on <code>m</code> length with <code>n</code> margin in a line. 300 * Without this option, fold length is 60 and fold margin is 10. 301 * 302 * === -F 303 * 304 * New line preserving line folding. 305 * 306 * === -Z[0-3] 307 * 308 * Convert X0208 alphabet (Fullwidth Alphabets) to ASCII. 309 * 310 * [-Z -Z0] Convert X0208 alphabet to ASCII. 311 * 312 * [-Z1] Converts X0208 kankaku to single ASCII space. 313 * 314 * [-Z2] Converts X0208 kankaku to double ASCII spaces. 315 * 316 * [-Z3] Replacing Fullwidth >, <, ", & into '>', '<', '"', '&' as in HTML. 317 * 318 * === -X -x 319 * 320 * Assume X0201 kana in MS-Kanji. 321 * With <b>-X</b> or without this option, X0201 is converted into X0208 Kana. 322 * With <b>-x</b>, try to preserve X0208 kana and do not convert X0201 kana to X0208. 323 * In JIS output, ESC-(-I is used. In EUC output, SSO is used. 324 * 325 * === -B[0-2] 326 * 327 * Assume broken JIS-Kanji input, which lost ESC. 328 * Useful when your site is using old B-News Nihongo patch. 329 * 330 * [-B1] allows any char after ESC-( or ESC-$. 331 * 332 * [-B2] forces ASCII after NL. 333 * 334 * === -I 335 * 336 * Replacing non iso-2022-jp char into a geta character 337 * (substitute character in Japanese). 338 * 339 * === -d -c 340 * 341 * Delete \r in line feed, Add \r in line feed. 342 * 343 * === -m[BQN0] 344 * 345 * MIME ISO-2022-JP/ISO8859-1 decode. (DEFAULT) 346 * To see ISO8859-1 (Latin-1) -l is necessary. 347 * 348 * [-mB] Decode MIME base64 encoded stream. Remove header or other part before 349 * conversion. 350 * 351 * [-mQ] Decode MIME quoted stream. '_' in quoted stream is converted to space. 352 * 353 * [-mN] Non-strict decoding. 354 * It allows line break in the middle of the base64 encoding. 355 * 356 * [-m0] No MIME decode. 357 * 358 * === -M 359 * 360 * MIME encode. Header style. All ASCII code and control characters are intact. 361 * Kanji conversion is performed before encoding, so this cannot be used as a picture encoder. 362 * 363 * [-MB] MIME encode Base64 stream. 364 * 365 * [-MQ] Perfome quoted encoding. 366 * 367 * === -l 368 * 369 * Input and output code is ISO8859-1 (Latin-1) and ISO-2022-JP. 370 * <b>-s</b>, <b>-e</b> and <b>-x</b> are not compatible with this option. 371 * 372 * === -L[uwm] 373 * 374 * new line mode 375 * Without this option, nkf doesn't convert line breaks. 376 * 377 * [-Lu] unix (LF) 378 * 379 * [-Lw] windows (CRLF) 380 * 381 * [-Lm] mac (CR) 382 * 383 * === --fj --unix --mac --msdos --windows 384 * 385 * convert for these system 386 * 387 * === --jis --euc --sjis --mime --base64 388 * 389 * convert for named code 390 * 391 * === --jis-input --euc-input --sjis-input --mime-input --base64-input 392 * 393 * assume input system 394 * 395 * === --ic=<code>input codeset</code> --oc=<code>output codeset</code> 396 * 397 * Set the input or output codeset. 398 * NKF supports following codesets and those codeset name are case insensitive. 399 * 400 * [ISO-2022-JP] a.k.a. RFC1468, 7bit JIS, JUNET 401 * 402 * [EUC-JP (eucJP-nkf)] a.k.a. AT&T JIS, Japanese EUC, UJIS 403 * 404 * [eucJP-ascii] a.k.a. x-eucjp-open-19970715-ascii 405 * 406 * [eucJP-ms] a.k.a. x-eucjp-open-19970715-ms 407 * 408 * [CP51932] Microsoft Version of EUC-JP. 409 * 410 * [Shift_JIS] SJIS, MS-Kanji 411 * 412 * [Windows-31J] a.k.a. CP932 413 * 414 * [UTF-8] same as UTF-8N 415 * 416 * [UTF-8N] UTF-8 without BOM 417 * 418 * [UTF-8-BOM] UTF-8 with BOM 419 * 420 * [UTF-16] same as UTF-16BE 421 * 422 * [UTF-16BE] UTF-16 Big Endian without BOM 423 * 424 * [UTF-16BE-BOM] UTF-16 Big Endian with BOM 425 * 426 * [UTF-16LE] UTF-16 Little Endian without BOM 427 * 428 * [UTF-16LE-BOM] UTF-16 Little Endian with BOM 429 * 430 * [UTF-32] same as UTF-32BE 431 * 432 * [UTF-32BE] UTF-32 Big Endian without BOM 433 * 434 * [UTF-32BE-BOM] UTF-32 Big Endian with BOM 435 * 436 * [UTF-32LE] UTF-32 Little Endian without BOM 437 * 438 * [UTF-32LE-BOM] UTF-32 Little Endian with BOM 439 * 440 * [UTF8-MAC] NKDed UTF-8, a.k.a. UTF8-NFD (input only) 441 * 442 * === --fb-{skip, html, xml, perl, java, subchar} 443 * 444 * Specify the way that nkf handles unassigned characters. 445 * Without this option, --fb-skip is assumed. 446 * 447 * === --prefix= <code>escape character</code> <code>target character</code> .. 448 * 449 * When nkf converts to Shift_JIS, 450 * nkf adds a specified escape character to specified 2nd byte of Shift_JIS characters. 451 * 1st byte of argument is the escape character and following bytes are target characters. 452 * 453 * === --no-cp932ext 454 * 455 * Handle the characters extended in CP932 as unassigned characters. 456 * 457 * == --no-best-fit-chars 458 * 459 * When Unicode to Encoded byte conversion, 460 * don't convert characters which is not round trip safe. 461 * When Unicode to Unicode conversion, 462 * with this and -x option, nkf can be used as UTF converter. 463 * (In other words, without this and -x option, nkf doesn't save some characters) 464 * 465 * When nkf convert string which related to path, you should use this opion. 466 * 467 * === --cap-input 468 * 469 * Decode hex encoded characters. 470 * 471 * === --url-input 472 * 473 * Unescape percent escaped characters. 474 * 475 * === -- 476 * 477 * Ignore rest of -option. 478 */ 479 480void 481Init_nkf() 482{ 483 VALUE mNKF = rb_define_module("NKF"); 484 485 rb_define_module_function(mNKF, "nkf", rb_nkf_convert, 2); 486 rb_define_module_function(mNKF, "guess", rb_nkf_guess, 1); 487 rb_define_alias(rb_singleton_class(mNKF), "guess", "guess"); 488 489 rb_define_const(mNKF, "AUTO", Qnil); 490 rb_define_const(mNKF, "NOCONV", Qnil); 491 rb_define_const(mNKF, "UNKNOWN", Qnil); 492 rb_define_const(mNKF, "BINARY", rb_enc_from_encoding(rb_nkf_enc_get("BINARY"))); 493 rb_define_const(mNKF, "ASCII", rb_enc_from_encoding(rb_nkf_enc_get("US-ASCII"))); 494 rb_define_const(mNKF, "JIS", rb_enc_from_encoding(rb_nkf_enc_get("ISO-2022-JP"))); 495 rb_define_const(mNKF, "EUC", rb_enc_from_encoding(rb_nkf_enc_get("EUC-JP"))); 496 rb_define_const(mNKF, "SJIS", rb_enc_from_encoding(rb_nkf_enc_get("Shift_JIS"))); 497 rb_define_const(mNKF, "UTF8", rb_enc_from_encoding(rb_utf8_encoding())); 498 rb_define_const(mNKF, "UTF16", rb_enc_from_encoding(rb_nkf_enc_get("UTF-16BE"))); 499 rb_define_const(mNKF, "UTF32", rb_enc_from_encoding(rb_nkf_enc_get("UTF-32BE"))); 500 501 /* Full version string of nkf */ 502 rb_define_const(mNKF, "VERSION", rb_str_new2(RUBY_NKF_VERSION)); 503 /* Version of nkf */ 504 rb_define_const(mNKF, "NKF_VERSION", rb_str_new2(NKF_VERSION)); 505 /* Release date of nkf */ 506 rb_define_const(mNKF, "NKF_RELEASE_DATE", rb_str_new2(NKF_RELEASE_DATE)); 507} 508