1/********************************************************************** 2 3 encoding.h - 4 5 $Author: matz $ 6 created at: Thu May 24 11:49:41 JST 2007 7 8 Copyright (C) 2007 Yukihiro Matsumoto 9 10**********************************************************************/ 11 12#ifndef RUBY_ENCODING_H 13#define RUBY_ENCODING_H 1 14 15#if defined(__cplusplus) 16extern "C" { 17#if 0 18} /* satisfy cc-mode */ 19#endif 20#endif 21 22#include <stdarg.h> 23#include "ruby/oniguruma.h" 24 25#if defined __GNUC__ && __GNUC__ >= 4 26#pragma GCC visibility push(default) 27#endif 28 29#define ENCODING_INLINE_MAX 1023 30#define ENCODING_SHIFT (FL_USHIFT+10) 31#define ENCODING_MASK (((VALUE)ENCODING_INLINE_MAX)<<ENCODING_SHIFT) 32 33#define ENCODING_SET_INLINED(obj,i) do {\ 34 RBASIC(obj)->flags &= ~ENCODING_MASK;\ 35 RBASIC(obj)->flags |= (VALUE)(i) << ENCODING_SHIFT;\ 36} while (0) 37#define ENCODING_SET(obj,i) do {\ 38 VALUE rb_encoding_set_obj = (obj); \ 39 int encoding_set_enc_index = (i); \ 40 if (encoding_set_enc_index < ENCODING_INLINE_MAX) \ 41 ENCODING_SET_INLINED(rb_encoding_set_obj, encoding_set_enc_index); \ 42 else \ 43 rb_enc_set_index(rb_encoding_set_obj, encoding_set_enc_index); \ 44} while (0) 45 46#define ENCODING_GET_INLINED(obj) (int)((RBASIC(obj)->flags & ENCODING_MASK)>>ENCODING_SHIFT) 47#define ENCODING_GET(obj) \ 48 (ENCODING_GET_INLINED(obj) != ENCODING_INLINE_MAX ? \ 49 ENCODING_GET_INLINED(obj) : \ 50 rb_enc_get_index(obj)) 51 52#define ENCODING_IS_ASCII8BIT(obj) (ENCODING_GET_INLINED(obj) == 0) 53 54#define ENCODING_MAXNAMELEN 42 55 56#define ENC_CODERANGE_MASK ((int)(FL_USER8|FL_USER9)) 57#define ENC_CODERANGE_UNKNOWN 0 58#define ENC_CODERANGE_7BIT ((int)FL_USER8) 59#define ENC_CODERANGE_VALID ((int)FL_USER9) 60#define ENC_CODERANGE_BROKEN ((int)(FL_USER8|FL_USER9)) 61#define ENC_CODERANGE(obj) ((int)RBASIC(obj)->flags & ENC_CODERANGE_MASK) 62#define ENC_CODERANGE_ASCIIONLY(obj) (ENC_CODERANGE(obj) == ENC_CODERANGE_7BIT) 63#define ENC_CODERANGE_SET(obj,cr) (RBASIC(obj)->flags = \ 64 (RBASIC(obj)->flags & ~ENC_CODERANGE_MASK) | (cr)) 65#define ENC_CODERANGE_CLEAR(obj) ENC_CODERANGE_SET((obj),0) 66 67/* assumed ASCII compatibility */ 68#define ENC_CODERANGE_AND(a, b) \ 69 ((a) == ENC_CODERANGE_7BIT ? (b) : \ 70 (a) == ENC_CODERANGE_VALID ? ((b) == ENC_CODERANGE_7BIT ? ENC_CODERANGE_VALID : (b)) : \ 71 ENC_CODERANGE_UNKNOWN) 72 73#define ENCODING_CODERANGE_SET(obj, encindex, cr) \ 74 do { \ 75 VALUE rb_encoding_coderange_obj = (obj); \ 76 ENCODING_SET(rb_encoding_coderange_obj, (encindex)); \ 77 ENC_CODERANGE_SET(rb_encoding_coderange_obj, (cr)); \ 78 } while (0) 79 80typedef OnigEncodingType rb_encoding; 81 82int rb_char_to_option_kcode(int c, int *option, int *kcode); 83 84int rb_enc_replicate(const char *, rb_encoding *); 85int rb_define_dummy_encoding(const char *); 86#define rb_enc_to_index(enc) ((enc) ? ENC_TO_ENCINDEX(enc) : 0) 87int rb_enc_get_index(VALUE obj); 88void rb_enc_set_index(VALUE obj, int encindex); 89int rb_enc_find_index(const char *name); 90int rb_to_encoding_index(VALUE); 91rb_encoding* rb_to_encoding(VALUE); 92rb_encoding* rb_find_encoding(VALUE); 93rb_encoding* rb_enc_get(VALUE); 94rb_encoding* rb_enc_compatible(VALUE,VALUE); 95rb_encoding* rb_enc_check(VALUE,VALUE); 96VALUE rb_enc_associate_index(VALUE, int); 97VALUE rb_enc_associate(VALUE, rb_encoding*); 98void rb_enc_copy(VALUE dst, VALUE src); 99 100VALUE rb_enc_str_new(const char*, long, rb_encoding*); 101VALUE rb_enc_reg_new(const char*, long, rb_encoding*, int); 102PRINTF_ARGS(VALUE rb_enc_sprintf(rb_encoding *, const char*, ...), 2, 3); 103VALUE rb_enc_vsprintf(rb_encoding *, const char*, va_list); 104long rb_enc_strlen(const char*, const char*, rb_encoding*); 105char* rb_enc_nth(const char*, const char*, long, rb_encoding*); 106VALUE rb_obj_encoding(VALUE); 107VALUE rb_enc_str_buf_cat(VALUE str, const char *ptr, long len, rb_encoding *enc); 108VALUE rb_enc_uint_chr(unsigned int code, rb_encoding *enc); 109 110VALUE rb_external_str_new_with_enc(const char *ptr, long len, rb_encoding *); 111VALUE rb_str_export_to_enc(VALUE, rb_encoding *); 112VALUE rb_str_conv_enc(VALUE str, rb_encoding *from, rb_encoding *to); 113VALUE rb_str_conv_enc_opts(VALUE str, rb_encoding *from, rb_encoding *to, int ecflags, VALUE ecopts); 114 115PRINTF_ARGS(NORETURN(void rb_enc_raise(rb_encoding *, VALUE, const char*, ...)), 3, 4); 116 117/* index -> rb_encoding */ 118rb_encoding* rb_enc_from_index(int idx); 119 120/* name -> rb_encoding */ 121rb_encoding * rb_enc_find(const char *name); 122 123/* rb_encoding * -> name */ 124#define rb_enc_name(enc) (enc)->name 125 126/* rb_encoding * -> minlen/maxlen */ 127#define rb_enc_mbminlen(enc) (enc)->min_enc_len 128#define rb_enc_mbmaxlen(enc) (enc)->max_enc_len 129 130/* -> mbclen (no error notification: 0 < ret <= e-p, no exception) */ 131int rb_enc_mbclen(const char *p, const char *e, rb_encoding *enc); 132 133/* -> mbclen (only for valid encoding) */ 134int rb_enc_fast_mbclen(const char *p, const char *e, rb_encoding *enc); 135 136/* -> chlen, invalid or needmore */ 137int rb_enc_precise_mbclen(const char *p, const char *e, rb_encoding *enc); 138#define MBCLEN_CHARFOUND_P(ret) ONIGENC_MBCLEN_CHARFOUND_P(ret) 139#define MBCLEN_CHARFOUND_LEN(ret) ONIGENC_MBCLEN_CHARFOUND_LEN(ret) 140#define MBCLEN_INVALID_P(ret) ONIGENC_MBCLEN_INVALID_P(ret) 141#define MBCLEN_NEEDMORE_P(ret) ONIGENC_MBCLEN_NEEDMORE_P(ret) 142#define MBCLEN_NEEDMORE_LEN(ret) ONIGENC_MBCLEN_NEEDMORE_LEN(ret) 143 144/* -> 0x00..0x7f, -1 */ 145int rb_enc_ascget(const char *p, const char *e, int *len, rb_encoding *enc); 146 147 148/* -> code (and len) or raise exception */ 149unsigned int rb_enc_codepoint_len(const char *p, const char *e, int *len, rb_encoding *enc); 150 151/* prototype for obsolete function */ 152unsigned int rb_enc_codepoint(const char *p, const char *e, rb_encoding *enc); 153/* overriding macro */ 154#define rb_enc_codepoint(p,e,enc) rb_enc_codepoint_len((p),(e),0,(enc)) 155#define rb_enc_mbc_to_codepoint(p, e, enc) ONIGENC_MBC_TO_CODE((enc),(UChar*)(p),(UChar*)(e)) 156 157/* -> codelen>0 or raise exception */ 158int rb_enc_codelen(int code, rb_encoding *enc); 159 160/* code,ptr,encoding -> write buf */ 161#define rb_enc_mbcput(c,buf,enc) ONIGENC_CODE_TO_MBC((enc),(c),(UChar*)(buf)) 162 163/* start, ptr, end, encoding -> prev_char */ 164#define rb_enc_prev_char(s,p,e,enc) ((char *)onigenc_get_prev_char_head((enc),(UChar*)(s),(UChar*)(p),(UChar*)(e))) 165/* start, ptr, end, encoding -> next_char */ 166#define rb_enc_left_char_head(s,p,e,enc) ((char *)onigenc_get_left_adjust_char_head((enc),(UChar*)(s),(UChar*)(p),(UChar*)(e))) 167#define rb_enc_right_char_head(s,p,e,enc) ((char *)onigenc_get_right_adjust_char_head((enc),(UChar*)(s),(UChar*)(p),(UChar*)(e))) 168#define rb_enc_step_back(s,p,e,n,enc) ((char *)onigenc_step_back((enc),(UChar*)(s),(UChar*)(p),(UChar*)(e),(int)(n))) 169 170/* ptr, ptr, encoding -> newline_or_not */ 171#define rb_enc_is_newline(p,end,enc) ONIGENC_IS_MBC_NEWLINE((enc),(UChar*)(p),(UChar*)(end)) 172 173#define rb_enc_isctype(c,t,enc) ONIGENC_IS_CODE_CTYPE((enc),(c),(t)) 174#define rb_enc_isascii(c,enc) ONIGENC_IS_CODE_ASCII(c) 175#define rb_enc_isalpha(c,enc) ONIGENC_IS_CODE_ALPHA((enc),(c)) 176#define rb_enc_islower(c,enc) ONIGENC_IS_CODE_LOWER((enc),(c)) 177#define rb_enc_isupper(c,enc) ONIGENC_IS_CODE_UPPER((enc),(c)) 178#define rb_enc_ispunct(c,enc) ONIGENC_IS_CODE_PUNCT((enc),(c)) 179#define rb_enc_isalnum(c,enc) ONIGENC_IS_CODE_ALNUM((enc),(c)) 180#define rb_enc_isprint(c,enc) ONIGENC_IS_CODE_PRINT((enc),(c)) 181#define rb_enc_isspace(c,enc) ONIGENC_IS_CODE_SPACE((enc),(c)) 182#define rb_enc_isdigit(c,enc) ONIGENC_IS_CODE_DIGIT((enc),(c)) 183 184#define rb_enc_asciicompat(enc) (rb_enc_mbminlen(enc)==1 && !rb_enc_dummy_p(enc)) 185 186int rb_enc_casefold(char *to, const char *p, const char *e, rb_encoding *enc); 187int rb_enc_toupper(int c, rb_encoding *enc); 188int rb_enc_tolower(int c, rb_encoding *enc); 189ID rb_intern3(const char*, long, rb_encoding*); 190ID rb_interned_id_p(const char *, long, rb_encoding *); 191int rb_enc_symname_p(const char*, rb_encoding*); 192int rb_enc_symname2_p(const char*, long, rb_encoding*); 193int rb_enc_str_coderange(VALUE); 194long rb_str_coderange_scan_restartable(const char*, const char*, rb_encoding*, int*); 195int rb_enc_str_asciionly_p(VALUE); 196#define rb_enc_str_asciicompat_p(str) rb_enc_asciicompat(rb_enc_get(str)) 197VALUE rb_enc_from_encoding(rb_encoding *enc); 198int rb_enc_unicode_p(rb_encoding *enc); 199rb_encoding *rb_ascii8bit_encoding(void); 200rb_encoding *rb_utf8_encoding(void); 201rb_encoding *rb_usascii_encoding(void); 202rb_encoding *rb_locale_encoding(void); 203rb_encoding *rb_filesystem_encoding(void); 204rb_encoding *rb_default_external_encoding(void); 205rb_encoding *rb_default_internal_encoding(void); 206int rb_ascii8bit_encindex(void); 207int rb_utf8_encindex(void); 208int rb_usascii_encindex(void); 209int rb_locale_encindex(void); 210int rb_filesystem_encindex(void); 211VALUE rb_enc_default_external(void); 212VALUE rb_enc_default_internal(void); 213void rb_enc_set_default_external(VALUE encoding); 214void rb_enc_set_default_internal(VALUE encoding); 215VALUE rb_locale_charmap(VALUE klass); 216long rb_memsearch(const void*,long,const void*,long,rb_encoding*); 217char *rb_enc_path_next(const char *,const char *,rb_encoding*); 218char *rb_enc_path_skip_prefix(const char *,const char *,rb_encoding*); 219char *rb_enc_path_last_separator(const char *,const char *,rb_encoding*); 220char *rb_enc_path_end(const char *,const char *,rb_encoding*); 221const char *ruby_enc_find_basename(const char *name, long *baselen, long *alllen, rb_encoding *enc); 222const char *ruby_enc_find_extname(const char *name, long *len, rb_encoding *enc); 223ID rb_check_id_cstr(const char *ptr, long len, rb_encoding *enc); 224 225RUBY_EXTERN VALUE rb_cEncoding; 226#define ENC_DUMMY_FLAG (1<<24) 227#define ENC_INDEX_MASK (~(~0U<<24)) 228 229#define ENC_TO_ENCINDEX(enc) (int)((enc)->ruby_encoding_index & ENC_INDEX_MASK) 230 231#define ENC_DUMMY_P(enc) ((enc)->ruby_encoding_index & ENC_DUMMY_FLAG) 232#define ENC_SET_DUMMY(enc) ((enc)->ruby_encoding_index |= ENC_DUMMY_FLAG) 233 234static inline int 235rb_enc_dummy_p(rb_encoding *enc) 236{ 237 return ENC_DUMMY_P(enc) != 0; 238} 239 240/* econv stuff */ 241 242typedef enum { 243 econv_invalid_byte_sequence, 244 econv_undefined_conversion, 245 econv_destination_buffer_full, 246 econv_source_buffer_empty, 247 econv_finished, 248 econv_after_output, 249 econv_incomplete_input 250} rb_econv_result_t; 251 252typedef struct rb_econv_t rb_econv_t; 253 254VALUE rb_str_encode(VALUE str, VALUE to, int ecflags, VALUE ecopts); 255int rb_econv_has_convpath_p(const char* from_encoding, const char* to_encoding); 256 257int rb_econv_prepare_options(VALUE opthash, VALUE *ecopts, int ecflags); 258int rb_econv_prepare_opts(VALUE opthash, VALUE *ecopts); 259 260rb_econv_t *rb_econv_open(const char *source_encoding, const char *destination_encoding, int ecflags); 261rb_econv_t *rb_econv_open_opts(const char *source_encoding, const char *destination_encoding, int ecflags, VALUE ecopts); 262 263rb_econv_result_t rb_econv_convert(rb_econv_t *ec, 264 const unsigned char **source_buffer_ptr, const unsigned char *source_buffer_end, 265 unsigned char **destination_buffer_ptr, unsigned char *destination_buffer_end, 266 int flags); 267void rb_econv_close(rb_econv_t *ec); 268 269/* result: 0:success -1:failure */ 270int rb_econv_set_replacement(rb_econv_t *ec, const unsigned char *str, size_t len, const char *encname); 271 272/* result: 0:success -1:failure */ 273int rb_econv_decorate_at_first(rb_econv_t *ec, const char *decorator_name); 274int rb_econv_decorate_at_last(rb_econv_t *ec, const char *decorator_name); 275 276VALUE rb_econv_open_exc(const char *senc, const char *denc, int ecflags); 277 278/* result: 0:success -1:failure */ 279int rb_econv_insert_output(rb_econv_t *ec, 280 const unsigned char *str, size_t len, const char *str_encoding); 281 282/* encoding that rb_econv_insert_output doesn't need conversion */ 283const char *rb_econv_encoding_to_insert_output(rb_econv_t *ec); 284 285/* raise an error if the last rb_econv_convert is error */ 286void rb_econv_check_error(rb_econv_t *ec); 287 288/* returns an exception object or nil */ 289VALUE rb_econv_make_exception(rb_econv_t *ec); 290 291int rb_econv_putbackable(rb_econv_t *ec); 292void rb_econv_putback(rb_econv_t *ec, unsigned char *p, int n); 293 294/* returns the corresponding ASCII compatible encoding for encname, 295 * or NULL if encname is not ASCII incompatible encoding. */ 296const char *rb_econv_asciicompat_encoding(const char *encname); 297 298VALUE rb_econv_str_convert(rb_econv_t *ec, VALUE src, int flags); 299VALUE rb_econv_substr_convert(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, int flags); 300VALUE rb_econv_str_append(rb_econv_t *ec, VALUE src, VALUE dst, int flags); 301VALUE rb_econv_substr_append(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, VALUE dst, int flags); 302 303void rb_econv_binmode(rb_econv_t *ec); 304 305/* flags for rb_econv_open */ 306 307#define ECONV_ERROR_HANDLER_MASK 0x000000ff 308 309#define ECONV_INVALID_MASK 0x0000000f 310#define ECONV_INVALID_REPLACE 0x00000002 311 312#define ECONV_UNDEF_MASK 0x000000f0 313#define ECONV_UNDEF_REPLACE 0x00000020 314#define ECONV_UNDEF_HEX_CHARREF 0x00000030 315 316#define ECONV_DECORATOR_MASK 0x0000ff00 317#define ECONV_NEWLINE_DECORATOR_MASK 0x00003f00 318#define ECONV_NEWLINE_DECORATOR_READ_MASK 0x00000f00 319#define ECONV_NEWLINE_DECORATOR_WRITE_MASK 0x00003000 320 321#define ECONV_UNIVERSAL_NEWLINE_DECORATOR 0x00000100 322#define ECONV_CRLF_NEWLINE_DECORATOR 0x00001000 323#define ECONV_CR_NEWLINE_DECORATOR 0x00002000 324#define ECONV_XML_TEXT_DECORATOR 0x00004000 325#define ECONV_XML_ATTR_CONTENT_DECORATOR 0x00008000 326 327#define ECONV_STATEFUL_DECORATOR_MASK 0x00f00000 328#define ECONV_XML_ATTR_QUOTE_DECORATOR 0x00100000 329 330#if defined(RUBY_TEST_CRLF_ENVIRONMENT) || defined(_WIN32) 331#define ECONV_DEFAULT_NEWLINE_DECORATOR ECONV_CRLF_NEWLINE_DECORATOR 332#else 333#define ECONV_DEFAULT_NEWLINE_DECORATOR 0 334#endif 335 336/* end of flags for rb_econv_open */ 337 338/* flags for rb_econv_convert */ 339#define ECONV_PARTIAL_INPUT 0x00010000 340#define ECONV_AFTER_OUTPUT 0x00020000 341/* end of flags for rb_econv_convert */ 342 343#if defined __GNUC__ && __GNUC__ >= 4 344#pragma GCC visibility pop 345#endif 346 347#if defined(__cplusplus) 348#if 0 349{ /* satisfy cc-mode */ 350#endif 351} /* extern "C" { */ 352#endif 353 354#endif /* RUBY_ENCODING_H */ 355