1/// \file 2/// Default implementation of CommonTokenStream 3/// 4 5// [The "BSD licence"] 6// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC 7// http://www.temporal-wave.com 8// http://www.linkedin.com/in/jimidle 9// 10// All rights reserved. 11// 12// Redistribution and use in source and binary forms, with or without 13// modification, are permitted provided that the following conditions 14// are met: 15// 1. Redistributions of source code must retain the above copyright 16// notice, this list of conditions and the following disclaimer. 17// 2. Redistributions in binary form must reproduce the above copyright 18// notice, this list of conditions and the following disclaimer in the 19// documentation and/or other materials provided with the distribution. 20// 3. The name of the author may not be used to endorse or promote products 21// derived from this software without specific prior written permission. 22// 23// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 34#include <antlr3tokenstream.h> 35 36#ifdef ANTLR3_WINDOWS 37#pragma warning( disable : 4100 ) 38#endif 39 40// COMMON_TOKEN_STREAM API 41// 42static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel); 43static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype); 44static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard); 45static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts); 46static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); 47static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types); 48static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list); 49static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type); 50 51// TOKEN_STREAM API 52// 53static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); 54static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); 55static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i); 56static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts); 57static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource); 58static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts); 59static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); 60static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop); 61static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger); 62 63// INT STREAM API 64// 65static void consume (pANTLR3_INT_STREAM is); 66static void dbgConsume (pANTLR3_INT_STREAM is); 67static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); 68static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); 69static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is); 70static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is); 71static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark); 72static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is); 73static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is); 74static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); 75static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); 76static void rewindLast (pANTLR3_INT_STREAM is); 77static void dbgRewindLast (pANTLR3_INT_STREAM is); 78static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); 79static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); 80static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is); 81static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream); 82static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream); 83 84// Helpers 85// 86static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream); 87static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); 88static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); 89static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); 90 91ANTLR3_API pANTLR3_TOKEN_STREAM 92antlr3TokenStreamNew() 93{ 94 pANTLR3_TOKEN_STREAM stream; 95 96 // Memory for the interface structure 97 // 98 stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM)); 99 100 if (stream == NULL) 101 { 102 return NULL; 103 } 104 105 // Install basic API 106 // 107 stream->free = antlr3TokenStreamFree; 108 109 110 return stream; 111} 112 113static void 114antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream) 115{ 116 ANTLR3_FREE(stream); 117} 118 119static void 120antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream) 121{ 122 // We only free up our subordinate interfaces if they belong 123 // to us, otherwise we let whoever owns them deal with them. 124 // 125 if (stream->tstream->super == stream) 126 { 127 if (stream->tstream->istream->super == stream->tstream) 128 { 129 stream->tstream->istream->free(stream->tstream->istream); 130 stream->tstream->istream = NULL; 131 } 132 stream->tstream->free(stream->tstream); 133 } 134 135 // Now we free our own resources 136 // 137 if (stream->tokens != NULL) 138 { 139 stream->tokens->free(stream->tokens); 140 stream->tokens = NULL; 141 } 142 if (stream->discardSet != NULL) 143 { 144 stream->discardSet->free(stream->discardSet); 145 stream->discardSet = NULL; 146 } 147 if (stream->channelOverrides != NULL) 148 { 149 stream->channelOverrides->free(stream->channelOverrides); 150 stream->channelOverrides = NULL; 151 } 152 153 // Free our memory now 154 // 155 ANTLR3_FREE(stream); 156} 157 158ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM 159antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger) 160{ 161 pANTLR3_COMMON_TOKEN_STREAM stream; 162 163 // Create a standard token stream 164 // 165 stream = antlr3CommonTokenStreamSourceNew(hint, source); 166 167 // Install the debugger object 168 // 169 stream->tstream->debugger = debugger; 170 171 // Override standard token stream methods with debugging versions 172 // 173 stream->tstream->initialStreamState = ANTLR3_FALSE; 174 175 stream->tstream->_LT = dbgTokLT; 176 177 stream->tstream->istream->consume = dbgConsume; 178 stream->tstream->istream->_LA = dbgLA; 179 stream->tstream->istream->mark = dbgMark; 180 stream->tstream->istream->rewind = dbgRewindStream; 181 stream->tstream->istream->rewindLast = dbgRewindLast; 182 stream->tstream->istream->seek = dbgSeek; 183 184 return stream; 185} 186 187ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM 188antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source) 189{ 190 pANTLR3_COMMON_TOKEN_STREAM stream; 191 192 stream = antlr3CommonTokenStreamNew(hint); 193 194 stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; 195 196 stream->channelOverrides = NULL; 197 stream->discardSet = NULL; 198 stream->discardOffChannel = ANTLR3_FALSE; 199 200 stream->tstream->setTokenSource(stream->tstream, source); 201 202 stream->free = antlr3CTSFree; 203 return stream; 204} 205 206ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM 207antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint) 208{ 209 pANTLR3_COMMON_TOKEN_STREAM stream; 210 211 /* Memory for the interface structure 212 */ 213 stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM)); 214 215 if (stream == NULL) 216 { 217 return NULL; 218 } 219 220 /* Create space for the token stream interface 221 */ 222 stream->tstream = antlr3TokenStreamNew(); 223 stream->tstream->super = stream; 224 225 /* Create space for the INT_STREAM interfacce 226 */ 227 stream->tstream->istream = antlr3IntStreamNew(); 228 stream->tstream->istream->super = (stream->tstream); 229 stream->tstream->istream->type = ANTLR3_TOKENSTREAM; 230 231 /* Install the token tracking tables 232 */ 233 stream->tokens = antlr3VectorNew(0); 234 235 /* Defaults 236 */ 237 stream->p = -1; 238 239 /* Install the common token stream API 240 */ 241 stream->setTokenTypeChannel = setTokenTypeChannel; 242 stream->discardTokenType = discardTokenType; 243 stream->discardOffChannelToks = discardOffChannel; 244 stream->getTokens = getTokens; 245 stream->getTokenRange = getTokenRange; 246 stream->getTokensSet = getTokensSet; 247 stream->getTokensList = getTokensList; 248 stream->getTokensType = getTokensType; 249 250 /* Install the token stream API 251 */ 252 stream->tstream->_LT = tokLT; 253 stream->tstream->get = get; 254 stream->tstream->getTokenSource = getTokenSource; 255 stream->tstream->setTokenSource = setTokenSource; 256 stream->tstream->toString = toString; 257 stream->tstream->toStringSS = toStringSS; 258 stream->tstream->toStringTT = toStringTT; 259 stream->tstream->setDebugListener = setDebugListener; 260 261 /* Install INT_STREAM interface 262 */ 263 stream->tstream->istream->_LA = _LA; 264 stream->tstream->istream->mark = mark; 265 stream->tstream->istream->release = release; 266 stream->tstream->istream->size = size; 267 stream->tstream->istream->index = tindex; 268 stream->tstream->istream->rewind = rewindStream; 269 stream->tstream->istream->rewindLast= rewindLast; 270 stream->tstream->istream->seek = seek; 271 stream->tstream->istream->consume = consume; 272 stream->tstream->istream->getSourceName = getSourceName; 273 274 return stream; 275} 276 277// Install a debug listener adn switch to debug mode methods 278// 279static void 280setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger) 281{ 282 // Install the debugger object 283 // 284 ts->debugger = debugger; 285 286 // Override standard token stream methods with debugging versions 287 // 288 ts->initialStreamState = ANTLR3_FALSE; 289 290 ts->_LT = dbgTokLT; 291 292 ts->istream->consume = dbgConsume; 293 ts->istream->_LA = dbgLA; 294 ts->istream->mark = dbgMark; 295 ts->istream->rewind = dbgRewindStream; 296 ts->istream->rewindLast = dbgRewindLast; 297 ts->istream->seek = dbgSeek; 298} 299 300/** Get the ith token from the current position 1..n where k=1 is the 301* first symbol of lookahead. 302*/ 303static pANTLR3_COMMON_TOKEN 304tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) 305{ 306 ANTLR3_INT32 i; 307 ANTLR3_INT32 n; 308 pANTLR3_COMMON_TOKEN_STREAM cts; 309 310 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; 311 312 if (k < 0) 313 { 314 return LB(cts, -k); 315 } 316 317 if (cts->p == -1) 318 { 319 fillBuffer(cts); 320 } 321 if (k == 0) 322 { 323 return NULL; 324 } 325 326 if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize) 327 { 328 pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); 329 330 teof->setStartIndex (teof, ts->istream->index (ts->istream)); 331 teof->setStopIndex (teof, ts->istream->index (ts->istream)); 332 return teof; 333 } 334 335 i = cts->p; 336 n = 1; 337 338 /* Need to find k good tokens, skipping ones that are off channel 339 */ 340 while ( n < k) 341 { 342 /* Skip off-channel tokens */ 343 i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */ 344 n++; 345 } 346 if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize) 347 { 348 pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); 349 350 teof->setStartIndex (teof, ts->istream->index(ts->istream)); 351 teof->setStopIndex (teof, ts->istream->index(ts->istream)); 352 return teof; 353 } 354 355 // Here the token must be in the input vector. Rather then incut 356 // function call penalty, we jsut return the pointer directly 357 // from the vector 358 // 359 return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; 360 //return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i); 361} 362 363/// Debug only method to flag consumption of initial off-channel 364/// tokens in the input stream 365/// 366static void 367consumeInitialHiddenTokens(pANTLR3_INT_STREAM is) 368{ 369 ANTLR3_MARKER first; 370 ANTLR3_INT32 i; 371 pANTLR3_TOKEN_STREAM ts; 372 373 ts = (pANTLR3_TOKEN_STREAM) is->super; 374 first = is->index(is); 375 376 for (i=0; i<first; i++) 377 { 378 ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i)); 379 } 380 381 ts->initialStreamState = ANTLR3_FALSE; 382 383} 384 385/// As per the normal tokLT but sends information to the debugger 386/// 387static pANTLR3_COMMON_TOKEN 388dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) 389{ 390 if (ts->initialStreamState == ANTLR3_TRUE) 391 { 392 consumeInitialHiddenTokens(ts->istream); 393 } 394 return tokLT(ts, k); 395} 396 397#ifdef ANTLR3_WINDOWS 398 /* When fully optimized VC7 complains about non reachable code. 399 * Not yet sure if this is an optimizer bug, or a bug in the flow analysis 400 */ 401#pragma warning( disable : 4702 ) 402#endif 403 404static pANTLR3_COMMON_TOKEN 405LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k) 406{ 407 ANTLR3_INT32 i; 408 ANTLR3_INT32 n; 409 410 if (cts->p == -1) 411 { 412 fillBuffer(cts); 413 } 414 if (k == 0) 415 { 416 return NULL; 417 } 418 if ((cts->p - k) < 0) 419 { 420 return NULL; 421 } 422 423 i = cts->p; 424 n = 1; 425 426 /* Need to find k good tokens, going backwards, skipping ones that are off channel 427 */ 428 while (n <= (ANTLR3_INT32) k) 429 { 430 /* Skip off-channel tokens 431 */ 432 433 i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */ 434 n++; 435 } 436 if (i < 0) 437 { 438 return NULL; 439 } 440 // Here the token must be in the input vector. Rather then incut 441 // function call penalty, we jsut return the pointer directly 442 // from the vector 443 // 444 return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; 445} 446 447static pANTLR3_COMMON_TOKEN 448get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i) 449{ 450 pANTLR3_COMMON_TOKEN_STREAM cts; 451 452 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; 453 454 return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */ 455} 456 457static pANTLR3_TOKEN_SOURCE 458getTokenSource (pANTLR3_TOKEN_STREAM ts) 459{ 460 return ts->tokenSource; 461} 462 463static void 464setTokenSource ( pANTLR3_TOKEN_STREAM ts, 465 pANTLR3_TOKEN_SOURCE tokenSource) 466{ 467 ts->tokenSource = tokenSource; 468} 469 470static pANTLR3_STRING 471toString (pANTLR3_TOKEN_STREAM ts) 472{ 473 pANTLR3_COMMON_TOKEN_STREAM cts; 474 475 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; 476 477 if (cts->p == -1) 478 { 479 fillBuffer(cts); 480 } 481 482 return ts->toStringSS(ts, 0, ts->istream->size(ts->istream)); 483} 484 485static pANTLR3_STRING 486toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) 487{ 488 pANTLR3_STRING string; 489 pANTLR3_TOKEN_SOURCE tsource; 490 pANTLR3_COMMON_TOKEN tok; 491 ANTLR3_UINT32 i; 492 pANTLR3_COMMON_TOKEN_STREAM cts; 493 494 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 495 496 if (cts->p == -1) 497 { 498 fillBuffer(cts); 499 } 500 if (stop >= ts->istream->size(ts->istream)) 501 { 502 stop = ts->istream->size(ts->istream) - 1; 503 } 504 505 /* Who is giving us these tokens? 506 */ 507 tsource = ts->getTokenSource(ts); 508 509 if (tsource != NULL && cts->tokens != NULL) 510 { 511 /* Finally, let's get a string 512 */ 513 string = tsource->strFactory->newRaw(tsource->strFactory); 514 515 for (i = start; i <= stop; i++) 516 { 517 tok = ts->get(ts, i); 518 if (tok != NULL) 519 { 520 string->appendS(string, tok->getText(tok)); 521 } 522 } 523 524 return string; 525 } 526 return NULL; 527 528} 529 530static pANTLR3_STRING 531toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop) 532{ 533 if (start != NULL && stop != NULL) 534 { 535 return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop)); 536 } 537 else 538 { 539 return NULL; 540 } 541} 542 543/** Move the input pointer to the next incoming token. The stream 544 * must become active with LT(1) available. consume() simply 545 * moves the input pointer so that LT(1) points at the next 546 * input symbol. Consume at least one token. 547 * 548 * Walk past any token not on the channel the parser is listening to. 549 */ 550static void 551consume (pANTLR3_INT_STREAM is) 552{ 553 pANTLR3_COMMON_TOKEN_STREAM cts; 554 pANTLR3_TOKEN_STREAM ts; 555 556 ts = (pANTLR3_TOKEN_STREAM) is->super; 557 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 558 559 if ((ANTLR3_UINT32)cts->p < cts->tokens->size(cts->tokens)) 560 { 561 cts->p++; 562 cts->p = skipOffTokenChannels(cts, cts->p); 563 } 564} 565 566 567/// As per ordinary consume but notifies the debugger about hidden 568/// tokens and so on. 569/// 570static void 571dbgConsume (pANTLR3_INT_STREAM is) 572{ 573 pANTLR3_TOKEN_STREAM ts; 574 ANTLR3_MARKER a; 575 ANTLR3_MARKER b; 576 pANTLR3_COMMON_TOKEN t; 577 578 ts = (pANTLR3_TOKEN_STREAM) is->super; 579 580 if (ts->initialStreamState == ANTLR3_TRUE) 581 { 582 consumeInitialHiddenTokens(is); 583 } 584 585 a = is->index(is); // Where are we right now? 586 t = ts->_LT(ts, 1); // Current token from stream 587 588 consume(is); // Standard consumer 589 590 b = is->index(is); // Where are we after consuming 1 on channel token? 591 592 ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token 593 594 if (b>a+1) 595 { 596 // The standard consume caused the index to advance by more than 1, 597 // which can only happen if it skipped some off-channel tokens. 598 // we need to tell the debugger about those tokens. 599 // 600 ANTLR3_MARKER i; 601 602 for (i = a+1; i<b; i++) 603 { 604 ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i)); 605 } 606 607 } 608} 609 610/** A simple filter mechanism whereby you can tell this token stream 611 * to force all tokens of type ttype to be on channel. For example, 612 * when interpreting, we cannot execute actions so we need to tell 613 * the stream to force all WS and NEWLINE to be a different, ignored, 614 * channel. 615 */ 616static void 617setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel) 618{ 619 if (tokenStream->channelOverrides == NULL) 620 { 621 tokenStream->channelOverrides = antlr3ListNew(10); 622 } 623 624 /* We add one to the channel so we can distinguish NULL as being no entry in the 625 * table for a particular token type. 626 */ 627 tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL); 628} 629 630static void 631discardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype) 632{ 633 if (tokenStream->discardSet == NULL) 634 { 635 tokenStream->discardSet = antlr3ListNew(31); 636 } 637 638 /* We add one to the channel so we can distinguish NULL as being no entry in the 639 * table for a particular token type. We could use bitsets for this I suppose too. 640 */ 641 tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL); 642} 643 644static void 645discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard) 646{ 647 tokenStream->discardOffChannel = discard; 648} 649 650static pANTLR3_VECTOR 651getTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream) 652{ 653 if (tokenStream->p == -1) 654 { 655 fillBuffer(tokenStream); 656 } 657 658 return tokenStream->tokens; 659} 660 661static pANTLR3_LIST 662getTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) 663{ 664 return tokenStream->getTokensSet(tokenStream, start, stop, NULL); 665} 666/** Given a start and stop index, return a List of all tokens in 667 * the token type BitSet. Return null if no tokens were found. This 668 * method looks at both on and off channel tokens. 669 */ 670static pANTLR3_LIST 671getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types) 672{ 673 pANTLR3_LIST filteredList; 674 ANTLR3_UINT32 i; 675 ANTLR3_UINT32 n; 676 pANTLR3_COMMON_TOKEN tok; 677 678 if (tokenStream->p == -1) 679 { 680 fillBuffer(tokenStream); 681 } 682 if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream)) 683 { 684 stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream); 685 } 686 if (start > stop) 687 { 688 return NULL; 689 } 690 691 /* We have the range set, now we need to iterate through the 692 * installed tokens and create a new list with just the ones we want 693 * in it. We are just moving pointers about really. 694 */ 695 filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream)); 696 697 for (i = start, n = 0; i<= stop; i++) 698 { 699 tok = tokenStream->tstream->get(tokenStream->tstream, i); 700 701 if ( types == NULL 702 || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE) 703 ) 704 { 705 filteredList->put(filteredList, n++, (void *)tok, NULL); 706 } 707 } 708 709 /* Did we get any then? 710 */ 711 if (filteredList->size(filteredList) == 0) 712 { 713 filteredList->free(filteredList); 714 filteredList = NULL; 715 } 716 717 return filteredList; 718} 719 720static pANTLR3_LIST 721getTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list) 722{ 723 pANTLR3_BITSET bitSet; 724 pANTLR3_LIST newlist; 725 726 bitSet = antlr3BitsetList(list->table); 727 728 newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); 729 730 bitSet->free(bitSet); 731 732 return newlist; 733 734} 735 736static pANTLR3_LIST 737getTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type) 738{ 739 pANTLR3_BITSET bitSet; 740 pANTLR3_LIST newlist; 741 742 bitSet = antlr3BitsetOf(type, -1); 743 newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); 744 745 bitSet->free(bitSet); 746 747 return newlist; 748} 749 750static ANTLR3_UINT32 751_LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) 752{ 753 pANTLR3_TOKEN_STREAM ts; 754 pANTLR3_COMMON_TOKEN tok; 755 756 ts = (pANTLR3_TOKEN_STREAM) is->super; 757 758 tok = ts->_LT(ts, i); 759 760 if (tok != NULL) 761 { 762 return tok->getType(tok); 763 } 764 else 765 { 766 return ANTLR3_TOKEN_INVALID; 767 } 768} 769 770/// As per _LA() but for debug mode. 771/// 772static ANTLR3_UINT32 773dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) 774{ 775 pANTLR3_TOKEN_STREAM ts; 776 777 ts = (pANTLR3_TOKEN_STREAM) is->super; 778 779 if (ts->initialStreamState == ANTLR3_TRUE) 780 { 781 consumeInitialHiddenTokens(is); 782 } 783 ts->debugger->LT(ts->debugger, i, tokLT(ts, i)); 784 return _LA(is, i); 785} 786 787static ANTLR3_MARKER 788mark (pANTLR3_INT_STREAM is) 789{ 790 is->lastMarker = is->index(is); 791 return is->lastMarker; 792} 793 794/// As per mark() but with a call to tell the debugger we are doing this 795/// 796static ANTLR3_MARKER 797dbgMark (pANTLR3_INT_STREAM is) 798{ 799 pANTLR3_TOKEN_STREAM ts; 800 801 ts = (pANTLR3_TOKEN_STREAM) is->super; 802 803 is->lastMarker = is->index(is); 804 ts->debugger->mark(ts->debugger, is->lastMarker); 805 806 return is->lastMarker; 807} 808 809static void 810release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark) 811{ 812 return; 813} 814 815static ANTLR3_UINT32 816size (pANTLR3_INT_STREAM is) 817{ 818 pANTLR3_COMMON_TOKEN_STREAM cts; 819 pANTLR3_TOKEN_STREAM ts; 820 821 if (is->cachedSize > 0) 822 { 823 return is->cachedSize; 824 } 825 ts = (pANTLR3_TOKEN_STREAM) is->super; 826 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 827 828 is->cachedSize = cts->tokens->count; 829 return is->cachedSize; 830} 831 832static ANTLR3_MARKER 833tindex (pANTLR3_INT_STREAM is) 834{ 835 pANTLR3_COMMON_TOKEN_STREAM cts; 836 pANTLR3_TOKEN_STREAM ts; 837 838 ts = (pANTLR3_TOKEN_STREAM) is->super; 839 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 840 841 return cts->p; 842} 843 844static void 845dbgRewindLast (pANTLR3_INT_STREAM is) 846{ 847 pANTLR3_TOKEN_STREAM ts; 848 849 ts = (pANTLR3_TOKEN_STREAM) is->super; 850 851 ts->debugger->rewindLast(ts->debugger); 852 853 is->rewind(is, is->lastMarker); 854} 855static void 856rewindLast (pANTLR3_INT_STREAM is) 857{ 858 is->rewind(is, is->lastMarker); 859} 860static void 861rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) 862{ 863 is->seek(is, (ANTLR3_UINT32)(marker)); 864} 865static void 866dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) 867{ 868 pANTLR3_TOKEN_STREAM ts; 869 870 ts = (pANTLR3_TOKEN_STREAM) is->super; 871 872 ts->debugger->rewind(ts->debugger, marker); 873 874 is->seek(is, (ANTLR3_UINT32)(marker)); 875} 876 877static void 878seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) 879{ 880 pANTLR3_COMMON_TOKEN_STREAM cts; 881 pANTLR3_TOKEN_STREAM ts; 882 883 ts = (pANTLR3_TOKEN_STREAM) is->super; 884 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 885 886 cts->p = (ANTLR3_UINT32)index; 887} 888static void 889dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) 890{ 891 // TODO: Implement seek in debugger when Ter adds it to Java 892 // 893 seek(is, index); 894} 895ANTLR3_API void 896fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream) 897{ 898 fillBuffer(tokenStream); 899} 900static void 901fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) { 902 ANTLR3_UINT32 index; 903 pANTLR3_COMMON_TOKEN tok; 904 ANTLR3_BOOLEAN discard; 905 void * channelI; 906 907 /* Start at index 0 of course 908 */ 909 index = 0; 910 911 /* Pick out the next token from the token source 912 * Remember we just get a pointer (reference if you like) here 913 * and so if we store it anywhere, we don't set any pointers to auto free it. 914 */ 915 tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); 916 917 while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF) 918 { 919 discard = ANTLR3_FALSE; /* Assume we are not discarding */ 920 921 /* I employ a bit of a trick, or perhaps hack here. Rather than 922 * store a pointer to a structure in the override map and discard set 923 * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0 924 * we can distinguish "not being there" from "being channel or type 0" 925 */ 926 927 if (tokenStream->discardSet != NULL 928 && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL) 929 { 930 discard = ANTLR3_TRUE; 931 } 932 else if ( tokenStream->discardOffChannel == ANTLR3_TRUE 933 && tok->getChannel(tok) != tokenStream->channel 934 ) 935 { 936 discard = ANTLR3_TRUE; 937 } 938 else if (tokenStream->channelOverrides != NULL) 939 { 940 /* See if this type is in the override map 941 */ 942 channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1); 943 944 if (channelI != NULL) 945 { 946 /* Override found 947 */ 948 tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1); 949 } 950 } 951 952 /* If not discarding it, add it to the list at the current index 953 */ 954 if (discard == ANTLR3_FALSE) 955 { 956 /* Add it, indicating that we will delete it and the table should not 957 */ 958 tok->setTokenIndex(tok, index); 959 tokenStream->p++; 960 tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL); 961 index++; 962 } 963 964 tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); 965 } 966 967 /* Cache the size so we don't keep doing indirect method calls. We do this as 968 * early as possible so that anything after this may utilize the cached value. 969 */ 970 tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count; 971 972 /* Set the consume pointer to the first token that is on our channel 973 */ 974 tokenStream->p = 0; 975 tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p); 976 977} 978 979/// Given a starting index, return the index of the first on-channel 980/// token. 981/// 982static ANTLR3_UINT32 983skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) { 984 ANTLR3_INT32 n; 985 pANTLR3_COMMON_TOKEN tok; 986 987 n = tokenStream->tstream->istream->cachedSize; 988 989 while (i < n) 990 { 991 tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element; 992 993 if (tok->channel!= tokenStream->channel) 994 { 995 i++; 996 } 997 else 998 { 999 return i; 1000 } 1001 } 1002 return i; 1003} 1004 1005static ANTLR3_UINT32 1006skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x) 1007{ 1008 pANTLR3_COMMON_TOKEN tok; 1009 1010 while (x >= 0) 1011 { 1012 tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element; 1013 1014 if ((tok->channel != tokenStream->channel)) 1015 { 1016 x--; 1017 } 1018 else 1019 { 1020 return x; 1021 } 1022 } 1023 return x; 1024} 1025 1026/// Return a string that represents the name assoicated with the input source 1027/// 1028/// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream. 1029/// 1030/// /returns 1031/// /implements ANTLR3_INT_STREAM_struct::getSourceName() 1032/// 1033static pANTLR3_STRING 1034getSourceName (pANTLR3_INT_STREAM is) 1035{ 1036 // Slightly convoluted as we must trace back to the lexer's input source 1037 // via the token source. The streamName that is here is not initialized 1038 // because this is a token stream, not a file or string stream, which are the 1039 // only things that have a context for a source name. 1040 // 1041 return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName; 1042} 1043