Lines Matching refs:tokens
1367 '[type]' tokens). If prefix and/or suffix are removed, the full
1490 Inside the {...} pair is the usual set of tokens, but again
1499 tokens = collections.deque(_scan(string))
1501 def get_subscripted(tokens):
1504 tuple after stripping off both tokens, or returns None
1505 and does not strip tokens.
1507 if len(tokens) == 0 or tokens[0].ttype != 'word':
1509 if len(tokens) > 1 and tokens[1].ttype == 'type':
1510 word = tokens.popleft()
1511 return word, tokens.popleft()
1592 while tokens:
1593 token = tokens.popleft()
1601 if len(tokens) == 0:
1603 cond = tokens.popleft()
1616 if len(tokens) == 0 or tokens[0].ttype != '(':
1618 tokens.popleft()
1619 repeat = get_subscripted(tokens)
1622 if len(tokens) == 0 or tokens[0].ttype != ')':
1625 tokens.popleft()
1636 if len(tokens) == 0 or tokens[0].ttype != 'type':
1638 type_or_size = tokens.popleft()
1645 if len(tokens) > 0 and tokens[0].ttype == 'aux':
1646 aux = tokens.popleft()