Lines Matching refs:tokenize
46 from cpp import tokenize
551 if parts[-1].token_type == tokenize.NAME:
581 if (type_name and type_name[-1].token_type == tokenize.NAME and
582 p.token_type == tokenize.NAME):
583 type_name.append(tokenize.Token(tokenize.SYNTAX, ' ', 0, 0))
739 if token.token_type == tokenize.NAME:
750 if next.token_type == tokenize.SYNTAX and next.name == '(':
755 syntax = tokenize.SYNTAX
764 new_temp = self._GetTokensUpTo(tokenize.SYNTAX, ';')
767 last_token = tokenize.Token(tokenize.SYNTAX, ';', 0, 0)
772 # unused_size = self._GetTokensUpTo(tokenize.SYNTAX, ']')
778 self._GetVarTokensUpTo(tokenize.SYNTAX, ';')
806 elif token.token_type == tokenize.SYNTAX:
812 if (token.token_type == tokenize.NAME and
816 elif token.token_type == tokenize.PREPROCESSOR:
869 if token.token_type != tokenize.PREPROCESSOR:
888 if token.token_type == tokenize.SYNTAX:
911 if token.whence == tokenize.WHENCE_STREAM:
912 token.whence = tokenize.WHENCE_QUEUE
915 assert token.whence == tokenize.WHENCE_QUEUE, token
920 if tokens[-1].whence == tokenize.WHENCE_STREAM:
922 token.whence = tokenize.WHENCE_QUEUE
925 assert tokens[-1].whence == tokenize.WHENCE_QUEUE, tokens
937 while (next_token.token_type == tokenize.NAME or
938 (next_token.token_type == tokenize.SYNTAX and
942 if last_token_was_name and next_token.token_type == tokenize.NAME:
944 last_token_was_name = next_token.token_type == tokenize.NAME
954 return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(')
964 assert token.token_type == tokenize.SYNTAX, token
970 assert token.token_type == tokenize.SYNTAX, token
988 name = tokenize.Token(tokenize.NAME, 'operator[]',
1013 while token.token_type == tokenize.NAME:
1040 assert token.token_type == tokenize.SYNTAX, token
1060 assert token.token_type == tokenize.SYNTAX, token
1092 assert token.token_type == tokenize.CONSTANT, token
1138 seq_copy.append(tokenize.Token(tokenize.SYNTAX, '', 0, 0))
1145 if next and next.token_type == tokenize.SYNTAX:
1200 if token.token_type == tokenize.SYNTAX and token.name == ';':
1204 if token.token_type == tokenize.NAME and self._handling_typedef:
1212 if token.token_type == tokenize.SYNTAX and token.name == '{':
1218 if next.token_type != tokenize.NAME:
1224 assert token.token_type == tokenize.NAME, token
1233 is_syntax = (var_token.token_type == tokenize.SYNTAX and
1235 is_variable = (var_token.token_type == tokenize.NAME and
1241 if temp.token_type == tokenize.SYNTAX and temp.name == '(':
1244 struct = tokenize.Token(tokenize.NAME, 'struct',
1268 if not (token.token_type == tokenize.NAME and token.name == 'class'):
1298 if token2.token_type == tokenize.SYNTAX and token2.name == '~':
1300 assert token.token_type == tokenize.NAME or token.name == '::', token
1301 return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(') # )
1327 tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1348 tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1354 if (token.token_type == tokenize.NAME and
1365 tokens.extend(self._GetTokensUpTo(tokenize.SYNTAX, ';'))
1387 if tokens and isinstance(tokens[0], tokenize.Token):
1427 assert token.token_type == tokenize.SYNTAX, token
1432 if token.token_type == tokenize.NAME:
1440 tokens, last = self._GetVarTokensUpTo(tokenize.SYNTAX, '(', ';')
1465 assert token.token_type == tokenize.NAME, token
1484 assert next_token.token_type == tokenize.SYNTAX, next_token
1495 if class_token.token_type != tokenize.NAME:
1496 assert class_token.token_type == tokenize.SYNTAX, class_token
1502 if next_token.token_type == tokenize.NAME:
1509 if token.token_type == tokenize.SYNTAX:
1534 if token.token_type == tokenize.SYNTAX and token.name == '{':
1535 assert token.token_type == tokenize.SYNTAX, token
1544 if token.token_type != tokenize.NAME:
1545 assert token.token_type == tokenize.SYNTAX, token
1568 if token.token_type == tokenize.NAME:
1572 assert token.token_type == tokenize.SYNTAX, token
1574 internal_token = tokenize.Token(_INTERNAL_TOKEN, _NAMESPACE_POP,
1592 tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1621 assert token.token_type == tokenize.SYNTAX
1631 tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1637 tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1660 self._IgnoreUpTo(tokenize.SYNTAX, ';')
1663 self._IgnoreUpTo(tokenize.SYNTAX, ';')
1676 return AstBuilder(tokenize.GetTokens(source), filename)