source_language/c_family/CLanguageTokenizer.cpp

75 	fCurrentChar(NULL),
86 	fCurrentChar = fString.String();
103 	while (*fCurrentChar != 0 && isspace(*fCurrentChar))
104 		fCurrentChar++;
106 	if (*fCurrentChar == 0) {
111 	bool decimal = *fCurrentChar == '.';
113 	if (decimal || isdigit(*fCurrentChar)) {
114 		if (*fCurrentChar == '0' && fCurrentChar[1] == 'x')
119 		const char* begin = fCurrentChar;
122 		while (isdigit(*fCurrentChar)) {
123 			temp << *fCurrentChar;
124 			fCurrentChar++;
129 		if (*fCurrentChar == '.') {
132 			fCurrentChar++;
135 			while (isdigit(*fCurrentChar)) {
136 				temp << *fCurrentChar;
137 				fCurrentChar++;
141 		int32 length = fCurrentChar - begin;
144 			fCurrentChar = begin;
165 	} else if (isalpha(*fCurrentChar) || *fCurrentChar == '_') {
166 		const char* begin = fCurrentChar;
167 		while (*fCurrentChar != 0 && (isalpha(*fCurrentChar)
168 			|| isdigit(*fCurrentChar) || *fCurrentChar == '_')) {
169 			fCurrentChar++;
171 		int32 length = fCurrentChar - begin;
174 	} else if (*fCurrentChar == '"' || *fCurrentChar == '\'') {
176 		const char* begin = fCurrentChar++;
177 		while (*fCurrentChar != 0) {
178 			if (*fCurrentChar == '\\') {
179 				if (*(fCurrentChar++) != 0)
180 					fCurrentChar++;
181 			} else if (*(fCurrentChar++) == *begin) {
190 			fCurrentChar = begin + 1;
193 		int32 length = fCurrentChar - begin;
199 			switch (*fCurrentChar) {
253 			fCurrentToken = Token(fCurrentChar, 1, _CurrentPos(),
255 			fCurrentChar++;
268 	switch (*fCurrentChar) {
407 	fCurrentToken = Token(fCurrentChar, length, _CurrentPos(), type);
408 	fCurrentChar += length;
425 		return *(fCurrentChar + 1);
441 	const char* begin = fCurrentChar;
442 	fCurrentChar += 2;
445 	if (!_IsHexDigit(*fCurrentChar))
448 	fCurrentChar++;
449 	while (_IsHexDigit(*fCurrentChar))
450 		fCurrentChar++;
452 	int32 length = fCurrentChar - begin;
472 	return fCurrentChar - fString.String();