token.h 1.67 KB
Newer Older
1

Guido van Rossum's avatar
Guido van Rossum committed
2 3
/* Token types */

4 5 6 7 8 9
#ifndef Py_TOKEN_H
#define Py_TOKEN_H
#ifdef __cplusplus
extern "C" {
#endif

Guido van Rossum's avatar
Guido van Rossum committed
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37
#define ENDMARKER	0
#define NAME		1
#define NUMBER		2
#define STRING		3
#define NEWLINE		4
#define INDENT		5
#define DEDENT		6
#define LPAR		7
#define RPAR		8
#define LSQB		9
#define RSQB		10
#define COLON		11
#define COMMA		12
#define SEMI		13
#define PLUS		14
#define MINUS		15
#define STAR		16
#define SLASH		17
#define VBAR		18
#define AMPER		19
#define LESS		20
#define GREATER		21
#define EQUAL		22
#define DOT		23
#define PERCENT		24
#define BACKQUOTE	25
#define LBRACE		26
#define RBRACE		27
Guido van Rossum's avatar
Guido van Rossum committed
38 39 40 41
#define EQEQUAL		28
#define NOTEQUAL	29
#define LESSEQUAL	30
#define GREATEREQUAL	31
42 43 44 45
#define TILDE		32
#define CIRCUMFLEX	33
#define LEFTSHIFT	34
#define RIGHTSHIFT	35
46
#define DOUBLESTAR	36
47 48 49 50 51 52 53 54 55 56 57
#define PLUSEQUAL	37
#define MINEQUAL	38
#define STAREQUAL	39
#define SLASHEQUAL	40
#define PERCENTEQUAL	41
#define AMPEREQUAL	42
#define VBAREQUAL	43
#define CIRCUMFLEXEQUAL	44
#define LEFTSHIFTEQUAL	45
#define RIGHTSHIFTEQUAL	46
#define DOUBLESTAREQUAL	47
58 59
#define DOUBLESLASH	48
#define DOUBLESLASHEQUAL 49
60
#define AT              50	
61
/* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
62 63 64
#define OP		51
#define ERRORTOKEN	52
#define N_TOKENS	53
Guido van Rossum's avatar
Guido van Rossum committed
65 66 67 68 69 70 71 72 73 74

/* Special definitions for cooperation with parser */

#define NT_OFFSET		256

#define ISTERMINAL(x)		((x) < NT_OFFSET)
#define ISNONTERMINAL(x)	((x) >= NT_OFFSET)
#define ISEOF(x)		((x) == ENDMARKER)


75 76 77 78
PyAPI_DATA(char *) _PyParser_TokenNames[]; /* Token names */
PyAPI_FUNC(int) PyToken_OneChar(int);
PyAPI_FUNC(int) PyToken_TwoChars(int, int);
PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int);
79 80 81 82 83

#ifdef __cplusplus
}
#endif
#endif /* !Py_TOKEN_H */