parsetok.c 3.25 KB
Newer Older
1

Guido van Rossum's avatar
Guido van Rossum committed
2 3
/* Parser-tokenizer link implementation */

Guido van Rossum's avatar
Guido van Rossum committed
4
#include "pgenheaders.h"
Guido van Rossum's avatar
Guido van Rossum committed
5 6 7 8
#include "tokenizer.h"
#include "node.h"
#include "grammar.h"
#include "parser.h"
Guido van Rossum's avatar
Guido van Rossum committed
9
#include "parsetok.h"
Guido van Rossum's avatar
Guido van Rossum committed
10 11
#include "errcode.h"

12 13
int Py_TabcheckFlag;

Guido van Rossum's avatar
Guido van Rossum committed
14

Guido van Rossum's avatar
Guido van Rossum committed
15
/* Forward */
16
static node *parsetok(struct tok_state *, grammar *, int, perrdetail *);
Guido van Rossum's avatar
Guido van Rossum committed
17

Guido van Rossum's avatar
Guido van Rossum committed
18
/* Parse input coming from a string.  Return error code, print some errors. */
Guido van Rossum's avatar
Guido van Rossum committed
19

20
node *
Thomas Wouters's avatar
Thomas Wouters committed
21
PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret)
Guido van Rossum's avatar
Guido van Rossum committed
22
{
23 24 25 26 27 28 29
	struct tok_state *tok;

	err_ret->error = E_OK;
	err_ret->filename = NULL;
	err_ret->lineno = 0;
	err_ret->offset = 0;
	err_ret->text = NULL;
30 31
	err_ret->token = -1;
	err_ret->expected = -1;
32

33
	if ((tok = PyTokenizer_FromString(s)) == NULL) {
34 35
		err_ret->error = E_NOMEM;
		return NULL;
Guido van Rossum's avatar
Guido van Rossum committed
36
	}
37

38 39 40 41 42 43 44
	if (Py_TabcheckFlag || Py_VerboseFlag) {
		tok->filename = "<string>";
		tok->altwarning = (tok->filename != NULL);
		if (Py_TabcheckFlag >= 2)
			tok->alterror++;
	}

45
	return parsetok(tok, g, start, err_ret);
Guido van Rossum's avatar
Guido van Rossum committed
46 47 48
}


Guido van Rossum's avatar
Guido van Rossum committed
49
/* Parse input coming from a file.  Return error code, print some errors. */
Guido van Rossum's avatar
Guido van Rossum committed
50

51
node *
Thomas Wouters's avatar
Thomas Wouters committed
52 53
PyParser_ParseFile(FILE *fp, char *filename, grammar *g, int start,
		   char *ps1, char *ps2, perrdetail *err_ret)
Guido van Rossum's avatar
Guido van Rossum committed
54
{
55 56 57 58 59 60 61 62
	struct tok_state *tok;

	err_ret->error = E_OK;
	err_ret->filename = filename;
	err_ret->lineno = 0;
	err_ret->offset = 0;
	err_ret->text = NULL;

63
	if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
64 65
		err_ret->error = E_NOMEM;
		return NULL;
Guido van Rossum's avatar
Guido van Rossum committed
66
	}
67 68 69 70 71 72
	if (Py_TabcheckFlag || Py_VerboseFlag) {
		tok->filename = filename;
		tok->altwarning = (filename != NULL);
		if (Py_TabcheckFlag >= 2)
			tok->alterror++;
	}
73

Guido van Rossum's avatar
Guido van Rossum committed
74

75 76
	return parsetok(tok, g, start, err_ret);
}
Guido van Rossum's avatar
Guido van Rossum committed
77 78 79 80

/* Parse input coming from the given tokenizer structure.
   Return error code. */

81
static node *
Thomas Wouters's avatar
Thomas Wouters committed
82
parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret)
Guido van Rossum's avatar
Guido van Rossum committed
83 84
{
	parser_state *ps;
85
	node *n;
86
	int started = 0;
87

88
	if ((ps = PyParser_New(g, start)) == NULL) {
Guido van Rossum's avatar
Guido van Rossum committed
89
		fprintf(stderr, "no mem for new parser\n");
90 91
		err_ret->error = E_NOMEM;
		return NULL;
Guido van Rossum's avatar
Guido van Rossum committed
92
	}
93

Guido van Rossum's avatar
Guido van Rossum committed
94 95 96
	for (;;) {
		char *a, *b;
		int type;
97
		size_t len;
Guido van Rossum's avatar
Guido van Rossum committed
98
		char *str;
99

100
		type = PyTokenizer_Get(tok, &a, &b);
Guido van Rossum's avatar
Guido van Rossum committed
101
		if (type == ERRORTOKEN) {
102
			err_ret->error = tok->done;
Guido van Rossum's avatar
Guido van Rossum committed
103 104
			break;
		}
105 106 107 108 109 110
		if (type == ENDMARKER && started) {
			type = NEWLINE; /* Add an extra newline */
			started = 0;
		}
		else
			started = 1;
111
		len = b - a; /* XXX this may compute NULL - NULL */
112
		str = PyMem_NEW(char, len + 1);
Guido van Rossum's avatar
Guido van Rossum committed
113 114
		if (str == NULL) {
			fprintf(stderr, "no mem for next token\n");
115
			err_ret->error = E_NOMEM;
Guido van Rossum's avatar
Guido van Rossum committed
116 117
			break;
		}
118 119
		if (len > 0)
			strncpy(str, a, len);
Guido van Rossum's avatar
Guido van Rossum committed
120
		str[len] = '\0';
121
		if ((err_ret->error =
122 123
		     PyParser_AddToken(ps, (int)type, str, tok->lineno,
				       &(err_ret->expected))) != E_OK) {
124 125
			if (err_ret->error != E_DONE)
				PyMem_DEL(str);
Guido van Rossum's avatar
Guido van Rossum committed
126
			break;
127
		}
Guido van Rossum's avatar
Guido van Rossum committed
128
	}
129 130 131 132 133 134 135 136

	if (err_ret->error == E_DONE) {
		n = ps->p_tree;
		ps->p_tree = NULL;
	}
	else
		n = NULL;

137
	PyParser_Delete(ps);
138 139 140 141 142 143 144

	if (n == NULL) {
		if (tok->lineno <= 1 && tok->done == E_EOF)
			err_ret->error = E_EOF;
		err_ret->lineno = tok->lineno;
		err_ret->offset = tok->cur - tok->buf;
		if (tok->buf != NULL) {
145
			size_t len = tok->inp - tok->buf;
146
			err_ret->text = PyMem_NEW(char, len + 1);
147
			if (err_ret->text != NULL) {
148 149
				if (len > 0)
					strncpy(err_ret->text, tok->buf, len);
150 151 152 153 154
				err_ret->text[len] = '\0';
			}
		}
	}

155
	PyTokenizer_Free(tok);
156 157

	return n;
Guido van Rossum's avatar
Guido van Rossum committed
158
}