marshal.c 17 KB
Newer Older
Guido van Rossum's avatar
Guido van Rossum committed
1 2 3 4 5 6

/* Write Python objects to files and read them back.
   This is intended for writing and reading compiled Python code only;
   a true persistent storage facility would be much harder, since
   it would have to take circular links and sharing into account. */

7
#include "Python.h"
Guido van Rossum's avatar
Guido van Rossum committed
8 9 10 11
#include "longintrepr.h"
#include "compile.h"
#include "marshal.h"

12 13 14 15 16 17
/* High water mark to determine when the marshalled object is dangerously deep
 * and risks coring the interpreter.  When the object stack gets this deep,
 * raise an exception instead of continuing.
 */
#define MAX_MARSHAL_STACK_DEPTH 5000

Guido van Rossum's avatar
Guido van Rossum committed
18 19
#define TYPE_NULL	'0'
#define TYPE_NONE	'N'
20
#define TYPE_ELLIPSIS   '.'
Guido van Rossum's avatar
Guido van Rossum committed
21
#define TYPE_INT	'i'
22
#define TYPE_INT64	'I'
Guido van Rossum's avatar
Guido van Rossum committed
23
#define TYPE_FLOAT	'f'
24
#define TYPE_COMPLEX	'x'
Guido van Rossum's avatar
Guido van Rossum committed
25 26 27 28 29
#define TYPE_LONG	'l'
#define TYPE_STRING	's'
#define TYPE_TUPLE	'('
#define TYPE_LIST	'['
#define TYPE_DICT	'{'
30
#define TYPE_CODE	'c'
31
#define TYPE_UNICODE	'u'
Guido van Rossum's avatar
Guido van Rossum committed
32 33
#define TYPE_UNKNOWN	'?'

34 35
typedef struct {
	FILE *fp;
36
	int error;
37
	int depth;
38
	/* If fp == NULL, the following are valid: */
39
	PyObject *str;
40 41 42
	char *ptr;
	char *end;
} WFILE;
Guido van Rossum's avatar
Guido van Rossum committed
43

44 45 46 47 48
#define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
		      else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
			   else w_more(c, p)

static void
49
w_more(int c, WFILE *p)
50 51 52 53
{
	int size, newsize;
	if (p->str == NULL)
		return; /* An error already occurred */
54
	size = PyString_Size(p->str);
55
	newsize = size + 1024;
56
	if (_PyString_Resize(&p->str, newsize) != 0) {
57 58 59
		p->ptr = p->end = NULL;
	}
	else {
60 61 62
		p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
		p->end =
			PyString_AS_STRING((PyStringObject *)p->str) + newsize;
63
		*p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
64 65 66 67
	}
}

static void
68
w_string(char *s, int n, WFILE *p)
69 70 71 72 73 74 75 76 77 78 79 80 81
{
	if (p->fp != NULL) {
		fwrite(s, 1, n, p->fp);
	}
	else {
		while (--n >= 0) {
			w_byte(*s, p);
			s++;
		}
	}
}

static void
82
w_short(int x, WFILE *p)
Guido van Rossum's avatar
Guido van Rossum committed
83
{
84 85
	w_byte( x      & 0xff, p);
	w_byte((x>> 8) & 0xff, p);
Guido van Rossum's avatar
Guido van Rossum committed
86 87
}

88
static void
89
w_long(long x, WFILE *p)
Guido van Rossum's avatar
Guido van Rossum committed
90
{
91 92 93 94
	w_byte((int)( x      & 0xff), p);
	w_byte((int)((x>> 8) & 0xff), p);
	w_byte((int)((x>>16) & 0xff), p);
	w_byte((int)((x>>24) & 0xff), p);
Guido van Rossum's avatar
Guido van Rossum committed
95 96
}

97
#if SIZEOF_LONG > 4
98
static void
99
w_long64(long x, WFILE *p)
100 101
{
	w_long(x, p);
102
	w_long(x>>32, p);
103
}
104
#endif
105

106
static void
107
w_object(PyObject *v, WFILE *p)
Guido van Rossum's avatar
Guido van Rossum committed
108
{
109
	int i, n;
110
	PyBufferProcs *pb;
111 112

	p->depth++;
113

114 115
	if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
		p->error = 2;
116
	}
117
	else if (v == NULL) {
118
		w_byte(TYPE_NULL, p);
119 120
	}
	else if (v == Py_None) {
121
		w_byte(TYPE_NONE, p);
122 123 124 125
	}
	else if (v == Py_Ellipsis) {
	        w_byte(TYPE_ELLIPSIS, p);
	}
126 127
	else if (PyInt_Check(v)) {
		long x = PyInt_AS_LONG((PyIntObject *)v);
128
#if SIZEOF_LONG > 4
129 130 131 132 133
		long y = x>>31;
		if (y && y != -1) {
			w_byte(TYPE_INT64, p);
			w_long64(x, p);
		}
134 135 136
		else
#endif
			{
137 138 139
			w_byte(TYPE_INT, p);
			w_long(x, p);
		}
Guido van Rossum's avatar
Guido van Rossum committed
140
	}
141 142
	else if (PyLong_Check(v)) {
		PyLongObject *ob = (PyLongObject *)v;
143
		w_byte(TYPE_LONG, p);
Guido van Rossum's avatar
Guido van Rossum committed
144
		n = ob->ob_size;
145
		w_long((long)n, p);
Guido van Rossum's avatar
Guido van Rossum committed
146 147 148
		if (n < 0)
			n = -n;
		for (i = 0; i < n; i++)
149
			w_short(ob->ob_digit[i], p);
Guido van Rossum's avatar
Guido van Rossum committed
150
	}
151
	else if (PyFloat_Check(v)) {
152
		extern void PyFloat_AsString(char *, PyFloatObject *);
Guido van Rossum's avatar
Guido van Rossum committed
153
		char buf[256]; /* Plenty to format any double */
154
		PyFloat_AsString(buf, (PyFloatObject *)v);
Guido van Rossum's avatar
Guido van Rossum committed
155
		n = strlen(buf);
156
		w_byte(TYPE_FLOAT, p);
157 158
		w_byte(n, p);
		w_string(buf, n, p);
Guido van Rossum's avatar
Guido van Rossum committed
159
	}
160
#ifndef WITHOUT_COMPLEX
161
	else if (PyComplex_Check(v)) {
162
		extern void PyFloat_AsString(char *, PyFloatObject *);
163
		char buf[256]; /* Plenty to format any double */
164
		PyFloatObject *temp;
165
		w_byte(TYPE_COMPLEX, p);
166 167 168 169
		temp = (PyFloatObject*)PyFloat_FromDouble(
			PyComplex_RealAsDouble(v));
		PyFloat_AsString(buf, temp);
		Py_DECREF(temp);
170 171 172
		n = strlen(buf);
		w_byte(n, p);
		w_string(buf, n, p);
173 174 175 176
		temp = (PyFloatObject*)PyFloat_FromDouble(
			PyComplex_ImagAsDouble(v));
		PyFloat_AsString(buf, temp);
		Py_DECREF(temp);
177 178 179 180 181
		n = strlen(buf);
		w_byte(n, p);
		w_string(buf, n, p);
	}
#endif
182
	else if (PyString_Check(v)) {
183
		w_byte(TYPE_STRING, p);
184
		n = PyString_GET_SIZE(v);
185
		w_long((long)n, p);
186 187 188 189 190 191
		w_string(PyString_AS_STRING(v), n, p);
	}
	else if (PyUnicode_Check(v)) {
	        PyObject *utf8;
		utf8 = PyUnicode_AsUTF8String(v);
		if (utf8 == NULL) {
192 193 194
			p->depth--;
			p->error = 1;
			return;
195 196 197 198 199 200
		}
		w_byte(TYPE_UNICODE, p);
		n = PyString_GET_SIZE(utf8);
		w_long((long)n, p);
		w_string(PyString_AS_STRING(utf8), n, p);
		Py_DECREF(utf8);
Guido van Rossum's avatar
Guido van Rossum committed
201
	}
202
	else if (PyTuple_Check(v)) {
203
		w_byte(TYPE_TUPLE, p);
204
		n = PyTuple_Size(v);
205
		w_long((long)n, p);
Guido van Rossum's avatar
Guido van Rossum committed
206
		for (i = 0; i < n; i++) {
207
			w_object(PyTuple_GET_ITEM(v, i), p);
Guido van Rossum's avatar
Guido van Rossum committed
208 209
		}
	}
210
	else if (PyList_Check(v)) {
211
		w_byte(TYPE_LIST, p);
212
		n = PyList_GET_SIZE(v);
213
		w_long((long)n, p);
Guido van Rossum's avatar
Guido van Rossum committed
214
		for (i = 0; i < n; i++) {
215
			w_object(PyList_GET_ITEM(v, i), p);
Guido van Rossum's avatar
Guido van Rossum committed
216 217
		}
	}
218
	else if (PyDict_Check(v)) {
219
		int pos;
220
		PyObject *key, *value;
221
		w_byte(TYPE_DICT, p);
222
		/* This one is NULL object terminated! */
223
		pos = 0;
224
		while (PyDict_Next(v, &pos, &key, &value)) {
225 226
			w_object(key, p);
			w_object(value, p);
227
		}
228
		w_object((PyObject *)NULL, p);
229
	}
230 231
	else if (PyCode_Check(v)) {
		PyCodeObject *co = (PyCodeObject *)v;
232
		w_byte(TYPE_CODE, p);
233 234
		w_short(co->co_argcount, p);
		w_short(co->co_nlocals, p);
235
		w_short(co->co_stacksize, p);
236
		w_short(co->co_flags, p);
237
		w_object(co->co_code, p);
238 239
		w_object(co->co_consts, p);
		w_object(co->co_names, p);
240
		w_object(co->co_varnames, p);
Jeremy Hylton's avatar
Jeremy Hylton committed
241 242
		w_object(co->co_freevars, p);
		w_object(co->co_cellvars, p);
243
		w_object(co->co_filename, p);
244
		w_object(co->co_name, p);
245 246
		w_short(co->co_firstlineno, p);
		w_object(co->co_lnotab, p);
Guido van Rossum's avatar
Guido van Rossum committed
247
	}
248 249 250 251 252 253 254 255 256 257 258 259
	else if ((pb = v->ob_type->tp_as_buffer) != NULL &&
		 pb->bf_getsegcount != NULL &&
		 pb->bf_getreadbuffer != NULL &&
		 (*pb->bf_getsegcount)(v, NULL) == 1)
	{
		/* Write unknown buffer-style objects as a string */
		char *s;
		w_byte(TYPE_STRING, p);
		n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
		w_long((long)n, p);
		w_string(s, n, p);
	}
Guido van Rossum's avatar
Guido van Rossum committed
260
	else {
261
		w_byte(TYPE_UNKNOWN, p);
262
		p->error = 1;
Guido van Rossum's avatar
Guido van Rossum committed
263
	}
264 265

	p->depth--;
Guido van Rossum's avatar
Guido van Rossum committed
266 267
}

268
void
269
PyMarshal_WriteLongToFile(long x, FILE *fp)
270 271 272
{
	WFILE wf;
	wf.fp = fp;
273
	wf.error = 0;
274
	wf.depth = 0;
275 276
	w_long(x, &wf);
}
Guido van Rossum's avatar
Guido van Rossum committed
277

278
void
279
PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp)
280 281 282
{
	WFILE wf;
	wf.fp = fp;
283
	wf.error = 0;
284
	wf.depth = 0;
285 286 287 288 289
	w_object(x, &wf);
}

typedef WFILE RFILE; /* Same struct with different invariants */

290 291 292
#define rs_byte(p) (((p)->ptr != (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)

#define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
293 294

static int
295
r_string(char *s, int n, RFILE *p)
296 297 298 299 300 301 302 303 304 305 306
{
	if (p->fp != NULL)
		return fread(s, 1, n, p->fp);
	if (p->end - p->ptr < n)
		n = p->end - p->ptr;
	memcpy(s, p->ptr, n);
	p->ptr += n;
	return n;
}

static int
307
r_short(RFILE *p)
Guido van Rossum's avatar
Guido van Rossum committed
308 309
{
	register short x;
310 311
	x = r_byte(p);
	x |= r_byte(p) << 8;
Tim Peters's avatar
Tim Peters committed
312 313
	/* Sign-extension, in case short greater than 16 bits */
	x |= -(x & 0x8000);
Guido van Rossum's avatar
Guido van Rossum committed
314 315 316
	return x;
}

317
static long
318
r_long(RFILE *p)
Guido van Rossum's avatar
Guido van Rossum committed
319 320
{
	register long x;
321 322 323 324 325 326 327 328 329 330 331 332 333
	register FILE *fp = p->fp;
	if (fp) {
		x = getc(fp);
		x |= (long)getc(fp) << 8;
		x |= (long)getc(fp) << 16;
		x |= (long)getc(fp) << 24;
	}
	else {
		x = rs_byte(p);
		x |= (long)rs_byte(p) << 8;
		x |= (long)rs_byte(p) << 16;
		x |= (long)rs_byte(p) << 24;
	}
334
#if SIZEOF_LONG > 4
335
	/* Sign extension for 64-bit machines */
Tim Peters's avatar
Tim Peters committed
336
	x |= -(x & 0x80000000L);
337
#endif
338 339 340 341
	return x;
}

static long
342
r_long64(RFILE *p)
343 344
{
	register long x;
345 346
	x = r_long(p);
#if SIZEOF_LONG > 4
Tim Peters's avatar
Tim Peters committed
347
	x = (x & 0xFFFFFFFFL) | (r_long(p) << 32);
348 349
#else
	if (r_long(p) != 0) {
350
		PyObject *f = PySys_GetObject("stderr");
351
		if (f != NULL)
352
			(void) PyFile_WriteString(
353 354
			    "Warning: un-marshal 64-bit int in 32-bit mode\n",
			    f);
355
	}
356
#endif
Guido van Rossum's avatar
Guido van Rossum committed
357 358 359
	return x;
}

360
static PyObject *
361
r_object(RFILE *p)
Guido van Rossum's avatar
Guido van Rossum committed
362
{
363
	PyObject *v, *v2;
Guido van Rossum's avatar
Guido van Rossum committed
364
	long i, n;
365
	int type = r_byte(p);
366

Guido van Rossum's avatar
Guido van Rossum committed
367
	switch (type) {
368

Guido van Rossum's avatar
Guido van Rossum committed
369
	case EOF:
370 371
		PyErr_SetString(PyExc_EOFError,
				"EOF read where object expected");
Guido van Rossum's avatar
Guido van Rossum committed
372
		return NULL;
373

Guido van Rossum's avatar
Guido van Rossum committed
374 375
	case TYPE_NULL:
		return NULL;
376

Guido van Rossum's avatar
Guido van Rossum committed
377
	case TYPE_NONE:
378 379
		Py_INCREF(Py_None);
		return Py_None;
380

381
	case TYPE_ELLIPSIS:
382
		Py_INCREF(Py_Ellipsis);
383
		return Py_Ellipsis;
384

Guido van Rossum's avatar
Guido van Rossum committed
385
	case TYPE_INT:
386
		return PyInt_FromLong(r_long(p));
387

388
	case TYPE_INT64:
389
		return PyInt_FromLong(r_long64(p));
390

Guido van Rossum's avatar
Guido van Rossum committed
391 392 393
	case TYPE_LONG:
		{
			int size;
394
			PyLongObject *ob;
395
			n = r_long(p);
Guido van Rossum's avatar
Guido van Rossum committed
396
			size = n<0 ? -n : n;
397
			ob = _PyLong_New(size);
Guido van Rossum's avatar
Guido van Rossum committed
398 399 400 401
			if (ob == NULL)
				return NULL;
			ob->ob_size = n;
			for (i = 0; i < size; i++)
402
				ob->ob_digit[i] = r_short(p);
403
			return (PyObject *)ob;
Guido van Rossum's avatar
Guido van Rossum committed
404
		}
405

Guido van Rossum's avatar
Guido van Rossum committed
406 407 408
	case TYPE_FLOAT:
		{
			char buf[256];
409
			double dx;
410 411
			n = r_byte(p);
			if (r_string(buf, (int)n, p) != n) {
412
				PyErr_SetString(PyExc_EOFError,
Guido van Rossum's avatar
Guido van Rossum committed
413 414 415 416
					"EOF read where object expected");
				return NULL;
			}
			buf[n] = '\0';
417 418
			PyFPE_START_PROTECT("atof", return 0)
			dx = atof(buf);
419
			PyFPE_END_PROTECT(dx)
420
			return PyFloat_FromDouble(dx);
Guido van Rossum's avatar
Guido van Rossum committed
421
		}
422

423 424 425 426
#ifndef WITHOUT_COMPLEX
	case TYPE_COMPLEX:
		{
			char buf[256];
427
			Py_complex c;
428 429
			n = r_byte(p);
			if (r_string(buf, (int)n, p) != n) {
430
				PyErr_SetString(PyExc_EOFError,
431 432 433 434
					"EOF read where object expected");
				return NULL;
			}
			buf[n] = '\0';
435
			PyFPE_START_PROTECT("atof", return 0)
436
			c.real = atof(buf);
437
			PyFPE_END_PROTECT(c)
438 439
			n = r_byte(p);
			if (r_string(buf, (int)n, p) != n) {
440
				PyErr_SetString(PyExc_EOFError,
441 442 443 444
					"EOF read where object expected");
				return NULL;
			}
			buf[n] = '\0';
445
			PyFPE_START_PROTECT("atof", return 0)
446
			c.imag = atof(buf);
447
			PyFPE_END_PROTECT(c)
448
			return PyComplex_FromCComplex(c);
449 450
		}
#endif
451

Guido van Rossum's avatar
Guido van Rossum committed
452
	case TYPE_STRING:
453
		n = r_long(p);
454 455 456 457
		if (n < 0) {
			PyErr_SetString(PyExc_ValueError, "bad marshal data");
			return NULL;
		}
458
		v = PyString_FromStringAndSize((char *)NULL, n);
Guido van Rossum's avatar
Guido van Rossum committed
459
		if (v != NULL) {
460
			if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
461
				Py_DECREF(v);
Guido van Rossum's avatar
Guido van Rossum committed
462
				v = NULL;
463
				PyErr_SetString(PyExc_EOFError,
Guido van Rossum's avatar
Guido van Rossum committed
464 465 466 467
					"EOF read where object expected");
			}
		}
		return v;
468

469 470 471 472 473 474 475 476 477
	case TYPE_UNICODE:
	    {
		char *buffer;

		n = r_long(p);
		if (n < 0) {
			PyErr_SetString(PyExc_ValueError, "bad marshal data");
			return NULL;
		}
478
		buffer = PyMem_NEW(char, n);
479
		if (buffer == NULL)
480
			return PyErr_NoMemory();
481
		if (r_string(buffer, (int)n, p) != n) {
482
			PyMem_DEL(buffer);
483 484 485 486 487
			PyErr_SetString(PyExc_EOFError,
				"EOF read where object expected");
			return NULL;
		}
		v = PyUnicode_DecodeUTF8(buffer, n, NULL);
488
		PyMem_DEL(buffer);
489 490
		return v;
	    }
491

Guido van Rossum's avatar
Guido van Rossum committed
492
	case TYPE_TUPLE:
493
		n = r_long(p);
494 495 496 497
		if (n < 0) {
			PyErr_SetString(PyExc_ValueError, "bad marshal data");
			return NULL;
		}
498
		v = PyTuple_New((int)n);
Guido van Rossum's avatar
Guido van Rossum committed
499 500
		if (v == NULL)
			return v;
501 502 503
		for (i = 0; i < n; i++) {
			v2 = r_object(p);
			if ( v2 == NULL ) {
504
				Py_DECREF(v);
505 506 507
				v = NULL;
				break;
			}
508
			PyTuple_SET_ITEM(v, (int)i, v2);
509
		}
Guido van Rossum's avatar
Guido van Rossum committed
510
		return v;
511

Guido van Rossum's avatar
Guido van Rossum committed
512
	case TYPE_LIST:
513
		n = r_long(p);
514 515 516 517
		if (n < 0) {
			PyErr_SetString(PyExc_ValueError, "bad marshal data");
			return NULL;
		}
518
		v = PyList_New((int)n);
Guido van Rossum's avatar
Guido van Rossum committed
519 520
		if (v == NULL)
			return v;
521 522 523
		for (i = 0; i < n; i++) {
			v2 = r_object(p);
			if ( v2 == NULL ) {
524
				Py_DECREF(v);
525 526 527
				v = NULL;
				break;
			}
528
			PyList_SetItem(v, (int)i, v2);
529
		}
Guido van Rossum's avatar
Guido van Rossum committed
530
		return v;
531

532
	case TYPE_DICT:
533
		v = PyDict_New();
534 535 536
		if (v == NULL)
			return NULL;
		for (;;) {
537
			PyObject *key, *val;
538
			key = r_object(p);
539
			if (key == NULL)
540
				break; /* XXX Assume TYPE_NULL, not an error */
541
			val = r_object(p);
542
			if (val != NULL)
543 544 545
				PyDict_SetItem(v, key, val);
			Py_DECREF(key);
			Py_XDECREF(val);
546 547
		}
		return v;
548

Guido van Rossum's avatar
Guido van Rossum committed
549 550
	case TYPE_CODE:
		{
551 552
			int argcount = r_short(p);
			int nlocals = r_short(p);
553
			int stacksize = r_short(p);
554
			int flags = r_short(p);
555 556 557 558
			PyObject *code = NULL;
			PyObject *consts = NULL;
			PyObject *names = NULL;
			PyObject *varnames = NULL;
Jeremy Hylton's avatar
Jeremy Hylton committed
559 560
			PyObject *freevars = NULL;
			PyObject *cellvars = NULL;
561 562
			PyObject *filename = NULL;
			PyObject *name = NULL;
Guido van Rossum's avatar
Guido van Rossum committed
563
			int firstlineno = 0;
564
			PyObject *lnotab = NULL;
565

566 567 568 569
			code = r_object(p);
			if (code) consts = r_object(p);
			if (consts) names = r_object(p);
			if (names) varnames = r_object(p);
Jeremy Hylton's avatar
Jeremy Hylton committed
570 571 572
			if (varnames) freevars = r_object(p);
			if (freevars) cellvars = r_object(p);
			if (cellvars) filename = r_object(p);
573
			if (filename) name = r_object(p);
574 575 576 577
			if (name) {
				firstlineno = r_short(p);
				lnotab = r_object(p);
			}
578

579 580
			if (!PyErr_Occurred()) {
				v = (PyObject *) PyCode_New(
581
					argcount, nlocals, stacksize, flags,
582
					code, consts, names, varnames,
583 584
					freevars, cellvars, filename, name,
					firstlineno, lnotab);
Guido van Rossum's avatar
Guido van Rossum committed
585 586 587
			}
			else
				v = NULL;
588 589 590 591
			Py_XDECREF(code);
			Py_XDECREF(consts);
			Py_XDECREF(names);
			Py_XDECREF(varnames);
Jeremy Hylton's avatar
Jeremy Hylton committed
592 593
			Py_XDECREF(freevars);
			Py_XDECREF(cellvars);
594 595
			Py_XDECREF(filename);
			Py_XDECREF(name);
596
			Py_XDECREF(lnotab);
Guido van Rossum's avatar
Guido van Rossum committed
597 598 599

		}
		return v;
600

Guido van Rossum's avatar
Guido van Rossum committed
601
	default:
602 603
		/* Bogus data got written, which isn't ideal.
		   This will let you keep working and recover. */
604 605
		PyErr_SetString(PyExc_ValueError, "bad marshal data");
		return NULL;
606

Guido van Rossum's avatar
Guido van Rossum committed
607 608 609
	}
}

610
long
611
PyMarshal_ReadLongFromFile(FILE *fp)
612 613 614 615 616 617
{
	RFILE rf;
	rf.fp = fp;
	return r_long(&rf);
}

618 619 620 621 622 623 624 625 626 627 628 629
#ifdef HAVE_FSTAT
/* Return size of file in bytes; < 0 if unknown. */
static off_t
getfilesize(FILE *fp)
{
	struct stat st;
	if (fstat(fileno(fp), &st) != 0)
		return -1;
	else
		return st.st_size;
}
#endif
630

631 632 633
/* If we can get the size of the file up-front, and it's reasonably small,
 * read it in one gulp and delegate to ...FromString() instead.  Much quicker
 * than reading a byte at a time from file; speeds .pyc imports.
634 635
 * CAUTION:  since this may read the entire remainder of the file, don't
 * call it unless you know you're done with the file.
636
 */
637
PyObject *
638
PyMarshal_ReadLastObjectFromFile(FILE *fp)
639
{
640 641 642 643 644 645 646 647
/* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
 * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
 */
#define SMALL_FILE_LIMIT (1L << 14)
#define REASONABLE_FILE_LIMIT (1L << 18)
#ifdef HAVE_FSTAT
	off_t filesize;
#endif
648
	if (PyErr_Occurred()) {
649 650 651
		fprintf(stderr, "XXX rd_object called with exception set\n");
		return NULL;
	}
652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668
#ifdef HAVE_FSTAT
	filesize = getfilesize(fp);
	if (filesize > 0) {
		char buf[SMALL_FILE_LIMIT];
		char* pBuf = NULL;
		if (filesize <= SMALL_FILE_LIMIT)
			pBuf = buf;
		else if (filesize <= REASONABLE_FILE_LIMIT)
			pBuf = (char *)PyMem_MALLOC(filesize);
		if (pBuf != NULL) {
			PyObject* v;
			size_t n = fread(pBuf, 1, filesize, fp);
			v = PyMarshal_ReadObjectFromString(pBuf, n);
			if (pBuf != buf)
				PyMem_FREE(pBuf);
			return v;
		}
669

670 671
	}
#endif
672 673 674 675 676
	/* We don't have fstat, or we do but the file is larger than
	 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
	 */
	return PyMarshal_ReadObjectFromFile(fp);

677 678
#undef SMALL_FILE_LIMIT
#undef REASONABLE_FILE_LIMIT
679 680
}

681 682 683 684 685 686 687 688 689 690 691 692
PyObject *
PyMarshal_ReadObjectFromFile(FILE *fp)
{
	RFILE rf;
	if (PyErr_Occurred()) {
		fprintf(stderr, "XXX rd_object called with exception set\n");
		return NULL;
	}
	rf.fp = fp;
	return r_object(&rf);
}

693
PyObject *
694
PyMarshal_ReadObjectFromString(char *str, int len)
695 696
{
	RFILE rf;
697
	if (PyErr_Occurred()) {
698 699 700
		fprintf(stderr, "XXX rds_object called with exception set\n");
		return NULL;
	}
701 702 703 704 705 706 707
	rf.fp = NULL;
	rf.str = NULL;
	rf.ptr = str;
	rf.end = str + len;
	return r_object(&rf);
}

708
PyObject *
709
PyMarshal_WriteObjectToString(PyObject *x) /* wrs_object() */
710 711 712
{
	WFILE wf;
	wf.fp = NULL;
713
	wf.str = PyString_FromStringAndSize((char *)NULL, 50);
714 715
	if (wf.str == NULL)
		return NULL;
716 717
	wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
	wf.end = wf.ptr + PyString_Size(wf.str);
718
	wf.error = 0;
719
	wf.depth = 0;
720 721
	w_object(x, &wf);
	if (wf.str != NULL)
722 723 724
		_PyString_Resize(&wf.str,
		    (int) (wf.ptr -
			   PyString_AS_STRING((PyStringObject *)wf.str)));
725
	if (wf.error) {
726
		Py_XDECREF(wf.str);
727
		PyErr_SetString(PyExc_ValueError,
728 729
				(wf.error==1)?"unmarshallable object"
				:"object too deeply nested to marshal");
730 731 732 733 734
		return NULL;
	}
	return wf.str;
}

735
/* And an interface for Python programs... */
Guido van Rossum's avatar
Guido van Rossum committed
736

737
static PyObject *
738
marshal_dump(PyObject *self, PyObject *args)
Guido van Rossum's avatar
Guido van Rossum committed
739
{
740
	WFILE wf;
741 742
	PyObject *x;
	PyObject *f;
743
	if (!PyArg_ParseTuple(args, "OO:dump", &x, &f))
Guido van Rossum's avatar
Guido van Rossum committed
744
		return NULL;
745 746 747
	if (!PyFile_Check(f)) {
		PyErr_SetString(PyExc_TypeError,
				"marshal.dump() 2nd arg must be file");
Guido van Rossum's avatar
Guido van Rossum committed
748 749
		return NULL;
	}
750
	wf.fp = PyFile_AsFile(f);
751 752
	wf.str = NULL;
	wf.ptr = wf.end = NULL;
753
	wf.error = 0;
754
	wf.depth = 0;
755
	w_object(x, &wf);
756
	if (wf.error) {
757
		PyErr_SetString(PyExc_ValueError,
758 759
				(wf.error==1)?"unmarshallable object"
				:"object too deeply nested to marshal");
760 761
		return NULL;
	}
762 763
	Py_INCREF(Py_None);
	return Py_None;
Guido van Rossum's avatar
Guido van Rossum committed
764 765
}

766
static PyObject *
767
marshal_load(PyObject *self, PyObject *args)
Guido van Rossum's avatar
Guido van Rossum committed
768
{
769
	RFILE rf;
770 771
	PyObject *f;
	PyObject *v;
772
	if (!PyArg_ParseTuple(args, "O:load", &f))
773
		return NULL;
774 775 776
	if (!PyFile_Check(f)) {
		PyErr_SetString(PyExc_TypeError,
				"marshal.load() arg must be file");
Guido van Rossum's avatar
Guido van Rossum committed
777 778
		return NULL;
	}
779
	rf.fp = PyFile_AsFile(f);
780 781
	rf.str = NULL;
	rf.ptr = rf.end = NULL;
782
	PyErr_Clear();
783
	v = r_object(&rf);
784 785
	if (PyErr_Occurred()) {
		Py_XDECREF(v);
786 787 788 789 790
		v = NULL;
	}
	return v;
}

791
static PyObject *
792
marshal_dumps(PyObject *self, PyObject *args)
793
{
794
	PyObject *x;
795
	if (!PyArg_ParseTuple(args, "O:dumps", &x))
796
		return NULL;
797
	return PyMarshal_WriteObjectToString(x);
798 799
}

800
static PyObject *
801
marshal_loads(PyObject *self, PyObject *args)
802 803
{
	RFILE rf;
804
	PyObject *v;
805 806
	char *s;
	int n;
807
	if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
808 809 810 811 812
		return NULL;
	rf.fp = NULL;
	rf.str = args;
	rf.ptr = s;
	rf.end = s + n;
813
	PyErr_Clear();
814
	v = r_object(&rf);
815 816
	if (PyErr_Occurred()) {
		Py_XDECREF(v);
Guido van Rossum's avatar
Guido van Rossum committed
817 818 819 820 821
		v = NULL;
	}
	return v;
}

822
static PyMethodDef marshal_methods[] = {
823 824 825 826
	{"dump",	marshal_dump,	1},
	{"load",	marshal_load,	1},
	{"dumps",	marshal_dumps,	1},
	{"loads",	marshal_loads,	1},
Guido van Rossum's avatar
Guido van Rossum committed
827 828 829 830
	{NULL,		NULL}		/* sentinel */
};

void
831
PyMarshal_Init(void)
Guido van Rossum's avatar
Guido van Rossum committed
832
{
833
	(void) Py_InitModule("marshal", marshal_methods);
Guido van Rossum's avatar
Guido van Rossum committed
834
}