Totally new "freeze" program.

Tools/freeze/findmodules.py

+# Determine the names and filenames of the modules imported by a
+# script, recursively.  This is done by scanning for lines containing
+# import statements.  (The scanning has only superficial knowledge of
+# Python syntax and no knowledge of semantics, so in theory the result
+# may be incorrect -- however this is quite unlikely if you don't
+# intentionally obscure your Python code.)
+
+import os
+import regex
+import string
+import sys
+
+
+# Top-level interface.
+# First argument is the main program (script).
+# Second optional argument is list of modules to be searched as well.
+
+def findmodules(scriptfile, modules = [], path = sys.path):
+	todo = {}
+	todo['__main__'] = scriptfile
+	for name in modules:
+		mod = os.path.basename(name)
+		if mod[-3:] == '.py': mod = mod[:-3]
+		todo[mod] = name
+	done = closure(todo)
+	return done
+
+
+# Compute the closure of scanfile() and findmodule().
+# Return a dictionary mapping module names to filenames.
+# Writes to stderr if a file can't be or read.
+
+def closure(todo):
+	done = {}
+	while todo:
+		newtodo = {}
+		for modname in todo.keys():
+			if not done.has_key(modname):
+				filename = todo[modname]
+				if filename is None:
+					filename = findmodule(modname)
+				done[modname] = filename
+				if filename in ('<builtin>', '<unknown>'):
+					continue
+				try:
+					modules = scanfile(filename)
+				except IOError, msg:
+					sys.stderr.write("%s: %s\n" %
+							 (filename, str(msg)))
+					continue
+				for m in modules:
+					if not done.has_key(m):
+						newtodo[m] = None
+		todo = newtodo
+	return done
+
+
+# Scan a file looking for import statements.
+# Return list of module names.
+# Can raise IOError.
+
+importstr = '\(^\|:\)[ \t]*import[ \t]+\([a-zA-Z0-9_, \t]+\)'
+fromstr   = '\(^\|:\)[ \t]*from[ \t]+\([a-zA-Z0-9_]+\)[ \t]+import[ \t]+'
+isimport = regex.compile(importstr)
+isfrom = regex.compile(fromstr)
+
+def scanfile(filename):
+	allmodules = {}
+	f = open(filename, 'r')
+	try:
+		while 1:
+			line = f.readline()
+			if not line: break # EOF
+			while line[-2:] == '\\\n': # Continuation line
+				line = line[:-2] + ' '
+				line = line + f.readline()
+			if isimport.search(line) >= 0:
+				rawmodules = isimport.group(2)
+				modules = string.splitfields(rawmodules, ',')
+				for i in range(len(modules)):
+					modules[i] = string.strip(modules[i])
+			elif isfrom.search(line) >= 0:
+				modules = [isfrom.group(2)]
+			else:
+				continue
+			for mod in modules:
+				allmodules[mod] = None
+	finally:
+		f.close()
+	return allmodules.keys()
+
+
+# Find the file containing a module, given its name.
+# Return filename, or '<builtin>', or '<unknown>'.
+
+builtins = sys.builtin_module_names
+if 'sys' not in builtins: builtins.append('sys')
+# XXX this table may have to be changed depending on your platform:
+tails = ['.so', 'module.so', '.py', '.pyc']
+
+def findmodule(modname, path = sys.path):
+	if modname in builtins: return '<builtin>'
+	for dirname in path:
+		for tail in tails:
+			fullname = os.path.join(dirname, modname + tail)
+			try:
+				f = open(fullname, 'r')
+			except IOError:
+				continue
+			f.close()
+			return fullname
+	return '<unknown>'
+
+
+# Test the above functions.
+
+def test():
+	if not sys.argv[1:]:
+		print 'usage: python findmodules.py scriptfile [morefiles ...]'
+		sys.exit(2)
+	done = findmodules(sys.argv[1], sys.argv[2:])
+	items = done.items()
+	items.sort()
+	for mod, file in [('Module', 'File')] + items:
+		print "%-15s %s" % (mod, file)
+
+if __name__ == '__main__':
+	test()

Tools/freeze/makeconfig.py

+import regex
+
+
+# Write the config.c file
+
+never = ['marshal', '__main__', '__builtin__', 'sys']
+
+def makeconfig(infp, outfp, modules):
+	m1 = regex.compile('-- ADDMODULE MARKER 1 --')
+	m2 = regex.compile('-- ADDMODULE MARKER 2 --')
+	while 1:
+		line = infp.readline()
+		if not line: break
+		outfp.write(line)
+		if m1 and m1.search(line) >= 0:
+			m1 = None
+			for mod in modules:
+				if mod in never:
+					continue
+				outfp.write('extern void init%s();\n' % mod)
+		elif m2 and m2.search(line) >= 0:
+			m2 = None
+			for mod in modules:
+				if mod in never:
+					continue
+				outfp.write('\t{"%s", init%s},\n' %
+					    (mod, mod))
+	if m1:
+		sys.stderr.write('MARKER 1 never found\n')
+	elif m2:
+		sys.stderr.write('MARKER 2 never found\n')
+
+
+# Test program.
+
+def test():
+	import sys
+	if not sys.argv[3:]:
+		print 'usage: python makeconfig.py config.c.in outputfile',
+		print 'modulename ...'
+		sys.exit(2)
+	if sys.argv[1] == '-':
+		infp = sys.stdin
+	else:
+		infp = open(sys.argv[1])
+	if sys.argv[2] == '-':
+		outfp = sys.stdout
+	else:
+		outfp = open(sys.argv[2], 'w')
+	makeconfig(infp, outfp, sys.argv[3:])
+	if outfp != sys.stdout:
+		outfp.close()
+	if infp != sys.stdin:
+		infp.close()
+
+if __name__ == '__main__':
+	test()

Tools/freeze/makefreeze.py

+import marshal
+
+
+# Write a file containing frozen code for the modules in the dictionary.
+
+header = """
+struct frozen {
+	char *name;
+	unsigned char *code;
+	int size;
+} frozen_modules[] = {
+"""
+trailer = """\
+	{0, 0, 0} /* sentinel */
+};
+"""
+
+def makefreeze(outfp, dict):
+	done = []
+	mods = dict.keys()
+	mods.sort()
+	for mod in mods:
+		modfn = dict[mod]
+		try:
+			str = makecode(modfn)
+		except IOError, msg:
+			sys.stderr.write("%s: %s\n" % (modfn, str(msg)))
+			continue
+		if str:
+			done.append(mod, len(str))
+			writecode(outfp, mod, str)
+	outfp.write(header)
+	for mod, size in done:
+		outfp.write('\t{"%s", M_%s, %d},\n' % (mod, mod, size))
+	outfp.write(trailer)
+
+
+# Return code string for a given module -- either a .py or a .pyc
+# file.  Return either a string or None (if it's not Python code).
+# May raise IOError.
+
+def makecode(filename):
+	if filename[-3:] == '.py':
+		f = open(filename, 'r')
+		try:
+			text = f.read()
+			code = compile(text, filename, 'exec')
+		finally:
+			f.close()
+		return marshal.dumps(code)
+	if filename[-4:] == '.pyc':
+		f = open(filename, 'rb')
+		try:
+			f.seek(8)
+			str = f.read()
+		finally:
+			f.close()
+		return str
+	# Can't generate code for this extension
+	return None
+
+
+# Write a C initializer for a module containing the frozen python code.
+# The array is called M_<mod>.
+
+def writecode(outfp, mod, str):
+	outfp.write('static unsigned char M_%s[] = {' % mod)
+	for i in range(0, len(str), 16):
+		outfp.write('\n\t')
+		for c in str[i:i+16]:
+			outfp.write('%d,' % ord(c))
+	outfp.write('\n};\n')
+
+
+# Test for the above functions.
+
+def test():
+	import os
+	import sys
+	if not sys.argv[1:]:
+		print 'usage: python freezepython.py file.py(c) ...'
+		sys.exit(2)
+	dict = {}
+	for arg in sys.argv[1:]:
+		base = os.path.basename(arg)
+		mod, ext = os.path.splitext(base)
+		dict[mod] = arg
+	makefreeze(sys.stdout, dict)
+
+if __name__ == '__main__':
+	test()

Tools/freeze/makemakefile.py

+# Write the actual Makefile.
+
+import os
+import string
+
+def makemakefile(outfp, makevars, files, target):
+	outfp.write("# Makefile generated by freeze.py script\n\n")
+
+	keys = makevars.keys()
+	keys.sort()
+	for key in keys:
+		outfp.write("%s=%s\n" % (key, makevars[key]))
+	outfp.write("\nall: %s\n" % target)
+
+	deps = []
+	for i in range(len(files)):
+		file = files[i]
+		if file[-2:] == '.c':
+			base = os.path.basename(file)
+			dest = base[:-2] + '.o'
+			outfp.write("%s: %s\n" % (dest, file))
+			outfp.write("\t$(CC) $(CFLAGS) -c %s\n" % file)
+			files[i] = dest
+			deps.append(dest)
+
+	outfp.write("\n%s: %s\n" % (target, string.join(deps)))
+	outfp.write("\t$(CC) %s -o %s\n" % (string.join(files), target))

Tools/freeze/parsesetup.py

+# Parse Makefiles and Python Setup(.in) files.
+
+import regex
+import string
+
+
+# Extract variable definitions from a Makefile.
+# Return a dictionary mapping names to values.
+# May raise IOError.
+
+makevardef = regex.compile('^\([a-zA-Z0-9_]+\)[ \t]*=\(.*\)')
+
+def getmakevars(filename):
+	variables = {}
+	fp = open(filename)
+	try:
+		while 1:
+			line = fp.readline()
+			if not line:
+				break
+			if makevardef.match(line) < 0:
+				continue
+			name, value = makevardef.group(1, 2)
+			# Strip trailing comment
+			i = string.find(value, '#')
+			if i >= 0:
+				value = value[:i]
+			value = string.strip(value)
+			variables[name] = value
+	finally:
+		fp.close()
+	return variables
+
+
+# Parse a Python Setup(.in) file.
+# Return two dictionaries, the first mapping modules to their
+# definitions, the second mapping variable names to their values.
+# May raise IOError.
+
+setupvardef = regex.compile('^\([a-zA-Z0-9_]+\)=\(.*\)')
+
+def getsetupinfo(filename):
+	modules = {}
+	variables = {}
+	fp = open(filename)
+	try:
+		while 1:
+			line = fp.readline()
+			if not line:
+				break
+			# Strip comments
+			i = string.find(line, '#')
+			if i >= 0:
+				line = line[:i]
+			if setupvardef.match(line) >= 0:
+				name, value = setupvardef.group(1, 2)
+				variables[name] = string.strip(value)
+			else:
+				words = string.split(line)
+				if words:
+					modules[words[0]] = words[1:]
+	finally:
+		fp.close()
+	return modules, variables
+
+
+# Test the above functions.
+
+def test():
+	import sys
+	import os
+	if not sys.argv[1:]:
+		print 'usage: python parsesetup.py Makefile*|Setup* ...'
+		sys.exit(2)
+	for arg in sys.argv[1:]:
+		base = os.path.basename(arg)
+		if base[:8] == 'Makefile':
+			print 'Make style parsing:', arg
+			v = getmakevars(arg)
+			prdict(v)
+		elif base[:5] == 'Setup':
+			print 'Setup style parsing:', arg
+			m, v = getsetupinfo(arg)
+			prdict(m)
+			prdict(v)
+		else:
+			print arg, 'is neither a Makefile nor a Setup file'
+			print '(name must begin with "Makefile" or "Setup")'
+
+def prdict(d):
+	keys = d.keys()
+	keys.sort()
+	for key in keys:
+		value = d[key]
+		print "%-15s" % key, str(value)
+
+if __name__ == '__main__':
+	test()