codekingpro/portable-devtools
115k
1"""
2Build a c-extension module on-the-fly in tests.
3See build_and_import_extensions for usage hints
4
5"""
6
7import os
8import pathlib
9import subprocess
10import sys
11import sysconfig
12import textwrap
13
14__all__ = ['build_and_import_extension', 'compile_extension_module']
15
16
17def build_and_import_extension(
18 modname, functions, *, prologue="", build_dir=None,
19 include_dirs=None, more_init=""):
20 """
21 Build and imports a c-extension module `modname` from a list of function
22 fragments `functions`.
23
24
25 Parameters
26 ----------
27 functions : list of fragments
28 Each fragment is a sequence of func_name, calling convention, snippet.
29 prologue : string
30 Code to precede the rest, usually extra ``#include`` or ``#define``
31 macros.
32 build_dir : pathlib.Path
33 Where to build the module, usually a temporary directory
34 include_dirs : list
35 Extra directories to find include files when compiling
36 more_init : string
37 Code to appear in the module PyMODINIT_FUNC
38
39 Returns
40 -------
41 out: module
42 The module will have been loaded and is ready for use
43
44 Examples
45 --------
46 >>> functions = [("test_bytes", "METH_O", \"\"\"
47 if ( !PyBytesCheck(args)) {
48 Py_RETURN_FALSE;
49 }
50 Py_RETURN_TRUE;
51 \"\"\")]
52 >>> mod = build_and_import_extension("testme", functions)
53 >>> assert not mod.test_bytes('abc')
54 >>> assert mod.test_bytes(b'abc')
55 """
56 if include_dirs is None:
57 include_dirs = []
58 body = prologue + _make_methods(functions, modname)
59 init = """
60 PyObject *mod = PyModule_Create(&moduledef);
61 #ifdef Py_GIL_DISABLED
62 PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED);
63 #endif
64 """
65 if not build_dir:
66 build_dir = pathlib.Path('.')
67 if more_init:
68 init += """#define INITERROR return NULL
69 """
70 init += more_init
71 init += "\nreturn mod;"
72 source_string = _make_source(modname, init, body)
73 mod_so = compile_extension_module(
74 modname, build_dir, include_dirs, source_string)
75 import importlib.util
76 spec = importlib.util.spec_from_file_location(modname, mod_so)
77 foo = importlib.util.module_from_spec(spec)
78 spec.loader.exec_module(foo)
79 return foo
80
81
82def compile_extension_module(
83 name, builddir, include_dirs,
84 source_string, libraries=None, library_dirs=None):
85 """
86 Build an extension module and return the filename of the resulting
87 native code file.
88
89 Parameters
90 ----------
91 name : string
92 name of the module, possibly including dots if it is a module inside a
93 package.
94 builddir : pathlib.Path
95 Where to build the module, usually a temporary directory
96 include_dirs : list
97 Extra directories to find include files when compiling
98 libraries : list
99 Libraries to link into the extension module
100 library_dirs: list
101 Where to find the libraries, ``-L`` passed to the linker
102 """
103 modname = name.split('.')[-1]
104 dirname = builddir / name
105 dirname.mkdir(exist_ok=True)
106 cfile = _convert_str_to_file(source_string, dirname)
107 include_dirs = include_dirs or []
108 libraries = libraries or []
109 library_dirs = library_dirs or []
110
111 return _c_compile(
112 cfile, outputfilename=dirname / modname,
113 include_dirs=include_dirs, libraries=libraries,
114 library_dirs=library_dirs,
115 )
116
117
118def _convert_str_to_file(source, dirname):
119 """Helper function to create a file ``source.c`` in `dirname` that contains
120 the string in `source`. Returns the file name
121 """
122 filename = dirname / 'source.c'
123 with filename.open('w') as f:
124 f.write(str(source))
125 return filename
126
127
128def _make_methods(functions, modname):
129 """ Turns the name, signature, code in functions into complete functions
130 and lists them in a methods_table. Then turns the methods_table into a
131 ``PyMethodDef`` structure and returns the resulting code fragment ready
132 for compilation
133 """
134 methods_table = []
135 codes = []
136 for funcname, flags, code in functions:
137 cfuncname = f"{modname}_{funcname}"
138 if 'METH_KEYWORDS' in flags:
139 signature = '(PyObject *self, PyObject *args, PyObject *kwargs)'
140 else:
141 signature = '(PyObject *self, PyObject *args)'
142 methods_table.append(
143 "{\"%s\", (PyCFunction)%s, %s}," % (funcname, cfuncname, flags))
144 func_code = f"""
145 static PyObject* {cfuncname}{signature}
146 {{
147 {code}
148 }}
149 """
150 codes.append(func_code)
151
152 body = "\n".join(codes) + """
153 static PyMethodDef methods[] = {
154 %(methods)s
155 { NULL }
156 };
157 static struct PyModuleDef moduledef = {
158 PyModuleDef_HEAD_INIT,
159 "%(modname)s", /* m_name */
160 NULL, /* m_doc */
161 -1, /* m_size */
162 methods, /* m_methods */
163 };
164 """ % {'methods': '\n'.join(methods_table), 'modname': modname}
165 return body
166
167
168def _make_source(name, init, body):
169 """ Combines the code fragments into source code ready to be compiled
170 """
171 code = """
172 #include <Python.h>
173
174 %(body)s
175
176 PyMODINIT_FUNC
177 PyInit_%(name)s(void) {
178 %(init)s
179 }
180 """ % {
181 'name': name, 'init': init, 'body': body,
182 }
183 return code
184
185
186def _c_compile(cfile, outputfilename, include_dirs, libraries,
187 library_dirs):
188 link_extra = []
189 if sys.platform == 'win32':
190 compile_extra = ["/we4013"]
191 link_extra.append('/DEBUG') # generate .pdb file
192 elif sys.platform.startswith('linux'):
193 compile_extra = [
194 "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
195 else:
196 compile_extra = []
197
198 return build(
199 cfile, outputfilename,
200 compile_extra, link_extra,
201 include_dirs, libraries, library_dirs)
202
203
204def build(cfile, outputfilename, compile_extra, link_extra,
205 include_dirs, libraries, library_dirs):
206 "use meson to build"
207
208 build_dir = cfile.parent / "build"
209 os.makedirs(build_dir, exist_ok=True)
210 with open(cfile.parent / "meson.build", "wt") as fid:
211 link_dirs = ['-L' + d for d in library_dirs]
212 fid.write(textwrap.dedent(f"""\
213 project('foo', 'c')
214 py = import('python').find_installation(pure: false)
215 py.extension_module(
216 '{outputfilename.parts[-1]}',
217 '{cfile.parts[-1]}',
218 c_args: {compile_extra},
219 link_args: {link_dirs},
220 include_directories: {include_dirs},
221 )
222 """))
223 native_file_name = cfile.parent / ".mesonpy-native-file.ini"
224 with open(native_file_name, "wt") as fid:
225 fid.write(textwrap.dedent(f"""\
226 [binaries]
227 python = '{sys.executable}'
228 """))
229 if sys.platform == "win32":
230 subprocess.check_call(["meson", "setup",
231 "--buildtype=release",
232 "--vsenv", ".."],
233 cwd=build_dir,
234 )
235 else:
236 subprocess.check_call(["meson", "setup", "--vsenv",
237 "..", f'--native-file={os.fspath(native_file_name)}'],
238 cwd=build_dir
239 )
240
241 so_name = outputfilename.parts[-1] + get_so_suffix()
242 subprocess.check_call(["meson", "compile"], cwd=build_dir)
243 os.rename(str(build_dir / so_name), cfile.parent / so_name)
244 return cfile.parent / so_name
245
246
247def get_so_suffix():
248 ret = sysconfig.get_config_var('EXT_SUFFIX')
249 assert ret
250 return ret
251