1 """This module contains the main part of Instant, the build_module function."""
2
3 import os, sys, shutil, glob
4 from itertools import chain
5
6
7 from output import *
8 from config import header_and_libs_from_pkgconfig
9 from paths import *
10 from signatures import *
11 from cache import *
12 from codegeneration import *
13 from locking import get_lock, release_lock
14
16 instant_assert(isinstance(x, str),
17 "In instant.build_module: Expecting string.")
18
20 instant_assert(isinstance(x, bool),
21 "In instant.build_module: Expecting bool.")
22
24 instant_assert(isinstance(x, (list, tuple)),
25 "In instant.build_module: Expecting sequence.")
26 instant_assert(all(isinstance(i, str) for i in x),
27 "In instant.build_module: Expecting sequence of strings.")
28
32
34 if isinstance(x, str):
35 x = x.split()
36 return strip_strings(x)
37
38
40 """Copy a list of files from a source directory to a destination directory.
41 This may seem a bit complicated, but a lot of this code is error checking."""
42 if os.path.exists(dest):
43 overwriting = set(files) & set(glob.glob(os.path.join(dest, "*")))
44 if overwriting:
45 instant_warning("In instant.copy_files: Path '%s' already exists, "\
46 "overwriting existing files: %r." % (dest, list(overwriting)))
47 else:
48 os.mkdir(dest)
49
50 if source != dest:
51 instant_debug("In instant.copy_files: Copying files %r from %r to %r"\
52 % (files, source, dest))
53
54 for f in files:
55 a = os.path.join(source, f)
56 b = os.path.join(dest, f)
57 instant_assert(a != b, "In instant.copy_files: Seems like the "\
58 "input files are absolute paths, should be relative to "\
59 "source. (%r, %r)" % (a, b))
60 instant_assert(os.path.isfile(a), "In instant.copy_files: "\
61 "Missing source file '%s'." % a)
62 if os.path.isfile(b):
63 os.remove(b)
64 shutil.copyfile(a, b)
65
66
67 -def recompile(modulename, module_path, setup_name, new_compilation_checksum):
68 """Recompile module if the new checksum is different from
69 the one in the checksum file in the module directory."""
70
71 compilation_checksum_filename = "%s.checksum" % modulename
72 if os.path.exists(compilation_checksum_filename):
73 checksum_file = open(compilation_checksum_filename)
74 old_compilation_checksum = checksum_file.readline()
75 checksum_file.close()
76 if old_compilation_checksum == new_compilation_checksum:
77 return
78
79
80 (swig_stat, swig_out) = get_status_output("swig -version")
81 if swig_stat != 0:
82 instant_error("In instant.recompile: Could not find swig!"\
83 " You can download swig from http://www.swig.org")
84
85
86 compile_log_filename = os.path.join(module_path, "compile.log")
87 compile_log_file = open(compile_log_filename, "w")
88 try:
89
90 cmd = "python %s build_ext" % setup_name
91 instant_info("--- Instant: compiling ---")
92 instant_debug("cmd = %s" % cmd)
93 ret, output = get_status_output(cmd)
94 compile_log_file.write(output)
95 compile_log_file.flush()
96 if ret != 0:
97 if os.path.exists(compilation_checksum_filename):
98 os.remove(compilation_checksum_filename)
99 instant_error("In instant.recompile: The module did not "\
100 "compile, see '%s'" % compile_log_filename)
101
102
103 cmd = "python %s install --install-platlib=." % setup_name
104 instant_debug("cmd = %s" % cmd)
105 ret, output = get_status_output(cmd)
106 compile_log_file.write(output)
107 compile_log_file.flush()
108 if ret != 0:
109 if os.path.exists(compilation_checksum_filename):
110 os.remove(compilation_checksum_filename)
111 instant_error("In instant.recompile: Could not 'install' "\
112 "the module, see '%s'" % compile_log_filename)
113 finally:
114 compile_log_file.close()
115
116
117 write_file(compilation_checksum_filename, new_compilation_checksum)
118
119
121 "Copy module directory to cache."
122
123
124
125 lock = get_lock(cache_dir, modulename)
126
127
128 cache_module_path = os.path.join(cache_dir, modulename)
129 if os.path.exists(cache_module_path):
130
131 instant_warning("In instant.build_module: Path '%s' already exists,"\
132 " but module wasn't found in cache previously. Not overwriting,"\
133 " assuming this module is valid." % cache_module_path)
134
135 release_lock(lock)
136 return cache_module_path
137
138
139
140
141
142 instant_assert(os.path.isdir(module_path), "In instant.build_module:"\
143 " Cannot copy non-existing directory %r!" % module_path)
144 instant_assert(not os.path.isdir(cache_module_path),
145 "In instant.build_module: Cache directory %r shouldn't exist "\
146 "at this point!" % cache_module_path)
147 instant_debug("In instant.build_module: Copying built module from %r"\
148 " to cache at %r" % (module_path, cache_module_path))
149
150
151 shutil.copytree(module_path, cache_module_path)
152 delete_temp_dir()
153 release_lock(lock)
154 return cache_module_path
155
156
157 -def build_module(modulename=None, source_directory=".",
158 code="", init_code="",
159 additional_definitions="", additional_declarations="",
160 sources=[], wrap_headers=[],
161 local_headers=[], system_headers=[],
162 include_dirs=['.'], library_dirs=[], libraries=[],
163 swigargs=['-c++', '-fcompact', '-O', '-I.', '-small'],
164 swig_include_dirs = [],
165 cppargs=['-O2'], lddargs=[],
166 object_files=[], arrays=[],
167 generate_interface=True, generate_setup=True,
168 signature=None, cache_dir=None):
169 """Generate and compile a module from C/C++ code using SWIG.
170
171 Arguments:
172 ==========
173 The keyword arguments are as follows:
174 - B{modulename}:
175 - The name you want for the module.
176 If specified, the module will not be cached.
177 If missing, a name will be constructed based on
178 a checksum of the other arguments, and the module
179 will be placed in the global cache. String.
180 - B{source_directory}:
181 - The directory where user supplied files reside. The files
182 given in B{sources}, B{wrap_headers}, and B{local_headers}
183 are expected to exist in this directory. String.
184 - B{code}:
185 - A string containing C or C++ code to be compiled and wrapped. String.
186 - B{init_code}:
187 - Code that should be executed when the Instant module is
188 imported. This code is inserted in the SWIG interface file, and is
189 used for instance for calling C{import_array()} used for the
190 initialization of NumPy arrays. String.
191 - B{additional_definitions}:
192 - Additional definitions (typically needed for inheritance)
193 for interface file. These definitions should be given as triple-quoted
194 strings in the case they span multiple lines, and are placed both in the
195 initial block for C/C++ code (C{%{,%}}-block), and the main section
196 of the interface file. String.
197 - B{additional_declarations}:
198 - Additional declarations (typically needed for inheritance)
199 for interface file. These declarations should be given as triple-quoted
200 strings in the case they span multiple lines, and are plaves in the main
201 section of the interface file. String.
202 - B{sources}:
203 - Source files to compile and link with the module. These
204 files are compiled togehter with the SWIG-generated wrapper file into
205 the final library file. Should reside in directory specified in
206 B{source_directory}. List of strings.
207 - B{wrap_headers}:
208 - Local header files that should be wrapped by SWIG. The
209 files specified will be included both in the initial block for C/C++ code
210 (with a C directive) and in the main section of the interface file (with
211 a SWIG directive). Should reside in directory specified in
212 B{source_directory}. List of strings.
213 - B{local_headers}:
214 - Local header files required to compile the wrapped
215 code. The files specified will be included in the initial block for
216 C/C++ code (with a C directive). Should reside in directory specified in
217 B{source_directory}. List of strings.
218 - B{system_headers}:
219 - System header files required to compile the wrapped
220 code. The files specified will be included in the initial block for C/C++
221 code (with a C directive). List of strings.
222 - B{include_dirs}:
223 - Directories to search for header files for building the
224 extension module. Needs to be absolute path names. List of strings.
225 - B{library_dirs}:
226 - Directories to search for libraries (C{-l}) for building
227 the extension module. Needs to be absolute paths. List of strings.
228 - B{libraries}:
229 - Libraries needed by the Instant module. The libraries will
230 be linked in from the shared object file. The initial C{-l} is added
231 automatically. List of strings.
232 - B{swigargs}:
233 - List of arguments to swig, e.g. C{["-lpointers.i"]}
234 to include the SWIG pointers.i library.
235 - B{swig_include_dirs}:
236 - A list of directories to include in the 'swig' command.
237 - B{cppargs}:
238 - List of arguments to the compiler, e.g. C{["-Wall", "-fopenmp"]}.
239 - B{lddargs}:
240 - List of arguments to the linker, e.g. C{["-E", "-U"]}.
241 - B{object_files}:
242 - If you want to compile the files yourself. TODO: Not yet supported.
243 - B{arrays}:
244 - A nested list describing the C arrays to be made from NumPy arrays.
245 If the NumPy array is 1D, the inner list should contain strings with
246 the variable names for length of the array and the array itself.
247 If the NumPy array is a matrix or a tensor, the inner list should
248 contain strings with variable names for the number of dimensions,
249 the length in each dimension, and the array itself, respectively.
250 - B{generate_interface}:
251 - A bool to indicate if you want to generate the interface files.
252 - B{generate_setup}:
253 - A bool to indicate if you want to generate the setup.py file.
254 - B{signature}:
255 - A signature string to identify the form instead of the source code.
256 - B{cache_dir}:
257 - A directory to look for cached modules and place new ones.
258 If missing, a default directory is used. Note that the module
259 will not be cached if B{modulename} is specified.
260 The cache directory should not be used for anything else.
261 """
262
263
264 original_path = os.getcwd()
265
266
267
268 instant_assert(modulename is None or isinstance(modulename, str),
269 "In instant.build_module: Expecting modulename to be string or None.")
270 assert_is_str(source_directory)
271 source_directory = os.path.abspath(source_directory)
272 assert_is_str(code)
273 assert_is_str(init_code)
274 assert_is_str(additional_definitions)
275 assert_is_str(additional_declarations)
276 sources = strip_strings(sources)
277 wrap_headers = strip_strings(wrap_headers)
278 local_headers = strip_strings(local_headers)
279 system_headers = strip_strings(system_headers)
280 include_dirs = strip_strings(include_dirs)
281 library_dirs = strip_strings(library_dirs)
282 libraries = strip_strings(libraries)
283 swigargs = arg_strings(swigargs)
284 swig_include_dirs = strip_strings(swig_include_dirs)
285 cppargs = arg_strings(cppargs)
286 lddargs = arg_strings(lddargs)
287 object_files = strip_strings(object_files)
288 arrays = [strip_strings(a) for a in arrays]
289 assert_is_bool(generate_interface)
290 assert_is_bool(generate_setup)
291 instant_assert( signature is None \
292 or isinstance(signature, str) \
293 or hasattr(signature, "signature"),
294 "In instant.build_module: Expecting modulename to be string or None.")
295 instant_assert(not (signature is not None and modulename is not None),
296 "In instant.build_module: Can't have both modulename and signature.")
297
298
299
300 cache_dir = validate_cache_dir(cache_dir)
301
302
303 csrcs = [f for f in sources if f.endswith('.c') or f.endswith('.C')]
304 cppsrcs = [f for f in sources if f.endswith('.cpp') or f.endswith('.cxx')]
305 instant_assert(len(csrcs) + len(cppsrcs) == len(sources),
306 "In instant.build_module: Source files must have '.c' or '.cpp' suffix")
307
308
309 instant_debug('In instant.build_module:')
310 instant_debug('::: Begin Arguments :::')
311 instant_debug(' modulename: %r' % modulename)
312 instant_debug(' source_directory: %r' % source_directory)
313 instant_debug(' code: %r' % code)
314 instant_debug(' init_code: %r' % init_code)
315 instant_debug(' additional_definitions: %r' % additional_definitions)
316 instant_debug(' additional_declarations: %r' % additional_declarations)
317 instant_debug(' sources: %r' % sources)
318 instant_debug(' csrcs: %r' % csrcs)
319 instant_debug(' cppsrcs: %r' % cppsrcs)
320 instant_debug(' wrap_headers: %r' % wrap_headers)
321 instant_debug(' local_headers: %r' % local_headers)
322 instant_debug(' system_headers: %r' % system_headers)
323 instant_debug(' include_dirs: %r' % include_dirs)
324 instant_debug(' library_dirs: %r' % library_dirs)
325 instant_debug(' libraries: %r' % libraries)
326 instant_debug(' swigargs: %r' % swigargs)
327 instant_debug(' swig_include_dirs: %r' % swig_include_dirs)
328 instant_debug(' cppargs: %r' % cppargs)
329 instant_debug(' lddargs: %r' % lddargs)
330 instant_debug(' object_files: %r' % object_files)
331 instant_debug(' arrays: %r' % arrays)
332 instant_debug(' generate_interface: %r' % generate_interface)
333 instant_debug(' generate_setup: %r' % generate_setup)
334 instant_debug(' signature: %r' % signature)
335 instant_debug(' cache_dir: %r' % cache_dir)
336 instant_debug('::: End Arguments :::')
337
338
339
340
341
342 if modulename is None:
343
344 if signature is None:
345
346
347
348 checksum_args = ( \
349
350
351
352
353 code, init_code,
354 additional_definitions,
355 additional_declarations,
356
357
358
359 system_headers,
360 include_dirs, library_dirs, libraries,
361 swig_include_dirs, swigargs, cppargs, lddargs,
362 object_files, arrays,
363 generate_interface, generate_setup,
364
365
366 )
367 allfiles = sources + wrap_headers + local_headers
368 allfiles = [os.path.join(source_directory, file) for file in allfiles]
369 text = "\n".join((str(a) for a in checksum_args))
370 signature = modulename_from_checksum(compute_checksum(text, allfiles))
371 modulename = signature
372 moduleids = [signature]
373 else:
374 module, moduleids = check_memory_cache(signature)
375 if module: return module
376 modulename = moduleids[-1]
377
378
379 module = check_disk_cache(modulename, cache_dir, moduleids)
380 if module: return module
381
382
383 module_path = os.path.join(get_temp_dir(), modulename)
384 instant_assert(not os.path.exists(module_path),
385 "In instant.build_module: Not expecting module_path to exist: '%s'"\
386 % module_path)
387 os.mkdir(module_path)
388 use_cache = True
389 else:
390 use_cache = False
391 moduleids = []
392 module_path = os.path.join(original_path, modulename)
393 if not os.path.exists(module_path):
394 os.mkdir(module_path)
395
396
397
398
399
400
401
402
403
404
405
406 try:
407
408
409 module_path = os.path.abspath(module_path)
410 files_to_copy = sources + wrap_headers + local_headers + object_files
411 copy_files(source_directory, module_path, files_to_copy)
412
413
414
415 os.chdir(module_path)
416
417
418 write_file("__init__.py", "from %s import *" % modulename)
419
420
421 ifile_name = "%s.i" % modulename
422 if generate_interface:
423 write_interfacefile(ifile_name, modulename, code, init_code,
424 additional_definitions, additional_declarations, system_headers,
425 local_headers, wrap_headers, arrays)
426
427
428 setup_name = "setup.py"
429 if generate_setup:
430 write_setup(setup_name, modulename, csrcs, cppsrcs, local_headers, \
431 include_dirs, library_dirs, libraries, swig_include_dirs, \
432 swigargs, cppargs, lddargs)
433
434
435
436
437
438
439
440
441
442
443
444
445 checksum_args = ( \
446
447
448
449
450
451
452
453
454
455
456 system_headers,
457 include_dirs, library_dirs, libraries,
458 swigargs, swig_include_dirs, cppargs, lddargs,
459 object_files,
460
461
462
463
464 )
465 text = "\n".join((str(a) for a in checksum_args))
466 allfiles = sources + wrap_headers + local_headers + [ifile_name]
467 new_compilation_checksum = compute_checksum(text, allfiles)
468
469
470 recompile(modulename, module_path, setup_name, new_compilation_checksum)
471
472
473
474
475 if use_cache:
476 module_path = copy_to_cache(module_path, cache_dir, modulename)
477
478
479
480 if use_cache:
481 lock = get_lock(cache_dir, modulename)
482 module = import_and_cache_module(module_path, modulename, moduleids)
483 if use_cache:
484 release_lock(lock)
485 if not module:
486 instant_error("Failed to import newly compiled module!")
487
488 instant_debug("In instant.build_module: Returning %s from build_module."\
489 % module)
490 return module
491
492
493 finally:
494
495 os.chdir(original_path)
496
497 instant_error("In instant.build_module: Should never reach this point!")
498
499