1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 """Python modules manipulation utility functions.
20
21 :type PY_SOURCE_EXTS: tuple(str)
22 :var PY_SOURCE_EXTS: list of possible python source file extension
23
24 :type STD_LIB_DIR: str
25 :var STD_LIB_DIR: directory where standard modules are located
26
27 :type BUILTIN_MODULES: dict
28 :var BUILTIN_MODULES: dictionary with builtin module names as key
29 """
30
31 __docformat__ = "restructuredtext en"
32
33 import sys
34 import os
35 from os.path import splitext, join, abspath, isdir, dirname, exists, basename
36 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
37 from distutils.sysconfig import get_config_var, get_python_lib, get_python_version
38 from distutils.errors import DistutilsPlatformError
39
40 from six.moves import range
41
42 try:
43 import zipimport
44 except ImportError:
45 zipimport = None
46
47 ZIPFILE = object()
48
49 from logilab.common import STD_BLACKLIST, _handle_blacklist
50
51
52
53
54
55
56
57 if sys.platform.startswith('win'):
58 PY_SOURCE_EXTS = ('py', 'pyw')
59 PY_COMPILED_EXTS = ('dll', 'pyd')
60 else:
61 PY_SOURCE_EXTS = ('py',)
62 PY_COMPILED_EXTS = ('so',)
63
64 try:
65 STD_LIB_DIR = get_python_lib(standard_lib=True)
66
67
68 except DistutilsPlatformError:
69 STD_LIB_DIR = '//'
70
71 EXT_LIB_DIR = get_python_lib()
72
73 BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True)
74
75
77 """exception raised when we are not able to get a python
78 source file for a precompiled file
79 """
80
83 self.module = module
84 self.obj = obj
85 self._imported = None
86
88 if self._imported is None:
89 self._imported = getattr(load_module_from_name(self.module),
90 self.obj)
91 return self._imported
92
94 try:
95 return super(LazyObject, self).__getattribute__(attr)
96 except AttributeError as ex:
97 return getattr(self._getobj(), attr)
98
100 return self._getobj()(*args, **kwargs)
101
102
104 """Load a Python module from its name.
105
106 :type dotted_name: str
107 :param dotted_name: python name of a module or package
108
109 :type path: list or None
110 :param path:
111 optional list of path where the module or package should be
112 searched (use sys.path if nothing or None is given)
113
114 :type use_sys: bool
115 :param use_sys:
116 boolean indicating whether the sys.modules dictionary should be
117 used or not
118
119
120 :raise ImportError: if the module or package is not found
121
122 :rtype: module
123 :return: the loaded module
124 """
125 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
126
127
129 """Load a python module from its splitted name.
130
131 :type parts: list(str) or tuple(str)
132 :param parts:
133 python name of a module or package splitted on '.'
134
135 :type path: list or None
136 :param path:
137 optional list of path where the module or package should be
138 searched (use sys.path if nothing or None is given)
139
140 :type use_sys: bool
141 :param use_sys:
142 boolean indicating whether the sys.modules dictionary should be used or not
143
144 :raise ImportError: if the module or package is not found
145
146 :rtype: module
147 :return: the loaded module
148 """
149 if use_sys:
150 try:
151 return sys.modules['.'.join(parts)]
152 except KeyError:
153 pass
154 modpath = []
155 prevmodule = None
156 for part in parts:
157 modpath.append(part)
158 curname = '.'.join(modpath)
159 module = None
160 if len(modpath) != len(parts):
161
162 module = sys.modules.get(curname)
163 elif use_sys:
164
165 module = sys.modules.get(curname)
166 if module is None:
167 mp_file, mp_filename, mp_desc = find_module(part, path)
168 module = load_module(curname, mp_file, mp_filename, mp_desc)
169 if prevmodule:
170 setattr(prevmodule, part, module)
171 _file = getattr(module, '__file__', '')
172 prevmodule = module
173 if not _file and _is_namespace(curname):
174 continue
175 if not _file and len(modpath) != len(parts):
176 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) )
177 path = [dirname( _file )]
178 return module
179
180
182 """Load a Python module from it's path.
183
184 :type filepath: str
185 :param filepath: path to the python module or package
186
187 :type path: list or None
188 :param path:
189 optional list of path where the module or package should be
190 searched (use sys.path if nothing or None is given)
191
192 :type use_sys: bool
193 :param use_sys:
194 boolean indicating whether the sys.modules dictionary should be
195 used or not
196
197
198 :raise ImportError: if the module or package is not found
199
200 :rtype: module
201 :return: the loaded module
202 """
203 modpath = modpath_from_file(filepath, extrapath)
204 return load_module_from_modpath(modpath, path, use_sys)
205
206
208 """check there are some __init__.py all along the way"""
209 modpath = []
210 for part in mod_path:
211 modpath.append(part)
212 path = join(path, part)
213 if not _is_namespace('.'.join(modpath)) and not _has_init(path):
214 return False
215 return True
216
217
219 """given a file path return the corresponding splitted module's name
220 (i.e name of a module or package splitted on '.')
221
222 :type filename: str
223 :param filename: file's path for which we want the module's name
224
225 :type extrapath: dict
226 :param extrapath:
227 optional extra search path, with path as key and package name for the path
228 as value. This is usually useful to handle package splitted in multiple
229 directories using __path__ trick.
230
231
232 :raise ImportError:
233 if the corresponding module's name has not been found
234
235 :rtype: list(str)
236 :return: the corresponding splitted module's name
237 """
238 base = splitext(abspath(filename))[0]
239 if extrapath is not None:
240 for path_ in extrapath:
241 path = abspath(path_)
242 if path and base[:len(path)] == path:
243 submodpath = [pkg for pkg in base[len(path):].split(os.sep)
244 if pkg]
245 if _check_init(path, submodpath[:-1]):
246 return extrapath[path_].split('.') + submodpath
247 for path in sys.path:
248 path = abspath(path)
249 if path and base.startswith(path):
250 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]
251 if _check_init(path, modpath[:-1]):
252 return modpath
253 raise ImportError('Unable to find module for %s in %s' % (
254 filename, ', \n'.join(sys.path)))
255
256
257
259 """given a mod path (i.e. splitted module / package name), return the
260 corresponding file, giving priority to source file over precompiled
261 file if it exists
262
263 :type modpath: list or tuple
264 :param modpath:
265 splitted module's name (i.e name of a module or package splitted
266 on '.')
267 (this means explicit relative imports that start with dots have
268 empty strings in this list!)
269
270 :type path: list or None
271 :param path:
272 optional list of path where the module or package should be
273 searched (use sys.path if nothing or None is given)
274
275 :type context_file: str or None
276 :param context_file:
277 context file to consider, necessary if the identifier has been
278 introduced using a relative import unresolvable in the actual
279 context (i.e. modutils)
280
281 :raise ImportError: if there is no such module in the directory
282
283 :rtype: str or None
284 :return:
285 the path to the module's file or None if it's an integrated
286 builtin module such as 'sys'
287 """
288 if context_file is not None:
289 context = dirname(context_file)
290 else:
291 context = context_file
292 if modpath[0] == 'xml':
293
294 try:
295 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)
296 except ImportError:
297 return _file_from_modpath(modpath, path, context)
298 elif modpath == ['os', 'path']:
299
300 return os.path.__file__
301 return _file_from_modpath(modpath, path, context)
302
303
304
306 """given a dotted name return the module part of the name :
307
308 >>> get_module_part('logilab.common.modutils.get_module_part')
309 'logilab.common.modutils'
310
311 :type dotted_name: str
312 :param dotted_name: full name of the identifier we are interested in
313
314 :type context_file: str or None
315 :param context_file:
316 context file to consider, necessary if the identifier has been
317 introduced using a relative import unresolvable in the actual
318 context (i.e. modutils)
319
320
321 :raise ImportError: if there is no such module in the directory
322
323 :rtype: str or None
324 :return:
325 the module part of the name or None if we have not been able at
326 all to import the given name
327
328 XXX: deprecated, since it doesn't handle package precedence over module
329 (see #10066)
330 """
331
332 if dotted_name.startswith('os.path'):
333 return 'os.path'
334 parts = dotted_name.split('.')
335 if context_file is not None:
336
337
338 if parts[0] in BUILTIN_MODULES:
339 if len(parts) > 2:
340 raise ImportError(dotted_name)
341 return parts[0]
342
343 path = None
344 starti = 0
345 if parts[0] == '':
346 assert context_file is not None, \
347 'explicit relative import, but no context_file?'
348 path = []
349 starti = 1
350 while parts[starti] == '':
351 starti += 1
352 context_file = dirname(context_file)
353 for i in range(starti, len(parts)):
354 try:
355 file_from_modpath(parts[starti:i+1],
356 path=path, context_file=context_file)
357 except ImportError:
358 if not i >= max(1, len(parts) - 2):
359 raise
360 return '.'.join(parts[:i])
361 return dotted_name
362
363
365 """given a package directory return a list of all available python
366 modules in the package and its subpackages
367
368 :type package: str
369 :param package: the python name for the package
370
371 :type src_directory: str
372 :param src_directory:
373 path of the directory corresponding to the package
374
375 :type blacklist: list or tuple
376 :param blacklist:
377 optional list of files or directory to ignore, default to
378 the value of `logilab.common.STD_BLACKLIST`
379
380 :rtype: list
381 :return:
382 the list of all available python modules in the package and its
383 subpackages
384 """
385 modules = []
386 for directory, dirnames, filenames in os.walk(src_directory):
387 _handle_blacklist(blacklist, dirnames, filenames)
388
389 if not '__init__.py' in filenames:
390 dirnames[:] = ()
391 continue
392 if directory != src_directory:
393 dir_package = directory[len(src_directory):].replace(os.sep, '.')
394 modules.append(package + dir_package)
395 for filename in filenames:
396 if _is_python_file(filename) and filename != '__init__.py':
397 src = join(directory, filename)
398 module = package + src[len(src_directory):-3]
399 modules.append(module.replace(os.sep, '.'))
400 return modules
401
402
403
405 """given a package directory return a list of all available python
406 module's files in the package and its subpackages
407
408 :type src_directory: str
409 :param src_directory:
410 path of the directory corresponding to the package
411
412 :type blacklist: list or tuple
413 :param blacklist:
414 optional list of files or directory to ignore, default to the value of
415 `logilab.common.STD_BLACKLIST`
416
417 :rtype: list
418 :return:
419 the list of all available python module's files in the package and
420 its subpackages
421 """
422 files = []
423 for directory, dirnames, filenames in os.walk(src_directory):
424 _handle_blacklist(blacklist, dirnames, filenames)
425
426 if not '__init__.py' in filenames:
427 dirnames[:] = ()
428 continue
429 for filename in filenames:
430 if _is_python_file(filename):
431 src = join(directory, filename)
432 files.append(src)
433 return files
434
435
437 """given a python module's file name return the matching source file
438 name (the filename will be returned identically if it's a already an
439 absolute path to a python source file...)
440
441 :type filename: str
442 :param filename: python module's file name
443
444
445 :raise NoSourceFile: if no source file exists on the file system
446
447 :rtype: str
448 :return: the absolute path of the source file if it exists
449 """
450 base, orig_ext = splitext(abspath(filename))
451 for ext in PY_SOURCE_EXTS:
452 source_path = '%s.%s' % (base, ext)
453 if exists(source_path):
454 return source_path
455 if include_no_ext and not orig_ext and exists(base):
456 return base
457 raise NoSourceFile(filename)
458
459
461 """remove submodules of `directories` from `sys.modules`"""
462 cleaned = []
463 for modname, module in list(sys.modules.items()):
464 modfile = getattr(module, '__file__', None)
465 if modfile:
466 for directory in directories:
467 if modfile.startswith(directory):
468 cleaned.append(modname)
469 del sys.modules[modname]
470 break
471 return cleaned
472
473
475 """
476 rtype: bool
477 return: True if the filename is a python source file
478 """
479 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
480
481
483 """try to guess if a module is a standard python module (by default,
484 see `std_path` parameter's description)
485
486 :type modname: str
487 :param modname: name of the module we are interested in
488
489 :type std_path: list(str) or tuple(str)
490 :param std_path: list of path considered as standard
491
492
493 :rtype: bool
494 :return:
495 true if the module:
496 - is located on the path listed in one of the directory in `std_path`
497 - is a built-in module
498
499 Note: this function is known to return wrong values when inside virtualenv.
500 See https://www.logilab.org/ticket/294756.
501 """
502 modname = modname.split('.')[0]
503 try:
504 filename = file_from_modpath([modname])
505 except ImportError as ex:
506
507
508 return False
509
510
511 if filename is None:
512
513 return not _is_namespace(modname)
514 filename = abspath(filename)
515 if filename.startswith(EXT_LIB_DIR):
516 return False
517 for path in std_path:
518 if filename.startswith(abspath(path)):
519 return True
520 return False
521
522
523
525 """return true if the given module name is relative to the given
526 file name
527
528 :type modname: str
529 :param modname: name of the module we are interested in
530
531 :type from_file: str
532 :param from_file:
533 path of the module from which modname has been imported
534
535 :rtype: bool
536 :return:
537 true if the module has been imported relatively to `from_file`
538 """
539 if not isdir(from_file):
540 from_file = dirname(from_file)
541 if from_file in sys.path:
542 return False
543 try:
544 find_module(modname.split('.')[0], [from_file])
545 return True
546 except ImportError:
547 return False
548
549
550
551
553 """given a mod path (i.e. splitted module / package name), return the
554 corresponding file
555
556 this function is used internally, see `file_from_modpath`'s
557 documentation for more information
558 """
559 assert len(modpath) > 0
560 if context is not None:
561 try:
562 mtype, mp_filename = _module_file(modpath, [context])
563 except ImportError:
564 mtype, mp_filename = _module_file(modpath, path)
565 else:
566 mtype, mp_filename = _module_file(modpath, path)
567 if mtype == PY_COMPILED:
568 try:
569 return get_source_file(mp_filename)
570 except NoSourceFile:
571 return mp_filename
572 elif mtype == C_BUILTIN:
573
574 return None
575 elif mtype == PKG_DIRECTORY:
576 mp_filename = _has_init(mp_filename)
577 return mp_filename
578
580 for filepath, importer in pic.items():
581 if importer is not None:
582 if importer.find_module(modpath[0]):
583 if not importer.find_module('/'.join(modpath)):
584 raise ImportError('No module named %s in %s/%s' % (
585 '.'.join(modpath[1:]), filepath, modpath))
586 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath
587 raise ImportError('No module named %s' % '.'.join(modpath))
588
589 try:
590 import pkg_resources
591 except ImportError:
592 pkg_resources = None
593
594
598
599
601 """get a module type / file path
602
603 :type modpath: list or tuple
604 :param modpath:
605 splitted module's name (i.e name of a module or package splitted
606 on '.'), with leading empty strings for explicit relative import
607
608 :type path: list or None
609 :param path:
610 optional list of path where the module or package should be
611 searched (use sys.path if nothing or None is given)
612
613
614 :rtype: tuple(int, str)
615 :return: the module type flag and the file path for a module
616 """
617
618 try:
619 pic = sys.path_importer_cache
620 _path = (path is None and sys.path or path)
621 for __path in _path:
622 if not __path in pic:
623 try:
624 pic[__path] = zipimport.zipimporter(__path)
625 except zipimport.ZipImportError:
626 pic[__path] = None
627 checkeggs = True
628 except AttributeError:
629 checkeggs = False
630
631 if (_is_namespace(modpath[0]) and modpath[0] in sys.modules):
632
633
634 module = sys.modules[modpath.pop(0)]
635 path = module.__path__
636 if not modpath:
637 return C_BUILTIN, None
638 imported = []
639 while modpath:
640 modname = modpath[0]
641
642
643
644
645
646
647
648
649
650 try:
651 _, mp_filename, mp_desc = find_module(modname, path)
652 except ImportError:
653 if checkeggs:
654 return _search_zip(modpath, pic)[:2]
655 raise
656 else:
657 if checkeggs and mp_filename:
658 fullabspath = [abspath(x) for x in _path]
659 try:
660 pathindex = fullabspath.index(dirname(abspath(mp_filename)))
661 emtype, emp_filename, zippath = _search_zip(modpath, pic)
662 if pathindex > _path.index(zippath):
663
664 return emtype, emp_filename
665 except ValueError:
666
667 pass
668 except ImportError:
669 pass
670 checkeggs = False
671 imported.append(modpath.pop(0))
672 mtype = mp_desc[2]
673 if modpath:
674 if mtype != PKG_DIRECTORY:
675 raise ImportError('No module %s in %s' % ('.'.join(modpath),
676 '.'.join(imported)))
677
678
679 try:
680 with open(join(mp_filename, '__init__.py')) as stream:
681 data = stream.read(4096)
682 except IOError:
683 path = [mp_filename]
684 else:
685 if 'pkgutil' in data and 'extend_path' in data:
686
687
688 path = [join(p, *imported) for p in sys.path
689 if isdir(join(p, *imported))]
690 else:
691 path = [mp_filename]
692 return mtype, mp_filename
693
695 """return true if the given filename should be considered as a python file
696
697 .pyc and .pyo are ignored
698 """
699 for ext in ('.py', '.so', '.pyd', '.pyw'):
700 if filename.endswith(ext):
701 return True
702 return False
703
704
706 """if the given directory has a valid __init__ file, return its path,
707 else return None
708 """
709 mod_or_pack = join(directory, '__init__')
710 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):
711 if exists(mod_or_pack + '.' + ext):
712 return mod_or_pack + '.' + ext
713 return None
714