codekingpro/portable-devtools
115k
1import functools
2import os
3import platform
4import sys
5import textwrap
6import types
7import warnings
8
9import numpy as np
10from numpy._core import ndarray
11from numpy._utils import set_module
12
13__all__ = [
14 'get_include', 'info', 'show_runtime'
15]
16
17
18@set_module('numpy')
19def show_runtime():
20 """
21 Print information about various resources in the system
22 including available intrinsic support and BLAS/LAPACK library
23 in use
24
25 .. versionadded:: 1.24.0
26
27 See Also
28 --------
29 show_config : Show libraries in the system on which NumPy was built.
30
31 Notes
32 -----
33 1. Information is derived with the help of `threadpoolctl <https://pypi.org/project/threadpoolctl/>`_
34 library if available.
35 2. SIMD related information is derived from ``__cpu_features__``,
36 ``__cpu_baseline__`` and ``__cpu_dispatch__``
37
38 """
39 from pprint import pprint
40
41 from numpy._core._multiarray_umath import (
42 __cpu_baseline__,
43 __cpu_dispatch__,
44 __cpu_features__,
45 )
46 config_found = [{
47 "numpy_version": np.__version__,
48 "python": sys.version,
49 "uname": platform.uname(),
50 }]
51 features_found, features_not_found = [], []
52 for feature in __cpu_dispatch__:
53 if __cpu_features__[feature]:
54 features_found.append(feature)
55 else:
56 features_not_found.append(feature)
57 config_found.append({
58 "simd_extensions": {
59 "baseline": __cpu_baseline__,
60 "found": features_found,
61 "not_found": features_not_found
62 }
63 })
64 config_found.append({
65 "ignore_floating_point_errors_in_matmul":
66 not np._core._multiarray_umath._blas_supports_fpe(None),
67 })
68
69 try:
70 from threadpoolctl import threadpool_info
71 config_found.extend(threadpool_info())
72 except ImportError:
73 print("WARNING: `threadpoolctl` not found in system!"
74 " Install it by `pip install threadpoolctl`."
75 " Once installed, try `np.show_runtime` again"
76 " for more detailed build information")
77 pprint(config_found)
78
79
80@set_module('numpy')
81def get_include():
82 """
83 Return the directory that contains the NumPy \\*.h header files.
84
85 Extension modules that need to compile against NumPy may need to use this
86 function to locate the appropriate include directory.
87
88 Notes
89 -----
90 When using ``setuptools``, for example in ``setup.py``::
91
92 import numpy as np
93 ...
94 Extension('extension_name', ...
95 include_dirs=[np.get_include()])
96 ...
97
98 Note that a CLI tool ``numpy-config`` was introduced in NumPy 2.0, using
99 that is likely preferred for build systems other than ``setuptools``::
100
101 $ numpy-config --cflags
102 -I/path/to/site-packages/numpy/_core/include
103
104 # Or rely on pkg-config:
105 $ export PKG_CONFIG_PATH=$(numpy-config --pkgconfigdir)
106 $ pkg-config --cflags
107 -I/path/to/site-packages/numpy/_core/include
108
109 Examples
110 --------
111 >>> np.get_include()
112 '.../site-packages/numpy/core/include' # may vary
113
114 """
115 import numpy
116 if numpy.show_config is None:
117 # running from numpy source directory
118 d = os.path.join(os.path.dirname(numpy.__file__), '_core', 'include')
119 else:
120 # using installed numpy core headers
121 import numpy._core as _core
122 d = os.path.join(os.path.dirname(_core.__file__), 'include')
123 return d
124
125
126class _Deprecate:
127 """
128 Decorator class to deprecate old functions.
129
130 Refer to `deprecate` for details.
131
132 See Also
133 --------
134 deprecate
135
136 """
137
138 def __init__(self, old_name=None, new_name=None, message=None):
139 self.old_name = old_name
140 self.new_name = new_name
141 self.message = message
142
143 def __call__(self, func, *args, **kwargs):
144 """
145 Decorator call. Refer to ``decorate``.
146
147 """
148 old_name = self.old_name
149 new_name = self.new_name
150 message = self.message
151
152 if old_name is None:
153 old_name = func.__name__
154 if new_name is None:
155 depdoc = f"`{old_name}` is deprecated!"
156 else:
157 depdoc = f"`{old_name}` is deprecated, use `{new_name}` instead!"
158
159 if message is not None:
160 depdoc += "\n" + message
161
162 @functools.wraps(func)
163 def newfunc(*args, **kwds):
164 warnings.warn(depdoc, DeprecationWarning, stacklevel=2)
165 return func(*args, **kwds)
166
167 newfunc.__name__ = old_name
168 doc = func.__doc__
169 if doc is None:
170 doc = depdoc
171 else:
172 lines = doc.expandtabs().split('\n')
173 indent = _get_indent(lines[1:])
174 if lines[0].lstrip():
175 # Indent the original first line to let inspect.cleandoc()
176 # dedent the docstring despite the deprecation notice.
177 doc = indent * ' ' + doc
178 else:
179 # Remove the same leading blank lines as cleandoc() would.
180 skip = len(lines[0]) + 1
181 for line in lines[1:]:
182 if len(line) > indent:
183 break
184 skip += len(line) + 1
185 doc = doc[skip:]
186 depdoc = textwrap.indent(depdoc, ' ' * indent)
187 doc = f'{depdoc}\n\n{doc}'
188 newfunc.__doc__ = doc
189
190 return newfunc
191
192
193def _get_indent(lines):
194 """
195 Determines the leading whitespace that could be removed from all the lines.
196 """
197 indent = sys.maxsize
198 for line in lines:
199 content = len(line.lstrip())
200 if content:
201 indent = min(indent, len(line) - content)
202 if indent == sys.maxsize:
203 indent = 0
204 return indent
205
206
207def deprecate(*args, **kwargs):
208 """
209 Issues a DeprecationWarning, adds warning to `old_name`'s
210 docstring, rebinds ``old_name.__name__`` and returns the new
211 function object.
212
213 This function may also be used as a decorator.
214
215 .. deprecated:: 2.0
216 Use `~warnings.warn` with :exc:`DeprecationWarning` instead.
217
218 Parameters
219 ----------
220 func : function
221 The function to be deprecated.
222 old_name : str, optional
223 The name of the function to be deprecated. Default is None, in
224 which case the name of `func` is used.
225 new_name : str, optional
226 The new name for the function. Default is None, in which case the
227 deprecation message is that `old_name` is deprecated. If given, the
228 deprecation message is that `old_name` is deprecated and `new_name`
229 should be used instead.
230 message : str, optional
231 Additional explanation of the deprecation. Displayed in the
232 docstring after the warning.
233
234 Returns
235 -------
236 old_func : function
237 The deprecated function.
238
239 Examples
240 --------
241 Note that ``olduint`` returns a value after printing Deprecation
242 Warning:
243
244 >>> olduint = np.lib.utils.deprecate(np.uint)
245 DeprecationWarning: `uint64` is deprecated! # may vary
246 >>> olduint(6)
247 6
248
249 """
250 # Deprecate may be run as a function or as a decorator
251 # If run as a function, we initialise the decorator class
252 # and execute its __call__ method.
253
254 # Deprecated in NumPy 2.0, 2023-07-11
255 warnings.warn(
256 "`deprecate` is deprecated, "
257 "use `warn` with `DeprecationWarning` instead. "
258 "(deprecated in NumPy 2.0)",
259 DeprecationWarning,
260 stacklevel=2
261 )
262
263 if args:
264 fn = args[0]
265 args = args[1:]
266
267 return _Deprecate(*args, **kwargs)(fn)
268 else:
269 return _Deprecate(*args, **kwargs)
270
271
272def deprecate_with_doc(msg):
273 """
274 Deprecates a function and includes the deprecation in its docstring.
275
276 .. deprecated:: 2.0
277 Use `~warnings.warn` with :exc:`DeprecationWarning` instead.
278
279 This function is used as a decorator. It returns an object that can be
280 used to issue a DeprecationWarning, by passing the to-be decorated
281 function as argument, this adds warning to the to-be decorated function's
282 docstring and returns the new function object.
283
284 See Also
285 --------
286 deprecate : Decorate a function such that it issues a
287 :exc:`DeprecationWarning`
288
289 Parameters
290 ----------
291 msg : str
292 Additional explanation of the deprecation. Displayed in the
293 docstring after the warning.
294
295 Returns
296 -------
297 obj : object
298
299 """
300
301 # Deprecated in NumPy 2.0, 2023-07-11
302 warnings.warn(
303 "`deprecate` is deprecated, "
304 "use `warn` with `DeprecationWarning` instead. "
305 "(deprecated in NumPy 2.0)",
306 DeprecationWarning,
307 stacklevel=2
308 )
309
310 return _Deprecate(message=msg)
311
312
313#-----------------------------------------------------------------------------
314
315
316# NOTE: pydoc defines a help function which works similarly to this
317# except it uses a pager to take over the screen.
318
319# combine name and arguments and split to multiple lines of width
320# characters. End lines on a comma and begin argument list indented with
321# the rest of the arguments.
322def _split_line(name, arguments, width):
323 firstwidth = len(name)
324 k = firstwidth
325 newstr = name
326 sepstr = ", "
327 arglist = arguments.split(sepstr)
328 for argument in arglist:
329 if k == firstwidth:
330 addstr = ""
331 else:
332 addstr = sepstr
333 k = k + len(argument) + len(addstr)
334 if k > width:
335 k = firstwidth + 1 + len(argument)
336 newstr = newstr + ",\n" + " " * (firstwidth + 2) + argument
337 else:
338 newstr = newstr + addstr + argument
339 return newstr
340
341
342_namedict = None
343_dictlist = None
344
345# Traverse all module directories underneath globals
346# to see if something is defined
347def _makenamedict(module='numpy'):
348 module = __import__(module, globals(), locals(), [])
349 thedict = {module.__name__: module.__dict__}
350 dictlist = [module.__name__]
351 totraverse = [module.__dict__]
352 while True:
353 if len(totraverse) == 0:
354 break
355 thisdict = totraverse.pop(0)
356 for x in thisdict.keys():
357 if isinstance(thisdict[x], types.ModuleType):
358 modname = thisdict[x].__name__
359 if modname not in dictlist:
360 moddict = thisdict[x].__dict__
361 dictlist.append(modname)
362 totraverse.append(moddict)
363 thedict[modname] = moddict
364 return thedict, dictlist
365
366
367def _info(obj, output=None):
368 """Provide information about ndarray obj.
369
370 Parameters
371 ----------
372 obj : ndarray
373 Must be ndarray, not checked.
374 output
375 Where printed output goes.
376
377 Notes
378 -----
379 Copied over from the numarray module prior to its removal.
380 Adapted somewhat as only numpy is an option now.
381
382 Called by info.
383
384 """
385 extra = ""
386 tic = ""
387 bp = lambda x: x
388 cls = getattr(obj, '__class__', type(obj))
389 nm = getattr(cls, '__name__', cls)
390 strides = obj.strides
391 endian = obj.dtype.byteorder
392
393 if output is None:
394 output = sys.stdout
395
396 print("class: ", nm, file=output)
397 print("shape: ", obj.shape, file=output)
398 print("strides: ", strides, file=output)
399 print("itemsize: ", obj.itemsize, file=output)
400 print("aligned: ", bp(obj.flags.aligned), file=output)
401 print("contiguous: ", bp(obj.flags.contiguous), file=output)
402 print("fortran: ", obj.flags.fortran, file=output)
403 print(
404 f"data pointer: {hex(obj.ctypes._as_parameter_.value)}{extra}",
405 file=output
406 )
407 print("byteorder: ", end=' ', file=output)
408 if endian in ['|', '=']:
409 print(f"{tic}{sys.byteorder}{tic}", file=output)
410 byteswap = False
411 elif endian == '>':
412 print(f"{tic}big{tic}", file=output)
413 byteswap = sys.byteorder != "big"
414 else:
415 print(f"{tic}little{tic}", file=output)
416 byteswap = sys.byteorder != "little"
417 print("byteswap: ", bp(byteswap), file=output)
418 print(f"type: {obj.dtype}", file=output)
419
420
421@set_module('numpy')
422def info(object=None, maxwidth=76, output=None, toplevel='numpy'):
423 """
424 Get help information for an array, function, class, or module.
425
426 Parameters
427 ----------
428 object : object or str, optional
429 Input object or name to get information about. If `object` is
430 an `ndarray` instance, information about the array is printed.
431 If `object` is a numpy object, its docstring is given. If it is
432 a string, available modules are searched for matching objects.
433 If None, information about `info` itself is returned.
434 maxwidth : int, optional
435 Printing width.
436 output : file like object, optional
437 File like object that the output is written to, default is
438 ``None``, in which case ``sys.stdout`` will be used.
439 The object has to be opened in 'w' or 'a' mode.
440 toplevel : str, optional
441 Start search at this level.
442
443 Notes
444 -----
445 When used interactively with an object, ``np.info(obj)`` is equivalent
446 to ``help(obj)`` on the Python prompt or ``obj?`` on the IPython
447 prompt.
448
449 Examples
450 --------
451 >>> np.info(np.polyval) # doctest: +SKIP
452 polyval(p, x)
453 Evaluate the polynomial p at x.
454 ...
455
456 When using a string for `object` it is possible to get multiple results.
457
458 >>> np.info('fft') # doctest: +SKIP
459 *** Found in numpy ***
460 Core FFT routines
461 ...
462 *** Found in numpy.fft ***
463 fft(a, n=None, axis=-1)
464 ...
465 *** Repeat reference found in numpy.fft.fftpack ***
466 *** Total of 3 references found. ***
467
468 When the argument is an array, information about the array is printed.
469
470 >>> a = np.array([[1 + 2j, 3, -4], [-5j, 6, 0]], dtype=np.complex64)
471 >>> np.info(a)
472 class: ndarray
473 shape: (2, 3)
474 strides: (24, 8)
475 itemsize: 8
476 aligned: True
477 contiguous: True
478 fortran: False
479 data pointer: 0x562b6e0d2860 # may vary
480 byteorder: little
481 byteswap: False
482 type: complex64
483
484 """
485 global _namedict, _dictlist
486 # Local import to speed up numpy's import time.
487 import inspect
488 import pydoc
489
490 if (hasattr(object, '_ppimport_importer') or
491 hasattr(object, '_ppimport_module')):
492 object = object._ppimport_module
493 elif hasattr(object, '_ppimport_attr'):
494 object = object._ppimport_attr
495
496 if output is None:
497 output = sys.stdout
498
499 if object is None:
500 info(info)
501 elif isinstance(object, ndarray):
502 _info(object, output=output)
503 elif isinstance(object, str):
504 if _namedict is None:
505 _namedict, _dictlist = _makenamedict(toplevel)
506 numfound = 0
507 objlist = []
508 for namestr in _dictlist:
509 try:
510 obj = _namedict[namestr][object]
511 if id(obj) in objlist:
512 print(f"\n *** Repeat reference found in {namestr} *** ",
513 file=output
514 )
515 else:
516 objlist.append(id(obj))
517 print(f" *** Found in {namestr} ***", file=output)
518 info(obj)
519 print("-" * maxwidth, file=output)
520 numfound += 1
521 except KeyError:
522 pass
523 if numfound == 0:
524 print(f"Help for {object} not found.", file=output)
525 else:
526 print("\n "
527 "*** Total of %d references found. ***" % numfound,
528 file=output
529 )
530
531 elif inspect.isfunction(object) or inspect.ismethod(object):
532 name = object.__name__
533 try:
534 arguments = str(inspect.signature(object))
535 except Exception:
536 arguments = "()"
537
538 if len(name + arguments) > maxwidth:
539 argstr = _split_line(name, arguments, maxwidth)
540 else:
541 argstr = name + arguments
542
543 print(" " + argstr + "\n", file=output)
544 print(inspect.getdoc(object), file=output)
545
546 elif inspect.isclass(object):
547 name = object.__name__
548 try:
549 arguments = str(inspect.signature(object))
550 except Exception:
551 arguments = "()"
552
553 if len(name + arguments) > maxwidth:
554 argstr = _split_line(name, arguments, maxwidth)
555 else:
556 argstr = name + arguments
557
558 print(" " + argstr + "\n", file=output)
559 doc1 = inspect.getdoc(object)
560 if doc1 is None:
561 if hasattr(object, '__init__'):
562 print(inspect.getdoc(object.__init__), file=output)
563 else:
564 print(inspect.getdoc(object), file=output)
565
566 methods = pydoc.allmethods(object)
567
568 public_methods = [meth for meth in methods if meth[0] != '_']
569 if public_methods:
570 print("\n\nMethods:\n", file=output)
571 for meth in public_methods:
572 thisobj = getattr(object, meth, None)
573 if thisobj is not None:
574 methstr, other = pydoc.splitdoc(
575 inspect.getdoc(thisobj) or "None"
576 )
577 print(f" {meth} -- {methstr}", file=output)
578
579 elif hasattr(object, '__doc__'):
580 print(inspect.getdoc(object), file=output)
581
582
583def safe_eval(source):
584 """
585 Protected string evaluation.
586
587 .. deprecated:: 2.0
588 Use `ast.literal_eval` instead.
589
590 Evaluate a string containing a Python literal expression without
591 allowing the execution of arbitrary non-literal code.
592
593 .. warning::
594
595 This function is identical to :py:meth:`ast.literal_eval` and
596 has the same security implications. It may not always be safe
597 to evaluate large input strings.
598
599 Parameters
600 ----------
601 source : str
602 The string to evaluate.
603
604 Returns
605 -------
606 obj : object
607 The result of evaluating `source`.
608
609 Raises
610 ------
611 SyntaxError
612 If the code has invalid Python syntax, or if it contains
613 non-literal code.
614
615 Examples
616 --------
617 >>> np.safe_eval('1')
618 1
619 >>> np.safe_eval('[1, 2, 3]')
620 [1, 2, 3]
621 >>> np.safe_eval('{"foo": ("bar", 10.0)}')
622 {'foo': ('bar', 10.0)}
623
624 >>> np.safe_eval('import os')
625 Traceback (most recent call last):
626 ...
627 SyntaxError: invalid syntax
628
629 >>> np.safe_eval('open("/home/user/.ssh/id_dsa").read()')
630 Traceback (most recent call last):
631 ...
632 ValueError: malformed node or string: <_ast.Call object at 0x...>
633
634 """
635
636 # Deprecated in NumPy 2.0, 2023-07-11
637 warnings.warn(
638 "`safe_eval` is deprecated. Use `ast.literal_eval` instead. "
639 "Be aware of security implications, such as memory exhaustion "
640 "based attacks (deprecated in NumPy 2.0)",
641 DeprecationWarning,
642 stacklevel=2
643 )
644
645 # Local import to speed up numpy's import time.
646 import ast
647 return ast.literal_eval(source)
648
649
650def _median_nancheck(data, result, axis):
651 """
652 Utility function to check median result from data for NaN values at the end
653 and return NaN in that case. Input result can also be a MaskedArray.
654
655 Parameters
656 ----------
657 data : array
658 Sorted input data to median function
659 result : Array or MaskedArray
660 Result of median function.
661 axis : int
662 Axis along which the median was computed.
663
664 Returns
665 -------
666 result : scalar or ndarray
667 Median or NaN in axes which contained NaN in the input. If the input
668 was an array, NaN will be inserted in-place. If a scalar, either the
669 input itself or a scalar NaN.
670 """
671 if data.size == 0:
672 return result
673 potential_nans = data.take(-1, axis=axis)
674 n = np.isnan(potential_nans)
675 # masked NaN values are ok, although for masked the copyto may fail for
676 # unmasked ones (this was always broken) when the result is a scalar.
677 if np.ma.isMaskedArray(n):
678 n = n.filled(False)
679
680 if not n.any():
681 return result
682
683 # Without given output, it is possible that the current result is a
684 # numpy scalar, which is not writeable. If so, just return nan.
685 if isinstance(result, np.generic):
686 return potential_nans
687
688 # Otherwise copy NaNs (if there are any)
689 np.copyto(result, potential_nans, where=n)
690 return result
691
692def _opt_info():
693 """
694 Returns a string containing the CPU features supported
695 by the current build.
696
697 The format of the string can be explained as follows:
698 - Dispatched features supported by the running machine end with `*`.
699 - Dispatched features not supported by the running machine
700 end with `?`.
701 - Remaining features represent the baseline.
702
703 Returns:
704 str: A formatted string indicating the supported CPU features.
705 """
706 from numpy._core._multiarray_umath import (
707 __cpu_baseline__,
708 __cpu_dispatch__,
709 __cpu_features__,
710 )
711
712 if len(__cpu_baseline__) == 0 and len(__cpu_dispatch__) == 0:
713 return ''
714
715 enabled_features = ' '.join(__cpu_baseline__)
716 for feature in __cpu_dispatch__:
717 if __cpu_features__[feature]:
718 enabled_features += f" {feature}*"
719 else:
720 enabled_features += f" {feature}?"
721
722 return enabled_features
723
724def drop_metadata(dtype, /):
725 """
726 Returns the dtype unchanged if it contained no metadata or a copy of the
727 dtype if it (or any of its structure dtypes) contained metadata.
728
729 This utility is used by `np.save` and `np.savez` to drop metadata before
730 saving.
731
732 .. note::
733
734 Due to its limitation this function may move to a more appropriate
735 home or change in the future and is considered semi-public API only.
736
737 .. warning::
738
739 This function does not preserve more strange things like record dtypes
740 and user dtypes may simply return the wrong thing. If you need to be
741 sure about the latter, check the result with:
742 ``np.can_cast(new_dtype, dtype, casting="no")``.
743
744 """
745 if dtype.fields is not None:
746 found_metadata = dtype.metadata is not None
747
748 names = []
749 formats = []
750 offsets = []
751 titles = []
752 for name, field in dtype.fields.items():
753 field_dt = drop_metadata(field[0])
754 if field_dt is not field[0]:
755 found_metadata = True
756
757 names.append(name)
758 formats.append(field_dt)
759 offsets.append(field[1])
760 titles.append(None if len(field) < 3 else field[2])
761
762 if not found_metadata:
763 return dtype
764
765 structure = {
766 'names': names, 'formats': formats, 'offsets': offsets, 'titles': titles,
767 'itemsize': dtype.itemsize}
768
769 # NOTE: Could pass (dtype.type, structure) to preserve record dtypes...
770 return np.dtype(structure, align=dtype.isalignedstruct)
771 elif dtype.subdtype is not None:
772 # subarray dtype
773 subdtype, shape = dtype.subdtype
774 new_subdtype = drop_metadata(subdtype)
775 if dtype.metadata is None and new_subdtype is subdtype:
776 return dtype
777
778 return np.dtype((new_subdtype, shape))
779 else:
780 # Normal unstructured dtype
781 if dtype.metadata is None:
782 return dtype
783 # Note that `dt.str` doesn't round-trip e.g. for user-dtypes.
784 return np.dtype(dtype.str)
785