_internal.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932
  1. """
  2. A place for internal code
  3. Some things are more easily handled Python.
  4. """
  5. import ast
  6. import re
  7. import sys
  8. import warnings
  9. from .multiarray import dtype, array, ndarray, promote_types
  10. try:
  11. import ctypes
  12. except ImportError:
  13. ctypes = None
  14. IS_PYPY = sys.implementation.name == 'pypy'
  15. if sys.byteorder == 'little':
  16. _nbo = '<'
  17. else:
  18. _nbo = '>'
  19. def _makenames_list(adict, align):
  20. allfields = []
  21. for fname, obj in adict.items():
  22. n = len(obj)
  23. if not isinstance(obj, tuple) or n not in (2, 3):
  24. raise ValueError("entry not a 2- or 3- tuple")
  25. if n > 2 and obj[2] == fname:
  26. continue
  27. num = int(obj[1])
  28. if num < 0:
  29. raise ValueError("invalid offset.")
  30. format = dtype(obj[0], align=align)
  31. if n > 2:
  32. title = obj[2]
  33. else:
  34. title = None
  35. allfields.append((fname, format, num, title))
  36. # sort by offsets
  37. allfields.sort(key=lambda x: x[2])
  38. names = [x[0] for x in allfields]
  39. formats = [x[1] for x in allfields]
  40. offsets = [x[2] for x in allfields]
  41. titles = [x[3] for x in allfields]
  42. return names, formats, offsets, titles
  43. # Called in PyArray_DescrConverter function when
  44. # a dictionary without "names" and "formats"
  45. # fields is used as a data-type descriptor.
  46. def _usefields(adict, align):
  47. try:
  48. names = adict[-1]
  49. except KeyError:
  50. names = None
  51. if names is None:
  52. names, formats, offsets, titles = _makenames_list(adict, align)
  53. else:
  54. formats = []
  55. offsets = []
  56. titles = []
  57. for name in names:
  58. res = adict[name]
  59. formats.append(res[0])
  60. offsets.append(res[1])
  61. if len(res) > 2:
  62. titles.append(res[2])
  63. else:
  64. titles.append(None)
  65. return dtype({"names": names,
  66. "formats": formats,
  67. "offsets": offsets,
  68. "titles": titles}, align)
  69. # construct an array_protocol descriptor list
  70. # from the fields attribute of a descriptor
  71. # This calls itself recursively but should eventually hit
  72. # a descriptor that has no fields and then return
  73. # a simple typestring
  74. def _array_descr(descriptor):
  75. fields = descriptor.fields
  76. if fields is None:
  77. subdtype = descriptor.subdtype
  78. if subdtype is None:
  79. if descriptor.metadata is None:
  80. return descriptor.str
  81. else:
  82. new = descriptor.metadata.copy()
  83. if new:
  84. return (descriptor.str, new)
  85. else:
  86. return descriptor.str
  87. else:
  88. return (_array_descr(subdtype[0]), subdtype[1])
  89. names = descriptor.names
  90. ordered_fields = [fields[x] + (x,) for x in names]
  91. result = []
  92. offset = 0
  93. for field in ordered_fields:
  94. if field[1] > offset:
  95. num = field[1] - offset
  96. result.append(('', f'|V{num}'))
  97. offset += num
  98. elif field[1] < offset:
  99. raise ValueError(
  100. "dtype.descr is not defined for types with overlapping or "
  101. "out-of-order fields")
  102. if len(field) > 3:
  103. name = (field[2], field[3])
  104. else:
  105. name = field[2]
  106. if field[0].subdtype:
  107. tup = (name, _array_descr(field[0].subdtype[0]),
  108. field[0].subdtype[1])
  109. else:
  110. tup = (name, _array_descr(field[0]))
  111. offset += field[0].itemsize
  112. result.append(tup)
  113. if descriptor.itemsize > offset:
  114. num = descriptor.itemsize - offset
  115. result.append(('', f'|V{num}'))
  116. return result
  117. # Build a new array from the information in a pickle.
  118. # Note that the name numpy.core._internal._reconstruct is embedded in
  119. # pickles of ndarrays made with NumPy before release 1.0
  120. # so don't remove the name here, or you'll
  121. # break backward compatibility.
  122. def _reconstruct(subtype, shape, dtype):
  123. return ndarray.__new__(subtype, shape, dtype)
  124. # format_re was originally from numarray by J. Todd Miller
  125. format_re = re.compile(r'(?P<order1>[<>|=]?)'
  126. r'(?P<repeats> *[(]?[ ,0-9]*[)]? *)'
  127. r'(?P<order2>[<>|=]?)'
  128. r'(?P<dtype>[A-Za-z0-9.?]*(?:\[[a-zA-Z0-9,.]+\])?)')
  129. sep_re = re.compile(r'\s*,\s*')
  130. space_re = re.compile(r'\s+$')
  131. # astr is a string (perhaps comma separated)
  132. _convorder = {'=': _nbo}
  133. def _commastring(astr):
  134. startindex = 0
  135. result = []
  136. while startindex < len(astr):
  137. mo = format_re.match(astr, pos=startindex)
  138. try:
  139. (order1, repeats, order2, dtype) = mo.groups()
  140. except (TypeError, AttributeError):
  141. raise ValueError(
  142. f'format number {len(result)+1} of "{astr}" is not recognized'
  143. ) from None
  144. startindex = mo.end()
  145. # Separator or ending padding
  146. if startindex < len(astr):
  147. if space_re.match(astr, pos=startindex):
  148. startindex = len(astr)
  149. else:
  150. mo = sep_re.match(astr, pos=startindex)
  151. if not mo:
  152. raise ValueError(
  153. 'format number %d of "%s" is not recognized' %
  154. (len(result)+1, astr))
  155. startindex = mo.end()
  156. if order2 == '':
  157. order = order1
  158. elif order1 == '':
  159. order = order2
  160. else:
  161. order1 = _convorder.get(order1, order1)
  162. order2 = _convorder.get(order2, order2)
  163. if (order1 != order2):
  164. raise ValueError(
  165. 'inconsistent byte-order specification %s and %s' %
  166. (order1, order2))
  167. order = order1
  168. if order in ('|', '=', _nbo):
  169. order = ''
  170. dtype = order + dtype
  171. if (repeats == ''):
  172. newitem = dtype
  173. else:
  174. newitem = (dtype, ast.literal_eval(repeats))
  175. result.append(newitem)
  176. return result
  177. class dummy_ctype:
  178. def __init__(self, cls):
  179. self._cls = cls
  180. def __mul__(self, other):
  181. return self
  182. def __call__(self, *other):
  183. return self._cls(other)
  184. def __eq__(self, other):
  185. return self._cls == other._cls
  186. def __ne__(self, other):
  187. return self._cls != other._cls
  188. def _getintp_ctype():
  189. val = _getintp_ctype.cache
  190. if val is not None:
  191. return val
  192. if ctypes is None:
  193. import numpy as np
  194. val = dummy_ctype(np.intp)
  195. else:
  196. char = dtype('p').char
  197. if char == 'i':
  198. val = ctypes.c_int
  199. elif char == 'l':
  200. val = ctypes.c_long
  201. elif char == 'q':
  202. val = ctypes.c_longlong
  203. else:
  204. val = ctypes.c_long
  205. _getintp_ctype.cache = val
  206. return val
  207. _getintp_ctype.cache = None
  208. # Used for .ctypes attribute of ndarray
  209. class _missing_ctypes:
  210. def cast(self, num, obj):
  211. return num.value
  212. class c_void_p:
  213. def __init__(self, ptr):
  214. self.value = ptr
  215. class _ctypes:
  216. def __init__(self, array, ptr=None):
  217. self._arr = array
  218. if ctypes:
  219. self._ctypes = ctypes
  220. self._data = self._ctypes.c_void_p(ptr)
  221. else:
  222. # fake a pointer-like object that holds onto the reference
  223. self._ctypes = _missing_ctypes()
  224. self._data = self._ctypes.c_void_p(ptr)
  225. self._data._objects = array
  226. if self._arr.ndim == 0:
  227. self._zerod = True
  228. else:
  229. self._zerod = False
  230. def data_as(self, obj):
  231. """
  232. Return the data pointer cast to a particular c-types object.
  233. For example, calling ``self._as_parameter_`` is equivalent to
  234. ``self.data_as(ctypes.c_void_p)``. Perhaps you want to use the data as a
  235. pointer to a ctypes array of floating-point data:
  236. ``self.data_as(ctypes.POINTER(ctypes.c_double))``.
  237. The returned pointer will keep a reference to the array.
  238. """
  239. # _ctypes.cast function causes a circular reference of self._data in
  240. # self._data._objects. Attributes of self._data cannot be released
  241. # until gc.collect is called. Make a copy of the pointer first then let
  242. # it hold the array reference. This is a workaround to circumvent the
  243. # CPython bug https://bugs.python.org/issue12836
  244. ptr = self._ctypes.cast(self._data, obj)
  245. ptr._arr = self._arr
  246. return ptr
  247. def shape_as(self, obj):
  248. """
  249. Return the shape tuple as an array of some other c-types
  250. type. For example: ``self.shape_as(ctypes.c_short)``.
  251. """
  252. if self._zerod:
  253. return None
  254. return (obj*self._arr.ndim)(*self._arr.shape)
  255. def strides_as(self, obj):
  256. """
  257. Return the strides tuple as an array of some other
  258. c-types type. For example: ``self.strides_as(ctypes.c_longlong)``.
  259. """
  260. if self._zerod:
  261. return None
  262. return (obj*self._arr.ndim)(*self._arr.strides)
  263. @property
  264. def data(self):
  265. """
  266. A pointer to the memory area of the array as a Python integer.
  267. This memory area may contain data that is not aligned, or not in correct
  268. byte-order. The memory area may not even be writeable. The array
  269. flags and data-type of this array should be respected when passing this
  270. attribute to arbitrary C-code to avoid trouble that can include Python
  271. crashing. User Beware! The value of this attribute is exactly the same
  272. as ``self._array_interface_['data'][0]``.
  273. Note that unlike ``data_as``, a reference will not be kept to the array:
  274. code like ``ctypes.c_void_p((a + b).ctypes.data)`` will result in a
  275. pointer to a deallocated array, and should be spelt
  276. ``(a + b).ctypes.data_as(ctypes.c_void_p)``
  277. """
  278. return self._data.value
  279. @property
  280. def shape(self):
  281. """
  282. (c_intp*self.ndim): A ctypes array of length self.ndim where
  283. the basetype is the C-integer corresponding to ``dtype('p')`` on this
  284. platform (see `~numpy.ctypeslib.c_intp`). This base-type could be
  285. `ctypes.c_int`, `ctypes.c_long`, or `ctypes.c_longlong` depending on
  286. the platform. The ctypes array contains the shape of
  287. the underlying array.
  288. """
  289. return self.shape_as(_getintp_ctype())
  290. @property
  291. def strides(self):
  292. """
  293. (c_intp*self.ndim): A ctypes array of length self.ndim where
  294. the basetype is the same as for the shape attribute. This ctypes array
  295. contains the strides information from the underlying array. This strides
  296. information is important for showing how many bytes must be jumped to
  297. get to the next element in the array.
  298. """
  299. return self.strides_as(_getintp_ctype())
  300. @property
  301. def _as_parameter_(self):
  302. """
  303. Overrides the ctypes semi-magic method
  304. Enables `c_func(some_array.ctypes)`
  305. """
  306. return self.data_as(ctypes.c_void_p)
  307. # Numpy 1.21.0, 2021-05-18
  308. def get_data(self):
  309. """Deprecated getter for the `_ctypes.data` property.
  310. .. deprecated:: 1.21
  311. """
  312. warnings.warn('"get_data" is deprecated. Use "data" instead',
  313. DeprecationWarning, stacklevel=2)
  314. return self.data
  315. def get_shape(self):
  316. """Deprecated getter for the `_ctypes.shape` property.
  317. .. deprecated:: 1.21
  318. """
  319. warnings.warn('"get_shape" is deprecated. Use "shape" instead',
  320. DeprecationWarning, stacklevel=2)
  321. return self.shape
  322. def get_strides(self):
  323. """Deprecated getter for the `_ctypes.strides` property.
  324. .. deprecated:: 1.21
  325. """
  326. warnings.warn('"get_strides" is deprecated. Use "strides" instead',
  327. DeprecationWarning, stacklevel=2)
  328. return self.strides
  329. def get_as_parameter(self):
  330. """Deprecated getter for the `_ctypes._as_parameter_` property.
  331. .. deprecated:: 1.21
  332. """
  333. warnings.warn(
  334. '"get_as_parameter" is deprecated. Use "_as_parameter_" instead',
  335. DeprecationWarning, stacklevel=2,
  336. )
  337. return self._as_parameter_
  338. def _newnames(datatype, order):
  339. """
  340. Given a datatype and an order object, return a new names tuple, with the
  341. order indicated
  342. """
  343. oldnames = datatype.names
  344. nameslist = list(oldnames)
  345. if isinstance(order, str):
  346. order = [order]
  347. seen = set()
  348. if isinstance(order, (list, tuple)):
  349. for name in order:
  350. try:
  351. nameslist.remove(name)
  352. except ValueError:
  353. if name in seen:
  354. raise ValueError(f"duplicate field name: {name}") from None
  355. else:
  356. raise ValueError(f"unknown field name: {name}") from None
  357. seen.add(name)
  358. return tuple(list(order) + nameslist)
  359. raise ValueError(f"unsupported order value: {order}")
  360. def _copy_fields(ary):
  361. """Return copy of structured array with padding between fields removed.
  362. Parameters
  363. ----------
  364. ary : ndarray
  365. Structured array from which to remove padding bytes
  366. Returns
  367. -------
  368. ary_copy : ndarray
  369. Copy of ary with padding bytes removed
  370. """
  371. dt = ary.dtype
  372. copy_dtype = {'names': dt.names,
  373. 'formats': [dt.fields[name][0] for name in dt.names]}
  374. return array(ary, dtype=copy_dtype, copy=True)
  375. def _promote_fields(dt1, dt2):
  376. """ Perform type promotion for two structured dtypes.
  377. Parameters
  378. ----------
  379. dt1 : structured dtype
  380. First dtype.
  381. dt2 : structured dtype
  382. Second dtype.
  383. Returns
  384. -------
  385. out : dtype
  386. The promoted dtype
  387. Notes
  388. -----
  389. If one of the inputs is aligned, the result will be. The titles of
  390. both descriptors must match (point to the same field).
  391. """
  392. # Both must be structured and have the same names in the same order
  393. if (dt1.names is None or dt2.names is None) or dt1.names != dt2.names:
  394. raise TypeError("invalid type promotion")
  395. # if both are identical, we can (maybe!) just return the same dtype.
  396. identical = dt1 is dt2
  397. new_fields = []
  398. for name in dt1.names:
  399. field1 = dt1.fields[name]
  400. field2 = dt2.fields[name]
  401. new_descr = promote_types(field1[0], field2[0])
  402. identical = identical and new_descr is field1[0]
  403. # Check that the titles match (if given):
  404. if field1[2:] != field2[2:]:
  405. raise TypeError("invalid type promotion")
  406. if len(field1) == 2:
  407. new_fields.append((name, new_descr))
  408. else:
  409. new_fields.append(((field1[2], name), new_descr))
  410. res = dtype(new_fields, align=dt1.isalignedstruct or dt2.isalignedstruct)
  411. # Might as well preserve identity (and metadata) if the dtype is identical
  412. # and the itemsize, offsets are also unmodified. This could probably be
  413. # sped up, but also probably just be removed entirely.
  414. if identical and res.itemsize == dt1.itemsize:
  415. for name in dt1.names:
  416. if dt1.fields[name][1] != res.fields[name][1]:
  417. return res # the dtype changed.
  418. return dt1
  419. return res
  420. def _getfield_is_safe(oldtype, newtype, offset):
  421. """ Checks safety of getfield for object arrays.
  422. As in _view_is_safe, we need to check that memory containing objects is not
  423. reinterpreted as a non-object datatype and vice versa.
  424. Parameters
  425. ----------
  426. oldtype : data-type
  427. Data type of the original ndarray.
  428. newtype : data-type
  429. Data type of the field being accessed by ndarray.getfield
  430. offset : int
  431. Offset of the field being accessed by ndarray.getfield
  432. Raises
  433. ------
  434. TypeError
  435. If the field access is invalid
  436. """
  437. if newtype.hasobject or oldtype.hasobject:
  438. if offset == 0 and newtype == oldtype:
  439. return
  440. if oldtype.names is not None:
  441. for name in oldtype.names:
  442. if (oldtype.fields[name][1] == offset and
  443. oldtype.fields[name][0] == newtype):
  444. return
  445. raise TypeError("Cannot get/set field of an object array")
  446. return
  447. def _view_is_safe(oldtype, newtype):
  448. """ Checks safety of a view involving object arrays, for example when
  449. doing::
  450. np.zeros(10, dtype=oldtype).view(newtype)
  451. Parameters
  452. ----------
  453. oldtype : data-type
  454. Data type of original ndarray
  455. newtype : data-type
  456. Data type of the view
  457. Raises
  458. ------
  459. TypeError
  460. If the new type is incompatible with the old type.
  461. """
  462. # if the types are equivalent, there is no problem.
  463. # for example: dtype((np.record, 'i4,i4')) == dtype((np.void, 'i4,i4'))
  464. if oldtype == newtype:
  465. return
  466. if newtype.hasobject or oldtype.hasobject:
  467. raise TypeError("Cannot change data-type for object array.")
  468. return
  469. # Given a string containing a PEP 3118 format specifier,
  470. # construct a NumPy dtype
  471. _pep3118_native_map = {
  472. '?': '?',
  473. 'c': 'S1',
  474. 'b': 'b',
  475. 'B': 'B',
  476. 'h': 'h',
  477. 'H': 'H',
  478. 'i': 'i',
  479. 'I': 'I',
  480. 'l': 'l',
  481. 'L': 'L',
  482. 'q': 'q',
  483. 'Q': 'Q',
  484. 'e': 'e',
  485. 'f': 'f',
  486. 'd': 'd',
  487. 'g': 'g',
  488. 'Zf': 'F',
  489. 'Zd': 'D',
  490. 'Zg': 'G',
  491. 's': 'S',
  492. 'w': 'U',
  493. 'O': 'O',
  494. 'x': 'V', # padding
  495. }
  496. _pep3118_native_typechars = ''.join(_pep3118_native_map.keys())
  497. _pep3118_standard_map = {
  498. '?': '?',
  499. 'c': 'S1',
  500. 'b': 'b',
  501. 'B': 'B',
  502. 'h': 'i2',
  503. 'H': 'u2',
  504. 'i': 'i4',
  505. 'I': 'u4',
  506. 'l': 'i4',
  507. 'L': 'u4',
  508. 'q': 'i8',
  509. 'Q': 'u8',
  510. 'e': 'f2',
  511. 'f': 'f',
  512. 'd': 'd',
  513. 'Zf': 'F',
  514. 'Zd': 'D',
  515. 's': 'S',
  516. 'w': 'U',
  517. 'O': 'O',
  518. 'x': 'V', # padding
  519. }
  520. _pep3118_standard_typechars = ''.join(_pep3118_standard_map.keys())
  521. _pep3118_unsupported_map = {
  522. 'u': 'UCS-2 strings',
  523. '&': 'pointers',
  524. 't': 'bitfields',
  525. 'X': 'function pointers',
  526. }
  527. class _Stream:
  528. def __init__(self, s):
  529. self.s = s
  530. self.byteorder = '@'
  531. def advance(self, n):
  532. res = self.s[:n]
  533. self.s = self.s[n:]
  534. return res
  535. def consume(self, c):
  536. if self.s[:len(c)] == c:
  537. self.advance(len(c))
  538. return True
  539. return False
  540. def consume_until(self, c):
  541. if callable(c):
  542. i = 0
  543. while i < len(self.s) and not c(self.s[i]):
  544. i = i + 1
  545. return self.advance(i)
  546. else:
  547. i = self.s.index(c)
  548. res = self.advance(i)
  549. self.advance(len(c))
  550. return res
  551. @property
  552. def next(self):
  553. return self.s[0]
  554. def __bool__(self):
  555. return bool(self.s)
  556. def _dtype_from_pep3118(spec):
  557. stream = _Stream(spec)
  558. dtype, align = __dtype_from_pep3118(stream, is_subdtype=False)
  559. return dtype
  560. def __dtype_from_pep3118(stream, is_subdtype):
  561. field_spec = dict(
  562. names=[],
  563. formats=[],
  564. offsets=[],
  565. itemsize=0
  566. )
  567. offset = 0
  568. common_alignment = 1
  569. is_padding = False
  570. # Parse spec
  571. while stream:
  572. value = None
  573. # End of structure, bail out to upper level
  574. if stream.consume('}'):
  575. break
  576. # Sub-arrays (1)
  577. shape = None
  578. if stream.consume('('):
  579. shape = stream.consume_until(')')
  580. shape = tuple(map(int, shape.split(',')))
  581. # Byte order
  582. if stream.next in ('@', '=', '<', '>', '^', '!'):
  583. byteorder = stream.advance(1)
  584. if byteorder == '!':
  585. byteorder = '>'
  586. stream.byteorder = byteorder
  587. # Byte order characters also control native vs. standard type sizes
  588. if stream.byteorder in ('@', '^'):
  589. type_map = _pep3118_native_map
  590. type_map_chars = _pep3118_native_typechars
  591. else:
  592. type_map = _pep3118_standard_map
  593. type_map_chars = _pep3118_standard_typechars
  594. # Item sizes
  595. itemsize_str = stream.consume_until(lambda c: not c.isdigit())
  596. if itemsize_str:
  597. itemsize = int(itemsize_str)
  598. else:
  599. itemsize = 1
  600. # Data types
  601. is_padding = False
  602. if stream.consume('T{'):
  603. value, align = __dtype_from_pep3118(
  604. stream, is_subdtype=True)
  605. elif stream.next in type_map_chars:
  606. if stream.next == 'Z':
  607. typechar = stream.advance(2)
  608. else:
  609. typechar = stream.advance(1)
  610. is_padding = (typechar == 'x')
  611. dtypechar = type_map[typechar]
  612. if dtypechar in 'USV':
  613. dtypechar += '%d' % itemsize
  614. itemsize = 1
  615. numpy_byteorder = {'@': '=', '^': '='}.get(
  616. stream.byteorder, stream.byteorder)
  617. value = dtype(numpy_byteorder + dtypechar)
  618. align = value.alignment
  619. elif stream.next in _pep3118_unsupported_map:
  620. desc = _pep3118_unsupported_map[stream.next]
  621. raise NotImplementedError(
  622. "Unrepresentable PEP 3118 data type {!r} ({})"
  623. .format(stream.next, desc))
  624. else:
  625. raise ValueError("Unknown PEP 3118 data type specifier %r" % stream.s)
  626. #
  627. # Native alignment may require padding
  628. #
  629. # Here we assume that the presence of a '@' character implicitly implies
  630. # that the start of the array is *already* aligned.
  631. #
  632. extra_offset = 0
  633. if stream.byteorder == '@':
  634. start_padding = (-offset) % align
  635. intra_padding = (-value.itemsize) % align
  636. offset += start_padding
  637. if intra_padding != 0:
  638. if itemsize > 1 or (shape is not None and _prod(shape) > 1):
  639. # Inject internal padding to the end of the sub-item
  640. value = _add_trailing_padding(value, intra_padding)
  641. else:
  642. # We can postpone the injection of internal padding,
  643. # as the item appears at most once
  644. extra_offset += intra_padding
  645. # Update common alignment
  646. common_alignment = _lcm(align, common_alignment)
  647. # Convert itemsize to sub-array
  648. if itemsize != 1:
  649. value = dtype((value, (itemsize,)))
  650. # Sub-arrays (2)
  651. if shape is not None:
  652. value = dtype((value, shape))
  653. # Field name
  654. if stream.consume(':'):
  655. name = stream.consume_until(':')
  656. else:
  657. name = None
  658. if not (is_padding and name is None):
  659. if name is not None and name in field_spec['names']:
  660. raise RuntimeError(f"Duplicate field name '{name}' in PEP3118 format")
  661. field_spec['names'].append(name)
  662. field_spec['formats'].append(value)
  663. field_spec['offsets'].append(offset)
  664. offset += value.itemsize
  665. offset += extra_offset
  666. field_spec['itemsize'] = offset
  667. # extra final padding for aligned types
  668. if stream.byteorder == '@':
  669. field_spec['itemsize'] += (-offset) % common_alignment
  670. # Check if this was a simple 1-item type, and unwrap it
  671. if (field_spec['names'] == [None]
  672. and field_spec['offsets'][0] == 0
  673. and field_spec['itemsize'] == field_spec['formats'][0].itemsize
  674. and not is_subdtype):
  675. ret = field_spec['formats'][0]
  676. else:
  677. _fix_names(field_spec)
  678. ret = dtype(field_spec)
  679. # Finished
  680. return ret, common_alignment
  681. def _fix_names(field_spec):
  682. """ Replace names which are None with the next unused f%d name """
  683. names = field_spec['names']
  684. for i, name in enumerate(names):
  685. if name is not None:
  686. continue
  687. j = 0
  688. while True:
  689. name = f'f{j}'
  690. if name not in names:
  691. break
  692. j = j + 1
  693. names[i] = name
  694. def _add_trailing_padding(value, padding):
  695. """Inject the specified number of padding bytes at the end of a dtype"""
  696. if value.fields is None:
  697. field_spec = dict(
  698. names=['f0'],
  699. formats=[value],
  700. offsets=[0],
  701. itemsize=value.itemsize
  702. )
  703. else:
  704. fields = value.fields
  705. names = value.names
  706. field_spec = dict(
  707. names=names,
  708. formats=[fields[name][0] for name in names],
  709. offsets=[fields[name][1] for name in names],
  710. itemsize=value.itemsize
  711. )
  712. field_spec['itemsize'] += padding
  713. return dtype(field_spec)
  714. def _prod(a):
  715. p = 1
  716. for x in a:
  717. p *= x
  718. return p
  719. def _gcd(a, b):
  720. """Calculate the greatest common divisor of a and b"""
  721. while b:
  722. a, b = b, a % b
  723. return a
  724. def _lcm(a, b):
  725. return a // _gcd(a, b) * b
  726. def array_ufunc_errmsg_formatter(dummy, ufunc, method, *inputs, **kwargs):
  727. """ Format the error message for when __array_ufunc__ gives up. """
  728. args_string = ', '.join(['{!r}'.format(arg) for arg in inputs] +
  729. ['{}={!r}'.format(k, v)
  730. for k, v in kwargs.items()])
  731. args = inputs + kwargs.get('out', ())
  732. types_string = ', '.join(repr(type(arg).__name__) for arg in args)
  733. return ('operand type(s) all returned NotImplemented from '
  734. '__array_ufunc__({!r}, {!r}, {}): {}'
  735. .format(ufunc, method, args_string, types_string))
  736. def array_function_errmsg_formatter(public_api, types):
  737. """ Format the error message for when __array_ufunc__ gives up. """
  738. func_name = '{}.{}'.format(public_api.__module__, public_api.__name__)
  739. return ("no implementation found for '{}' on types that implement "
  740. '__array_function__: {}'.format(func_name, list(types)))
  741. def _ufunc_doc_signature_formatter(ufunc):
  742. """
  743. Builds a signature string which resembles PEP 457
  744. This is used to construct the first line of the docstring
  745. """
  746. # input arguments are simple
  747. if ufunc.nin == 1:
  748. in_args = 'x'
  749. else:
  750. in_args = ', '.join(f'x{i+1}' for i in range(ufunc.nin))
  751. # output arguments are both keyword or positional
  752. if ufunc.nout == 0:
  753. out_args = ', /, out=()'
  754. elif ufunc.nout == 1:
  755. out_args = ', /, out=None'
  756. else:
  757. out_args = '[, {positional}], / [, out={default}]'.format(
  758. positional=', '.join(
  759. 'out{}'.format(i+1) for i in range(ufunc.nout)),
  760. default=repr((None,)*ufunc.nout)
  761. )
  762. # keyword only args depend on whether this is a gufunc
  763. kwargs = (
  764. ", casting='same_kind'"
  765. ", order='K'"
  766. ", dtype=None"
  767. ", subok=True"
  768. )
  769. # NOTE: gufuncs may or may not support the `axis` parameter
  770. if ufunc.signature is None:
  771. kwargs = f", where=True{kwargs}[, signature, extobj]"
  772. else:
  773. kwargs += "[, signature, extobj, axes, axis]"
  774. # join all the parts together
  775. return '{name}({in_args}{out_args}, *{kwargs})'.format(
  776. name=ufunc.__name__,
  777. in_args=in_args,
  778. out_args=out_args,
  779. kwargs=kwargs
  780. )
  781. def npy_ctypes_check(cls):
  782. # determine if a class comes from ctypes, in order to work around
  783. # a bug in the buffer protocol for those objects, bpo-10746
  784. try:
  785. # ctypes class are new-style, so have an __mro__. This probably fails
  786. # for ctypes classes with multiple inheritance.
  787. if IS_PYPY:
  788. # (..., _ctypes.basics._CData, Bufferable, object)
  789. ctype_base = cls.__mro__[-3]
  790. else:
  791. # # (..., _ctypes._CData, object)
  792. ctype_base = cls.__mro__[-2]
  793. # right now, they're part of the _ctypes module
  794. return '_ctypes' in ctype_base.__module__
  795. except Exception:
  796. return False