1"""

2 Record Arrays

3=============

4 Record arrays expose the fields of structured arrays as properties.

5

6 Most commonly, ndarrays contain elements of a single type, e.g. floats,

7 integers, bools etc. However, it is possible for elements to be combinations

8 of these using structured types, such as::

9

10 >>> a = np.array([(1, 2.0), (1, 2.0)], dtype=[('x', np.int64), ('y', np.float64)])

11 >>> a

12 array([(1, 2.), (1, 2.)], dtype=[('x', '<i8'), ('y', '<f8')])

13

14 Here, each element consists of two fields: x (and int), and y (a float).

15 This is known as a structured array. The different fields are analogous

16 to columns in a spread-sheet. The different fields can be accessed as

17 one would a dictionary::

18

19 >>> a['x']

20 array([1, 1])

21

22 >>> a['y']

23 array([2., 2.])

24

25 Record arrays allow us to access fields as properties::

26

27 >>> ar = np.rec.array(a)

28

29 >>> ar.x

30 array([1, 1])

31

32 >>> ar.y

33 array([2., 2.])

34

35"""

36 import warnings

37 from collections import Counter

38 from contextlib import nullcontext

39

40 from . import numeric as sb

41 from . import numerictypes as nt

42 from numpy.compat import os_fspath

43 from numpy.core.overrides import set_module

44 from .arrayprint import _get_legacy_print_mode

45

46# All of the functions allow formats to be a dtype

47__all__ = [

48 'record', 'recarray', 'format_parser',

49 'fromarrays', 'fromrecords', 'fromstring', 'fromfile', 'array',

50]

51

52

53 ndarray = sb.ndarray

54

55_byteorderconv = {'b':'>',

56 'l':'<',

57 'n':'=',

58 'B':'>',

59 'L':'<',

60 'N':'=',

61 'S':'s',

62 's':'s',

63 '>':'>',

64 '<':'<',

65 '=':'=',

66 '|':'|',

67 'I':'|',

68 'i':'|'}

69

70# formats regular expression

71# allows multidimensional spec with a tuple syntax in front

72# of the letter code '(2,3)f4' and ' ( 2 , 3 ) f4 '

73# are equally allowed

74

75 numfmt = nt.sctypeDict

76

77

78 def find_duplicate(list):

79 """Find duplication in a list, return a list of duplicated elements"""

80 return [

81 item

82 for item, counts in Counter(list).items()

83 if counts > 1

84 ]

85

86

87@set_module('numpy')

88 class format_parser:

89 """

90 Class to convert formats, names, titles description to a dtype.

91

92 After constructing the format_parser object, the dtype attribute is

93 the converted data-type:

94 ``dtype = format_parser(formats, names, titles).dtype``

95

96 Attributes

97 ----------

98 dtype : dtype

99 The converted data-type.

100

101 Parameters

102 ----------

103 formats : str or list of str

104 The format description, either specified as a string with

105 comma-separated format descriptions in the form ``'f8, i4, a5'``, or

106 a list of format description strings in the form

107 ``['f8', 'i4', 'a5']``.

108 names : str or list/tuple of str

109 The field names, either specified as a comma-separated string in the

110 form ``'col1, col2, col3'``, or as a list or tuple of strings in the

111 form ``['col1', 'col2', 'col3']``.

112 An empty list can be used, in that case default field names

113 ('f0', 'f1', ...) are used.

114 titles : sequence

115 Sequence of title strings. An empty list can be used to leave titles

116 out.

117 aligned : bool, optional

118 If True, align the fields by padding as the C-compiler would.

119 Default is False.

120 byteorder : str, optional

121 If specified, all the fields will be changed to the

122 provided byte-order. Otherwise, the default byte-order is

123 used. For all available string specifiers, see `dtype.newbyteorder`.

124

125 See Also

126 --------

127 dtype, typename, sctype2char

128

129 Examples

130 --------

131 >>> np.format_parser(['<f8', '<i4', '<a5'], ['col1', 'col2', 'col3'],

132 ... ['T1', 'T2', 'T3']).dtype

133 dtype([(('T1', 'col1'), '<f8'), (('T2', 'col2'), '<i4'), (('T3', 'col3'), 'S5')])

134

135 `names` and/or `titles` can be empty lists. If `titles` is an empty list,

136 titles will simply not appear. If `names` is empty, default field names

137 will be used.

138

139 >>> np.format_parser(['f8', 'i4', 'a5'], ['col1', 'col2', 'col3'],

140 ... []).dtype

141 dtype([('col1', '<f8'), ('col2', '<i4'), ('col3', '<S5')])

142 >>> np.format_parser(['<f8', '<i4', '<a5'], [], []).dtype

143 dtype([('f0', '<f8'), ('f1', '<i4'), ('f2', 'S5')])

144

145 """

146

147 def __init__(self, formats, names, titles, aligned=False, byteorder=None):

148 self._parseFormats(formats, aligned)

149 self._setfieldnames(names, titles)

150 self._createdtype(byteorder)

151

152 def _parseFormats(self, formats, aligned=False):

153 """ Parse the field formats """

154

155 if formats is None:

156 raise ValueError("Need formats argument")

157 if isinstance(formats, list):

158 dtype = sb.dtype(

159 [('f{}'.format(i), format_) for i, format_ in enumerate(formats)],

160 aligned,

161 )

162 else:

163 dtype = sb.dtype(formats, aligned)

164 fields = dtype.fields

165 if fields is None:

166 dtype = sb.dtype([('f1', dtype)], aligned)

167 fields = dtype.fields

168 keys = dtype.names

169 self._f_formats = [fields[key][0] for key in keys]

170 self._offsets = [fields[key][1] for key in keys]

171 self._nfields = len(keys)

172

173 def _setfieldnames(self, names, titles):

174 """convert input field names into a list and assign to the _names

175 attribute """

176

177 if names:

178 if type(names) in [list, tuple]:

179 pass

180 elif isinstance(names, str):

181 names = names.split(',')

182 else:

183 raise NameError("illegal input names %s" % repr(names))

184

185 self._names = [n.strip() for n in names[:self._nfields]]

186 else:

187 self._names = []

188

189 # if the names are not specified, they will be assigned as

190 # "f0, f1, f2,..."

191 # if not enough names are specified, they will be assigned as "f[n],

192 # f[n+1],..." etc. where n is the number of specified names..."

193 self._names += ['f%d' % i for i in range(len(self._names),

194 self._nfields)]

195 # check for redundant names

196 _dup = find_duplicate(self._names)

197 if _dup:

198 raise ValueError("Duplicate field names: %s" % _dup)

199

200 if titles:

201 self._titles = [n.strip() for n in titles[:self._nfields]]

202 else:

203 self._titles = []

204 titles = []

205

206 if self._nfields > len(titles):

207 self._titles += [None] * (self._nfields - len(titles))

208

209 def _createdtype(self, byteorder):

210 dtype = sb.dtype({

211 'names': self._names,

212 'formats': self._f_formats,

213 'offsets': self._offsets,

214 'titles': self._titles,

215 })

216 if byteorder is not None:

217 byteorder = _byteorderconv[byteorder[0]]

218 dtype = dtype.newbyteorder(byteorder)

219

220 self.dtype = dtype

221

222

223 class record(nt.void):

224 """A data-type scalar that allows field access as attribute lookup.

225 """

226

227 # manually set name and module so that this class's type shows up

228 # as numpy.record when printed

229 __name__ = 'record'

230 __module__ = 'numpy'

231

232 def __repr__(self):

233 if _get_legacy_print_mode() <= 113:

234 return self.__str__()

235 return super().__repr__()

236

237 def __str__(self):

238 if _get_legacy_print_mode() <= 113:

239 return str(self.item())

240 return super().__str__()

241

242 def __getattribute__(self, attr):

243 if attr in ('setfield', 'getfield', 'dtype'):

244 return nt.void.__getattribute__(self, attr)

245 try:

246 return nt.void.__getattribute__(self, attr)

247 except AttributeError:

248 pass

249 fielddict = nt.void.__getattribute__(self, 'dtype').fields

250 res = fielddict.get(attr, None)

251 if res:

252 obj = self.getfield(*res[:2])

253 # if it has fields return a record,

254 # otherwise return the object

255 try:

256 dt = obj.dtype

257 except AttributeError:

258 #happens if field is Object type

259 return obj

260 if dt.names is not None:

261 return obj.view((self.__class__, obj.dtype))

262 return obj

263 else:

264 raise AttributeError("'record' object has no "

265 "attribute '%s'" % attr)

266

267 def __setattr__(self, attr, val):

268 if attr in ('setfield', 'getfield', 'dtype'):

269 raise AttributeError("Cannot set '%s' attribute" % attr)

270 fielddict = nt.void.__getattribute__(self, 'dtype').fields

271 res = fielddict.get(attr, None)

272 if res:

273 return self.setfield(val, *res[:2])

274 else:

275 if getattr(self, attr, None):

276 return nt.void.__setattr__(self, attr, val)

277 else:

278 raise AttributeError("'record' object has no "

279 "attribute '%s'" % attr)

280

281 def __getitem__(self, indx):

282 obj = nt.void.__getitem__(self, indx)

283

284 # copy behavior of record.__getattribute__,

285 if isinstance(obj, nt.void) and obj.dtype.names is not None:

286 return obj.view((self.__class__, obj.dtype))

287 else:

288 # return a single element

289 return obj

290

291 def pprint(self):

292 """Pretty-print all fields."""

293 # pretty-print all fields

294 names = self.dtype.names

295 maxlen = max(len(name) for name in names)

296 fmt = '%% %ds: %%s' % maxlen

297 rows = [fmt % (name, getattr(self, name)) for name in names]

298 return "\n".join(rows)

299

300# The recarray is almost identical to a standard array (which supports

301# named fields already) The biggest difference is that it can use

302# attribute-lookup to find the fields and it is constructed using

303# a record.

304

305# If byteorder is given it forces a particular byteorder on all

306# the fields (and any subfields)

307

308 class recarray(ndarray):

309 """Construct an ndarray that allows field access using attributes.

310

311 Arrays may have a data-types containing fields, analogous

312 to columns in a spread sheet. An example is ``[(x, int), (y, float)]``,

313 where each entry in the array is a pair of ``(int, float)``. Normally,

314 these attributes are accessed using dictionary lookups such as ``arr['x']``

315 and ``arr['y']``. Record arrays allow the fields to be accessed as members

316 of the array, using ``arr.x`` and ``arr.y``.

317

318 Parameters

319 ----------

320 shape : tuple

321 Shape of output array.

322 dtype : data-type, optional

323 The desired data-type. By default, the data-type is determined

324 from `formats`, `names`, `titles`, `aligned` and `byteorder`.

325 formats : list of data-types, optional

326 A list containing the data-types for the different columns, e.g.

327 ``['i4', 'f8', 'i4']``. `formats` does *not* support the new

328 convention of using types directly, i.e. ``(int, float, int)``.

329 Note that `formats` must be a list, not a tuple.

330 Given that `formats` is somewhat limited, we recommend specifying

331 `dtype` instead.

332 names : tuple of str, optional

333 The name of each column, e.g. ``('x', 'y', 'z')``.

334 buf : buffer, optional

335 By default, a new array is created of the given shape and data-type.

336 If `buf` is specified and is an object exposing the buffer interface,

337 the array will use the memory from the existing buffer. In this case,

338 the `offset` and `strides` keywords are available.

339

340 Other Parameters

341 ----------------

342 titles : tuple of str, optional

343 Aliases for column names. For example, if `names` were

344 ``('x', 'y', 'z')`` and `titles` is

345 ``('x_coordinate', 'y_coordinate', 'z_coordinate')``, then

346 ``arr['x']`` is equivalent to both ``arr.x`` and ``arr.x_coordinate``.

347 byteorder : {'<', '>', '='}, optional

348 Byte-order for all fields.

349 aligned : bool, optional

350 Align the fields in memory as the C-compiler would.

351 strides : tuple of ints, optional

352 Buffer (`buf`) is interpreted according to these strides (strides

353 define how many bytes each array element, row, column, etc.

354 occupy in memory).

355 offset : int, optional

356 Start reading buffer (`buf`) from this offset onwards.

357 order : {'C', 'F'}, optional

358 Row-major (C-style) or column-major (Fortran-style) order.

359

360 Returns

361 -------

362 rec : recarray

363 Empty array of the given shape and type.

364

365 See Also

366 --------

367 core.records.fromrecords : Construct a record array from data.

368 record : fundamental data-type for `recarray`.

369 format_parser : determine a data-type from formats, names, titles.

370

371 Notes

372 -----

373 This constructor can be compared to ``empty``: it creates a new record

374 array but does not fill it with data. To create a record array from data,

375 use one of the following methods:

376

377 1. Create a standard ndarray and convert it to a record array,

378 using ``arr.view(np.recarray)``

379 2. Use the `buf` keyword.

380 3. Use `np.rec.fromrecords`.

381

382 Examples

383 --------

384 Create an array with two fields, ``x`` and ``y``:

385

386 >>> x = np.array([(1.0, 2), (3.0, 4)], dtype=[('x', '<f8'), ('y', '<i8')])

387 >>> x

388 array([(1., 2), (3., 4)], dtype=[('x', '<f8'), ('y', '<i8')])

389

390 >>> x['x']

391 array([1., 3.])

392

393 View the array as a record array:

394

395 >>> x = x.view(np.recarray)

396

397 >>> x.x

398 array([1., 3.])

399

400 >>> x.y

401 array([2, 4])

402

403 Create a new, empty record array:

404

405 >>> np.recarray((2,),

406 ... dtype=[('x', int), ('y', float), ('z', int)]) #doctest: +SKIP

407 rec.array([(-1073741821, 1.2249118382103472e-301, 24547520),

408 (3471280, 1.2134086255804012e-316, 0)],

409 dtype=[('x', '<i4'), ('y', '<f8'), ('z', '<i4')])

410

411 """

412

413 # manually set name and module so that this class's type shows

414 # up as "numpy.recarray" when printed

415 __name__ = 'recarray'

416 __module__ = 'numpy'

417

418 def __new__(subtype, shape, dtype=None, buf=None, offset=0, strides=None,

419 formats=None, names=None, titles=None,

420 byteorder=None, aligned=False, order='C'):

421

422 if dtype is not None:

423 descr = sb.dtype(dtype)

424 else:

425 descr = format_parser(formats, names, titles, aligned, byteorder).dtype

426

427 if buf is None:

428 self = ndarray.__new__(subtype, shape, (record, descr), order=order)

429 else:

430 self = ndarray.__new__(subtype, shape, (record, descr),

431 buffer=buf, offset=offset,

432 strides=strides, order=order)

433 return self

434

435 def __array_finalize__(self, obj):

436 if self.dtype.type is not record and self.dtype.names is not None:

437 # if self.dtype is not np.record, invoke __setattr__ which will

438 # convert it to a record if it is a void dtype.

439 self.dtype = self.dtype

440

441 def __getattribute__(self, attr):

442 # See if ndarray has this attr, and return it if so. (note that this

443 # means a field with the same name as an ndarray attr cannot be

444 # accessed by attribute).

445 try:

446 return object.__getattribute__(self, attr)

447 except AttributeError: # attr must be a fieldname

448 pass

449

450 # look for a field with this name

451 fielddict = ndarray.__getattribute__(self, 'dtype').fields

452 try:

453 res = fielddict[attr][:2]

454 except (TypeError, KeyError) as e:

455 raise AttributeError("recarray has no attribute %s" % attr) from e

456 obj = self.getfield(*res)

457

458 # At this point obj will always be a recarray, since (see

459 # PyArray_GetField) the type of obj is inherited. Next, if obj.dtype is

460 # non-structured, convert it to an ndarray. Then if obj is structured

461 # with void type convert it to the same dtype.type (eg to preserve

462 # numpy.record type if present), since nested structured fields do not

463 # inherit type. Don't do this for non-void structures though.

464 if obj.dtype.names is not None:

465 if issubclass(obj.dtype.type, nt.void):

466 return obj.view(dtype=(self.dtype.type, obj.dtype))

467 return obj

468 else:

469 return obj.view(ndarray)

470

471 # Save the dictionary.

472 # If the attr is a field name and not in the saved dictionary

473 # Undo any "setting" of the attribute and do a setfield

474 # Thus, you can't create attributes on-the-fly that are field names.

475 def __setattr__(self, attr, val):

476

477 # Automatically convert (void) structured types to records

478 # (but not non-void structures, subarrays, or non-structured voids)

479 if attr == 'dtype' and issubclass(val.type, nt.void) and val.names is not None:

480 val = sb.dtype((record, val))

481

482 newattr = attr not in self.__dict__

483 try:

484 ret = object.__setattr__(self, attr, val)

485 except Exception:

486 fielddict = ndarray.__getattribute__(self, 'dtype').fields or {}

487 if attr not in fielddict:

488 raise

489 else:

490 fielddict = ndarray.__getattribute__(self, 'dtype').fields or {}

491 if attr not in fielddict:

492 return ret

493 if newattr:

494 # We just added this one or this setattr worked on an

495 # internal attribute.

496 try:

497 object.__delattr__(self, attr)

498 except Exception:

499 return ret

500 try:

501 res = fielddict[attr][:2]

502 except (TypeError, KeyError) as e:

503 raise AttributeError(

504 "record array has no attribute %s" % attr

505 ) from e

506 return self.setfield(val, *res)

507

508 def __getitem__(self, indx):

509 obj = super().__getitem__(indx)

510

511 # copy behavior of getattr, except that here

512 # we might also be returning a single element

513 if isinstance(obj, ndarray):

514 if obj.dtype.names is not None:

515 obj = obj.view(type(self))

516 if issubclass(obj.dtype.type, nt.void):

517 return obj.view(dtype=(self.dtype.type, obj.dtype))

518 return obj

519 else:

520 return obj.view(type=ndarray)

521 else:

522 # return a single element

523 return obj

524

525 def __repr__(self):

526

527 repr_dtype = self.dtype

528 if self.dtype.type is record or not issubclass(self.dtype.type, nt.void):

529 # If this is a full record array (has numpy.record dtype),

530 # or if it has a scalar (non-void) dtype with no records,

531 # represent it using the rec.array function. Since rec.array

532 # converts dtype to a numpy.record for us, convert back

533 # to non-record before printing

534 if repr_dtype.type is record:

535 repr_dtype = sb.dtype((nt.void, repr_dtype))

536 prefix = "rec.array("

537 fmt = 'rec.array(%s,%sdtype=%s)'

538 else:

539 # otherwise represent it using np.array plus a view

540 # This should only happen if the user is playing

541 # strange games with dtypes.

542 prefix = "array("

543 fmt = 'array(%s,%sdtype=%s).view(numpy.recarray)'

544

545 # get data/shape string. logic taken from numeric.array_repr

546 if self.size > 0 or self.shape == (0,):

547 lst = sb.array2string(

548 self, separator=', ', prefix=prefix, suffix=',')

549 else:

550 # show zero-length shape unless it is (0,)

551 lst = "[], shape=%s" % (repr(self.shape),)

552

553 lf = '\n'+' '*len(prefix)

554 if _get_legacy_print_mode() <= 113:

555 lf = ' ' + lf # trailing space

556 return fmt % (lst, lf, repr_dtype)

557

558 def field(self, attr, val=None):

559 if isinstance(attr, int):

560 names = ndarray.__getattribute__(self, 'dtype').names

561 attr = names[attr]

562

563 fielddict = ndarray.__getattribute__(self, 'dtype').fields

564

565 res = fielddict[attr][:2]

566

567 if val is None:

568 obj = self.getfield(*res)

569 if obj.dtype.names is not None:

570 return obj

571 return obj.view(ndarray)

572 else:

573 return self.setfield(val, *res)

574

575

576 def _deprecate_shape_0_as_None(shape):

577 if shape == 0:

578 warnings.warn(

579 "Passing `shape=0` to have the shape be inferred is deprecated, "

580 "and in future will be equivalent to `shape=(0,)`. To infer "

581 "the shape and suppress this warning, pass `shape=None` instead.",

582 FutureWarning, stacklevel=3)

583 return None

584 else:

585 return shape

586

587

588@set_module("numpy.rec")

589 def fromarrays(arrayList, dtype=None, shape=None, formats=None,

590 names=None, titles=None, aligned=False, byteorder=None):

591 """Create a record array from a (flat) list of arrays

592

593 Parameters

594 ----------

595 arrayList : list or tuple

596 List of array-like objects (such as lists, tuples,

597 and ndarrays).

598 dtype : data-type, optional

599 valid dtype for all arrays

600 shape : int or tuple of ints, optional

601 Shape of the resulting array. If not provided, inferred from

602 ``arrayList[0]``.

603 formats, names, titles, aligned, byteorder :

604 If `dtype` is ``None``, these arguments are passed to

605 `numpy.format_parser` to construct a dtype. See that function for

606 detailed documentation.

607

608 Returns

609 -------

610 np.recarray

611 Record array consisting of given arrayList columns.

612

613 Examples

614 --------

615 >>> x1=np.array([1,2,3,4])

616 >>> x2=np.array(['a','dd','xyz','12'])

617 >>> x3=np.array([1.1,2,3,4])

618 >>> r = np.core.records.fromarrays([x1,x2,x3],names='a,b,c')

619 >>> print(r[1])

620 (2, 'dd', 2.0) # may vary

621 >>> x1[1]=34

622 >>> r.a

623 array([1, 2, 3, 4])

624

625 >>> x1 = np.array([1, 2, 3, 4])

626 >>> x2 = np.array(['a', 'dd', 'xyz', '12'])

627 >>> x3 = np.array([1.1, 2, 3,4])

628 >>> r = np.core.records.fromarrays(

629 ... [x1, x2, x3],

630 ... dtype=np.dtype([('a', np.int32), ('b', 'S3'), ('c', np.float32)]))

631 >>> r

632 rec.array([(1, b'a', 1.1), (2, b'dd', 2. ), (3, b'xyz', 3. ),

633 (4, b'12', 4. )],

634 dtype=[('a', '<i4'), ('b', 'S3'), ('c', '<f4')])

635 """

636

637 arrayList = [sb.asarray(x) for x in arrayList]

638

639 # NumPy 1.19.0, 2020-01-01

640 shape = _deprecate_shape_0_as_None(shape)

641

642 if shape is None:

643 shape = arrayList[0].shape

644 elif isinstance(shape, int):

645 shape = (shape,)

646

647 if formats is None and dtype is None:

648 # go through each object in the list to see if it is an ndarray

649 # and determine the formats.

650 formats = [obj.dtype for obj in arrayList]

651

652 if dtype is not None:

653 descr = sb.dtype(dtype)

654 else:

655 descr = format_parser(formats, names, titles, aligned, byteorder).dtype

656 _names = descr.names

657

658 # Determine shape from data-type.

659 if len(descr) != len(arrayList):

660 raise ValueError("mismatch between the number of fields "

661 "and the number of arrays")

662

663 d0 = descr[0].shape

664 nn = len(d0)

665 if nn > 0:

666 shape = shape[:-nn]

667

668 _array = recarray(shape, descr)

669

670 # populate the record array (makes a copy)

671 for k, obj in enumerate(arrayList):

672 nn = descr[k].ndim

673 testshape = obj.shape[:obj.ndim - nn]

674 name = _names[k]

675 if testshape != shape:

676 raise ValueError(f'array-shape mismatch in array {k} ("{name}")')

677

678 _array[name] = obj

679

680 return _array

681

682

683@set_module("numpy.rec")

684 def fromrecords(recList, dtype=None, shape=None, formats=None, names=None,

685 titles=None, aligned=False, byteorder=None):

686 """Create a recarray from a list of records in text form.

687

688 Parameters

689 ----------

690 recList : sequence

691 data in the same field may be heterogeneous - they will be promoted

692 to the highest data type.

693 dtype : data-type, optional

694 valid dtype for all arrays

695 shape : int or tuple of ints, optional

696 shape of each array.

697 formats, names, titles, aligned, byteorder :

698 If `dtype` is ``None``, these arguments are passed to

699 `numpy.format_parser` to construct a dtype. See that function for

700 detailed documentation.

701

702 If both `formats` and `dtype` are None, then this will auto-detect

703 formats. Use list of tuples rather than list of lists for faster

704 processing.

705

706 Returns

707 -------

708 np.recarray

709 record array consisting of given recList rows.

710

711 Examples

712 --------

713 >>> r=np.core.records.fromrecords([(456,'dbe',1.2),(2,'de',1.3)],

714 ... names='col1,col2,col3')

715 >>> print(r[0])

716 (456, 'dbe', 1.2)

717 >>> r.col1

718 array([456, 2])

719 >>> r.col2

720 array(['dbe', 'de'], dtype='<U3')

721 >>> import pickle

722 >>> pickle.loads(pickle.dumps(r))

723 rec.array([(456, 'dbe', 1.2), ( 2, 'de', 1.3)],

724 dtype=[('col1', '<i8'), ('col2', '<U3'), ('col3', '<f8')])

725 """

726

727 if formats is None and dtype is None: # slower

728 obj = sb.array(recList, dtype=object)

729 arrlist = [sb.array(obj[..., i].tolist()) for i in range(obj.shape[-1])]

730 return fromarrays(arrlist, formats=formats, shape=shape, names=names,

731 titles=titles, aligned=aligned, byteorder=byteorder)

732

733 if dtype is not None:

734 descr = sb.dtype((record, dtype))

735 else:

736 descr = format_parser(formats, names, titles, aligned, byteorder).dtype

737

738 try:

739 retval = sb.array(recList, dtype=descr)

740 except (TypeError, ValueError):

741 # NumPy 1.19.0, 2020-01-01

742 shape = _deprecate_shape_0_as_None(shape)

743 if shape is None:

744 shape = len(recList)

745 if isinstance(shape, int):

746 shape = (shape,)

747 if len(shape) > 1:

748 raise ValueError("Can only deal with 1-d array.")

749 _array = recarray(shape, descr)

750 for k in range(_array.size):

751 _array[k] = tuple(recList[k])

752 # list of lists instead of list of tuples ?

753 # 2018-02-07, 1.14.1

754 warnings.warn(

755 "fromrecords expected a list of tuples, may have received a list "

756 "of lists instead. In the future that will raise an error",

757 FutureWarning, stacklevel=2)

758 return _array

759 else:

760 if shape is not None and retval.shape != shape:

761 retval.shape = shape

762

763 res = retval.view(recarray)

764

765 return res

766

767

768@set_module("numpy.rec")

769 def fromstring(datastring, dtype=None, shape=None, offset=0, formats=None,

770 names=None, titles=None, aligned=False, byteorder=None):

771 r"""Create a record array from binary data

772

773 Note that despite the name of this function it does not accept `str`

774 instances.

775

776 Parameters

777 ----------

778 datastring : bytes-like

779 Buffer of binary data

780 dtype : data-type, optional

781 Valid dtype for all arrays

782 shape : int or tuple of ints, optional

783 Shape of each array.

784 offset : int, optional

785 Position in the buffer to start reading from.

786 formats, names, titles, aligned, byteorder :

787 If `dtype` is ``None``, these arguments are passed to

788 `numpy.format_parser` to construct a dtype. See that function for

789 detailed documentation.

790

791

792 Returns

793 -------

794 np.recarray

795 Record array view into the data in datastring. This will be readonly

796 if `datastring` is readonly.

797

798 See Also

799 --------

800 numpy.frombuffer

801

802 Examples

803 --------

804 >>> a = b'\x01\x02\x03abc'

805 >>> np.core.records.fromstring(a, dtype='u1,u1,u1,S3')

806 rec.array([(1, 2, 3, b'abc')],

807 dtype=[('f0', 'u1'), ('f1', 'u1'), ('f2', 'u1'), ('f3', 'S3')])

808

809 >>> grades_dtype = [('Name', (np.str_, 10)), ('Marks', np.float64),

810 ... ('GradeLevel', np.int32)]

811 >>> grades_array = np.array([('Sam', 33.3, 3), ('Mike', 44.4, 5),

812 ... ('Aadi', 66.6, 6)], dtype=grades_dtype)

813 >>> np.core.records.fromstring(grades_array.tobytes(), dtype=grades_dtype)

814 rec.array([('Sam', 33.3, 3), ('Mike', 44.4, 5), ('Aadi', 66.6, 6)],

815 dtype=[('Name', '<U10'), ('Marks', '<f8'), ('GradeLevel', '<i4')])

816

817 >>> s = '\x01\x02\x03abc'

818 >>> np.core.records.fromstring(s, dtype='u1,u1,u1,S3')

819 Traceback (most recent call last)

820 ...

821 TypeError: a bytes-like object is required, not 'str'

822 """

823

824 if dtype is None and formats is None:

825 raise TypeError("fromstring() needs a 'dtype' or 'formats' argument")

826

827 if dtype is not None:

828 descr = sb.dtype(dtype)

829 else:

830 descr = format_parser(formats, names, titles, aligned, byteorder).dtype

831

832 itemsize = descr.itemsize

833

834 # NumPy 1.19.0, 2020-01-01

835 shape = _deprecate_shape_0_as_None(shape)

836

837 if shape in (None, -1):

838 shape = (len(datastring) - offset) // itemsize

839

840 _array = recarray(shape, descr, buf=datastring, offset=offset)

841 return _array

842

843 def get_remaining_size(fd):

844 pos = fd.tell()

845 try:

846 fd.seek(0, 2)

847 return fd.tell() - pos

848 finally:

849 fd.seek(pos, 0)

850

851

852@set_module("numpy.rec")

853 def fromfile(fd, dtype=None, shape=None, offset=0, formats=None,

854 names=None, titles=None, aligned=False, byteorder=None):

855 """Create an array from binary file data

856

857 Parameters

858 ----------

859 fd : str or file type

860 If file is a string or a path-like object then that file is opened,

861 else it is assumed to be a file object. The file object must

862 support random access (i.e. it must have tell and seek methods).

863 dtype : data-type, optional

864 valid dtype for all arrays

865 shape : int or tuple of ints, optional

866 shape of each array.

867 offset : int, optional

868 Position in the file to start reading from.

869 formats, names, titles, aligned, byteorder :

870 If `dtype` is ``None``, these arguments are passed to

871 `numpy.format_parser` to construct a dtype. See that function for

872 detailed documentation

873

874 Returns

875 -------

876 np.recarray

877 record array consisting of data enclosed in file.

878

879 Examples

880 --------

881 >>> from tempfile import TemporaryFile

882 >>> a = np.empty(10,dtype='f8,i4,a5')

883 >>> a[5] = (0.5,10,'abcde')

884 >>>

885 >>> fd=TemporaryFile()

886 >>> a = a.newbyteorder('<')

887 >>> a.tofile(fd)

888 >>>

889 >>> _ = fd.seek(0)

890 >>> r=np.core.records.fromfile(fd, formats='f8,i4,a5', shape=10,

891 ... byteorder='<')

892 >>> print(r[5])

893 (0.5, 10, 'abcde')

894 >>> r.shape

895 (10,)

896 """

897

898 if dtype is None and formats is None:

899 raise TypeError("fromfile() needs a 'dtype' or 'formats' argument")

900

901 # NumPy 1.19.0, 2020-01-01

902 shape = _deprecate_shape_0_as_None(shape)

903

904 if shape is None:

905 shape = (-1,)

906 elif isinstance(shape, int):

907 shape = (shape,)

908

909 if hasattr(fd, 'readinto'):

910 # GH issue 2504. fd supports io.RawIOBase or io.BufferedIOBase interface.

911 # Example of fd: gzip, BytesIO, BufferedReader

912 # file already opened

913 ctx = nullcontext(fd)

914 else:

915 # open file

916 ctx = open(os_fspath(fd), 'rb')

917

918 with ctx as fd:

919 if offset > 0:

920 fd.seek(offset, 1)

921 size = get_remaining_size(fd)

922

923 if dtype is not None:

924 descr = sb.dtype(dtype)

925 else:

926 descr = format_parser(formats, names, titles, aligned, byteorder).dtype

927

928 itemsize = descr.itemsize

929

930 shapeprod = sb.array(shape).prod(dtype=nt.intp)

931 shapesize = shapeprod * itemsize

932 if shapesize < 0:

933 shape = list(shape)

934 shape[shape.index(-1)] = size // -shapesize

935 shape = tuple(shape)

936 shapeprod = sb.array(shape).prod(dtype=nt.intp)

937

938 nbytes = shapeprod * itemsize

939

940 if nbytes > size:

941 raise ValueError(

942 "Not enough bytes left in file for specified shape and type")

943

944 # create the array

945 _array = recarray(shape, descr)

946 nbytesread = fd.readinto(_array.data)

947 if nbytesread != nbytes:

948 raise OSError("Didn't read as many bytes as expected")

949

950 return _array

951

952

953@set_module("numpy.rec")

954 def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None,

955 names=None, titles=None, aligned=False, byteorder=None, copy=True):

956 """

957 Construct a record array from a wide-variety of objects.

958

959 A general-purpose record array constructor that dispatches to the

960 appropriate `recarray` creation function based on the inputs (see Notes).

961

962 Parameters

963 ----------

964 obj : any

965 Input object. See Notes for details on how various input types are

966 treated.

967 dtype : data-type, optional

968 Valid dtype for array.

969 shape : int or tuple of ints, optional

970 Shape of each array.

971 offset : int, optional

972 Position in the file or buffer to start reading from.

973 strides : tuple of ints, optional

974 Buffer (`buf`) is interpreted according to these strides (strides

975 define how many bytes each array element, row, column, etc.

976 occupy in memory).

977 formats, names, titles, aligned, byteorder :

978 If `dtype` is ``None``, these arguments are passed to

979 `numpy.format_parser` to construct a dtype. See that function for

980 detailed documentation.

981 copy : bool, optional

982 Whether to copy the input object (True), or to use a reference instead.

983 This option only applies when the input is an ndarray or recarray.

984 Defaults to True.

985

986 Returns

987 -------

988 np.recarray

989 Record array created from the specified object.

990

991 Notes

992 -----

993 If `obj` is ``None``, then call the `~numpy.recarray` constructor. If

994 `obj` is a string, then call the `fromstring` constructor. If `obj` is a

995 list or a tuple, then if the first object is an `~numpy.ndarray`, call

996 `fromarrays`, otherwise call `fromrecords`. If `obj` is a

997 `~numpy.recarray`, then make a copy of the data in the recarray

998 (if ``copy=True``) and use the new formats, names, and titles. If `obj`

999 is a file, then call `fromfile`. Finally, if obj is an `ndarray`, then

1000 return ``obj.view(recarray)``, making a copy of the data if ``copy=True``.

1001

1002 Examples

1003 --------

1004 >>> a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])

1005 array([[1, 2, 3],

1006 [4, 5, 6],

1007 [7, 8, 9]])

1008

1009 >>> np.core.records.array(a)

1010 rec.array([[1, 2, 3],

1011 [4, 5, 6],

1012 [7, 8, 9]],

1013 dtype=int32)

1014

1015 >>> b = [(1, 1), (2, 4), (3, 9)]

1016 >>> c = np.core.records.array(b, formats = ['i2', 'f2'], names = ('x', 'y'))

1017 >>> c

1018 rec.array([(1, 1.0), (2, 4.0), (3, 9.0)],

1019 dtype=[('x', '<i2'), ('y', '<f2')])

1020

1021 >>> c.x

1022 rec.array([1, 2, 3], dtype=int16)

1023

1024 >>> c.y

1025 rec.array([ 1.0, 4.0, 9.0], dtype=float16)

1026

1027 >>> r = np.rec.array(['abc','def'], names=['col1','col2'])

1028 >>> print(r.col1)

1029 abc

1030

1031 >>> r.col1

1032 array('abc', dtype='<U3')

1033

1034 >>> r.col2

1035 array('def', dtype='<U3')

1036 """

1037

1038 if ((isinstance(obj, (type(None), str)) or hasattr(obj, 'readinto')) and

1039 formats is None and dtype is None):

1040 raise ValueError("Must define formats (or dtype) if object is "

1041 "None, string, or an open file")

1042

1043 kwds = {}

1044 if dtype is not None:

1045 dtype = sb.dtype(dtype)

1046 elif formats is not None:

1047 dtype = format_parser(formats, names, titles,

1048 aligned, byteorder).dtype

1049 else:

1050 kwds = {'formats': formats,

1051 'names': names,

1052 'titles': titles,

1053 'aligned': aligned,

1054 'byteorder': byteorder

1055 }

1056

1057 if obj is None:

1058 if shape is None:

1059 raise ValueError("Must define a shape if obj is None")

1060 return recarray(shape, dtype, buf=obj, offset=offset, strides=strides)

1061

1062 elif isinstance(obj, bytes):

1063 return fromstring(obj, dtype, shape=shape, offset=offset, **kwds)

1064

1065 elif isinstance(obj, (list, tuple)):

1066 if isinstance(obj[0], (tuple, list)):

1067 return fromrecords(obj, dtype=dtype, shape=shape, **kwds)

1068 else:

1069 return fromarrays(obj, dtype=dtype, shape=shape, **kwds)

1070

1071 elif isinstance(obj, recarray):

1072 if dtype is not None and (obj.dtype != dtype):

1073 new = obj.view(dtype)

1074 else:

1075 new = obj

1076 if copy:

1077 new = new.copy()

1078 return new

1079

1080 elif hasattr(obj, 'readinto'):

1081 return fromfile(obj, dtype=dtype, shape=shape, offset=offset)

1082

1083 elif isinstance(obj, ndarray):

1084 if dtype is not None and (obj.dtype != dtype):

1085 new = obj.view(dtype)

1086 else:

1087 new = obj

1088 if copy:

1089 new = new.copy()

1090 return new.view(recarray)

1091

1092 else:

1093 interface = getattr(obj, "__array_interface__", None)

1094 if interface is None or not isinstance(interface, dict):

1095 raise ValueError("Unknown input type")

1096 obj = sb.array(obj)

1097 if dtype is not None and (obj.dtype != dtype):

1098 obj = obj.view(dtype)

1099 return obj.view(recarray)