codekingpro/portable-devtools
114k
1import collections.abc
2import pickle
3import textwrap
4from io import BytesIO
5from os import path
6from pathlib import Path
7
8import pytest
9
10import numpy as np
11from numpy.testing import (
12 assert_,
13 assert_array_almost_equal,
14 assert_array_equal,
15 assert_equal,
16 assert_raises,
17 temppath,
18)
19
20
21class TestFromrecords:
22 def test_fromrecords(self):
23 r = np.rec.fromrecords([[456, 'dbe', 1.2], [2, 'de', 1.3]],
24 names='col1,col2,col3')
25 assert_equal(r[0].item(), (456, 'dbe', 1.2))
26 assert_equal(r['col1'].dtype.kind, 'i')
27 assert_equal(r['col2'].dtype.kind, 'U')
28 assert_equal(r['col2'].dtype.itemsize, 12)
29 assert_equal(r['col3'].dtype.kind, 'f')
30
31 def test_fromrecords_0len(self):
32 """ Verify fromrecords works with a 0-length input """
33 dtype = [('a', float), ('b', float)]
34 r = np.rec.fromrecords([], dtype=dtype)
35 assert_equal(r.shape, (0,))
36
37 def test_fromrecords_2d(self):
38 data = [
39 [(1, 2), (3, 4), (5, 6)],
40 [(6, 5), (4, 3), (2, 1)]
41 ]
42 expected_a = [[1, 3, 5], [6, 4, 2]]
43 expected_b = [[2, 4, 6], [5, 3, 1]]
44
45 # try with dtype
46 r1 = np.rec.fromrecords(data, dtype=[('a', int), ('b', int)])
47 assert_equal(r1['a'], expected_a)
48 assert_equal(r1['b'], expected_b)
49
50 # try with names
51 r2 = np.rec.fromrecords(data, names=['a', 'b'])
52 assert_equal(r2['a'], expected_a)
53 assert_equal(r2['b'], expected_b)
54
55 assert_equal(r1, r2)
56
57 def test_method_array(self):
58 r = np.rec.array(
59 b'abcdefg' * 100, formats='i2,S3,i4', shape=3, byteorder='big'
60 )
61 assert_equal(r[1].item(), (25444, b'efg', 1633837924))
62
63 def test_method_array2(self):
64 r = np.rec.array(
65 [
66 (1, 11, 'a'), (2, 22, 'b'), (3, 33, 'c'), (4, 44, 'd'),
67 (5, 55, 'ex'), (6, 66, 'f'), (7, 77, 'g')
68 ],
69 formats='u1,f4,S1'
70 )
71 assert_equal(r[1].item(), (2, 22.0, b'b'))
72
73 def test_recarray_slices(self):
74 r = np.rec.array(
75 [
76 (1, 11, 'a'), (2, 22, 'b'), (3, 33, 'c'), (4, 44, 'd'),
77 (5, 55, 'ex'), (6, 66, 'f'), (7, 77, 'g')
78 ],
79 formats='u1,f4,S1'
80 )
81 assert_equal(r[1::2][1].item(), (4, 44.0, b'd'))
82
83 def test_recarray_fromarrays(self):
84 x1 = np.array([1, 2, 3, 4])
85 x2 = np.array(['a', 'dd', 'xyz', '12'])
86 x3 = np.array([1.1, 2, 3, 4])
87 r = np.rec.fromarrays([x1, x2, x3], names='a,b,c')
88 assert_equal(r[1].item(), (2, 'dd', 2.0))
89 x1[1] = 34
90 assert_equal(r.a, np.array([1, 2, 3, 4]))
91
92 def test_recarray_fromfile(self):
93 data_dir = path.join(path.dirname(__file__), 'data')
94 filename = path.join(data_dir, 'recarray_from_file.fits')
95 fd = open(filename, 'rb')
96 fd.seek(2880 * 2)
97 r1 = np.rec.fromfile(fd, formats='f8,i4,S5', shape=3, byteorder='big')
98 fd.seek(2880 * 2)
99 r2 = np.rec.array(fd, formats='f8,i4,S5', shape=3, byteorder='big')
100 fd.seek(2880 * 2)
101 bytes_array = BytesIO()
102 bytes_array.write(fd.read())
103 bytes_array.seek(0)
104 r3 = np.rec.fromfile(
105 bytes_array, formats='f8,i4,S5', shape=3, byteorder='big'
106 )
107 fd.close()
108 assert_equal(r1, r2)
109 assert_equal(r2, r3)
110
111 def test_recarray_from_obj(self):
112 count = 10
113 a = np.zeros(count, dtype='O')
114 b = np.zeros(count, dtype='f8')
115 c = np.zeros(count, dtype='f8')
116 for i in range(len(a)):
117 a[i] = list(range(1, 10))
118
119 mine = np.rec.fromarrays([a, b, c], names='date,data1,data2')
120 for i in range(len(a)):
121 assert_(mine.date[i] == list(range(1, 10)))
122 assert_(mine.data1[i] == 0.0)
123 assert_(mine.data2[i] == 0.0)
124
125 def test_recarray_repr(self):
126 a = np.array([(1, 0.1), (2, 0.2)],
127 dtype=[('foo', '<i4'), ('bar', '<f8')])
128 a = np.rec.array(a)
129 assert_equal(
130 repr(a),
131 textwrap.dedent("""\
132 rec.array([(1, 0.1), (2, 0.2)],
133 dtype=[('foo', '<i4'), ('bar', '<f8')])""")
134 )
135
136 # make sure non-structured dtypes also show up as rec.array
137 a = np.array(np.ones(4, dtype='f8'))
138 assert_(repr(np.rec.array(a)).startswith('rec.array'))
139
140 # check that the 'np.record' part of the dtype isn't shown
141 a = np.rec.array(np.ones(3, dtype='i4,i4'))
142 assert_equal(repr(a).find('numpy.record'), -1)
143 a = np.rec.array(np.ones(3, dtype='i4'))
144 assert_(repr(a).find('dtype=int32') != -1)
145
146 def test_0d_recarray_repr(self):
147 arr_0d = np.rec.array((1, 2.0, '2003'), dtype='<i4,<f8,<M8[Y]')
148 assert_equal(repr(arr_0d), textwrap.dedent("""\
149 rec.array((1, 2., '2003'),
150 dtype=[('f0', '<i4'), ('f1', '<f8'), ('f2', '<M8[Y]')])"""))
151
152 record = arr_0d[()]
153 assert_equal(repr(record),
154 "np.record((1, 2.0, '2003'), "
155 "dtype=[('f0', '<i4'), ('f1', '<f8'), ('f2', '<M8[Y]')])")
156 # 1.13 converted to python scalars before the repr
157 try:
158 np.set_printoptions(legacy='1.13')
159 assert_equal(repr(record), '(1, 2.0, datetime.date(2003, 1, 1))')
160 finally:
161 np.set_printoptions(legacy=False)
162
163 def test_recarray_from_repr(self):
164 a = np.array([(1, 'ABC'), (2, "DEF")],
165 dtype=[('foo', int), ('bar', 'S4')])
166 recordarr = np.rec.array(a)
167 recarr = a.view(np.recarray)
168 recordview = a.view(np.dtype((np.record, a.dtype)))
169
170 recordarr_r = eval("np." + repr(recordarr), {'np': np})
171 recarr_r = eval("np." + repr(recarr), {'np': np})
172 # Prints the type `numpy.record` as part of the dtype:
173 recordview_r = eval("np." + repr(recordview), {'np': np, 'numpy': np})
174
175 assert_equal(type(recordarr_r), np.recarray)
176 assert_equal(recordarr_r.dtype.type, np.record)
177 assert_equal(recordarr, recordarr_r)
178
179 assert_equal(type(recarr_r), np.recarray)
180 assert_equal(recarr_r.dtype.type, np.record)
181 assert_equal(recarr, recarr_r)
182
183 assert_equal(type(recordview_r), np.ndarray)
184 assert_equal(recordview.dtype.type, np.record)
185 assert_equal(recordview, recordview_r)
186
187 def test_recarray_views(self):
188 a = np.array([(1, 'ABC'), (2, "DEF")],
189 dtype=[('foo', int), ('bar', 'S4')])
190 b = np.array([1, 2, 3, 4, 5], dtype=np.int64)
191
192 # check that np.rec.array gives right dtypes
193 assert_equal(np.rec.array(a).dtype.type, np.record)
194 assert_equal(type(np.rec.array(a)), np.recarray)
195 assert_equal(np.rec.array(b).dtype.type, np.int64)
196 assert_equal(type(np.rec.array(b)), np.recarray)
197
198 # check that viewing as recarray does the same
199 assert_equal(a.view(np.recarray).dtype.type, np.record)
200 assert_equal(type(a.view(np.recarray)), np.recarray)
201 assert_equal(b.view(np.recarray).dtype.type, np.int64)
202 assert_equal(type(b.view(np.recarray)), np.recarray)
203
204 # check that view to non-structured dtype preserves type=np.recarray
205 r = np.rec.array(np.ones(4, dtype="f4,i4"))
206 rv = r.view('f8').view('f4,i4')
207 assert_equal(type(rv), np.recarray)
208 assert_equal(rv.dtype.type, np.record)
209
210 # check that getitem also preserves np.recarray and np.record
211 r = np.rec.array(np.ones(4, dtype=[('a', 'i4'), ('b', 'i4'),
212 ('c', 'i4,i4')]))
213 assert_equal(r['c'].dtype.type, np.record)
214 assert_equal(type(r['c']), np.recarray)
215
216 # and that it preserves subclasses (gh-6949)
217 class C(np.recarray):
218 pass
219
220 c = r.view(C)
221 assert_equal(type(c['c']), C)
222
223 # check that accessing nested structures keep record type, but
224 # not for subarrays, non-void structures, non-structured voids
225 test_dtype = [('a', 'f4,f4'), ('b', 'V8'), ('c', ('f4', 2)),
226 ('d', ('i8', 'i4,i4'))]
227 r = np.rec.array([((1, 1), b'11111111', [1, 1], 1),
228 ((1, 1), b'11111111', [1, 1], 1)], dtype=test_dtype)
229 assert_equal(r.a.dtype.type, np.record)
230 assert_equal(r.b.dtype.type, np.void)
231 assert_equal(r.c.dtype.type, np.float32)
232 assert_equal(r.d.dtype.type, np.int64)
233 # check the same, but for views
234 r = np.rec.array(np.ones(4, dtype='i4,i4'))
235 assert_equal(r.view('f4,f4').dtype.type, np.record)
236 assert_equal(r.view(('i4', 2)).dtype.type, np.int32)
237 assert_equal(r.view('V8').dtype.type, np.void)
238 assert_equal(r.view(('i8', 'i4,i4')).dtype.type, np.int64)
239
240 # check that we can undo the view
241 arrs = [np.ones(4, dtype='f4,i4'), np.ones(4, dtype='f8')]
242 for arr in arrs:
243 rec = np.rec.array(arr)
244 # recommended way to view as an ndarray:
245 arr2 = rec.view(rec.dtype.fields or rec.dtype, np.ndarray)
246 assert_equal(arr2.dtype.type, arr.dtype.type)
247 assert_equal(type(arr2), type(arr))
248
249 def test_recarray_from_names(self):
250 ra = np.rec.array([
251 (1, 'abc', 3.7000002861022949, 0),
252 (2, 'xy', 6.6999998092651367, 1),
253 (0, ' ', 0.40000000596046448, 0)],
254 names='c1, c2, c3, c4')
255 pa = np.rec.fromrecords([
256 (1, 'abc', 3.7000002861022949, 0),
257 (2, 'xy', 6.6999998092651367, 1),
258 (0, ' ', 0.40000000596046448, 0)],
259 names='c1, c2, c3, c4')
260 assert_(ra.dtype == pa.dtype)
261 assert_(ra.shape == pa.shape)
262 for k in range(len(ra)):
263 assert_(ra[k].item() == pa[k].item())
264
265 def test_recarray_conflict_fields(self):
266 ra = np.rec.array([(1, 'abc', 2.3), (2, 'xyz', 4.2),
267 (3, 'wrs', 1.3)],
268 names='field, shape, mean')
269 ra.mean = [1.1, 2.2, 3.3]
270 assert_array_almost_equal(ra['mean'], [1.1, 2.2, 3.3])
271 assert_(type(ra.mean) is type(ra.var))
272 ra.shape = (1, 3)
273 assert_(ra.shape == (1, 3))
274 ra.shape = ['A', 'B', 'C']
275 assert_array_equal(ra['shape'], [['A', 'B', 'C']])
276 ra.field = 5
277 assert_array_equal(ra['field'], [[5, 5, 5]])
278 assert_(isinstance(ra.field, collections.abc.Callable))
279
280 def test_fromrecords_with_explicit_dtype(self):
281 a = np.rec.fromrecords([(1, 'a'), (2, 'bbb')],
282 dtype=[('a', int), ('b', object)])
283 assert_equal(a.a, [1, 2])
284 assert_equal(a[0].a, 1)
285 assert_equal(a.b, ['a', 'bbb'])
286 assert_equal(a[-1].b, 'bbb')
287 #
288 ndtype = np.dtype([('a', int), ('b', object)])
289 a = np.rec.fromrecords([(1, 'a'), (2, 'bbb')], dtype=ndtype)
290 assert_equal(a.a, [1, 2])
291 assert_equal(a[0].a, 1)
292 assert_equal(a.b, ['a', 'bbb'])
293 assert_equal(a[-1].b, 'bbb')
294
295 def test_recarray_stringtypes(self):
296 # Issue #3993
297 a = np.array([('abc ', 1), ('abc', 2)],
298 dtype=[('foo', 'S4'), ('bar', int)])
299 a = a.view(np.recarray)
300 assert_equal(a.foo[0] == a.foo[1], False)
301
302 def test_recarray_returntypes(self):
303 qux_fields = {'C': (np.dtype('S5'), 0), 'D': (np.dtype('S5'), 6)}
304 a = np.rec.array([('abc ', (1, 1), 1, ('abcde', 'fgehi')),
305 ('abc', (2, 3), 1, ('abcde', 'jklmn'))],
306 dtype=[('foo', 'S4'),
307 ('bar', [('A', int), ('B', int)]),
308 ('baz', int), ('qux', qux_fields)])
309 assert_equal(type(a.foo), np.ndarray)
310 assert_equal(type(a['foo']), np.ndarray)
311 assert_equal(type(a.bar), np.recarray)
312 assert_equal(type(a['bar']), np.recarray)
313 assert_equal(a.bar.dtype.type, np.record)
314 assert_equal(type(a['qux']), np.recarray)
315 assert_equal(a.qux.dtype.type, np.record)
316 assert_equal(dict(a.qux.dtype.fields), qux_fields)
317 assert_equal(type(a.baz), np.ndarray)
318 assert_equal(type(a['baz']), np.ndarray)
319 assert_equal(type(a[0].bar), np.record)
320 assert_equal(type(a[0]['bar']), np.record)
321 assert_equal(a[0].bar.A, 1)
322 assert_equal(a[0].bar['A'], 1)
323 assert_equal(a[0]['bar'].A, 1)
324 assert_equal(a[0]['bar']['A'], 1)
325 assert_equal(a[0].qux.D, b'fgehi')
326 assert_equal(a[0].qux['D'], b'fgehi')
327 assert_equal(a[0]['qux'].D, b'fgehi')
328 assert_equal(a[0]['qux']['D'], b'fgehi')
329
330 def test_zero_width_strings(self):
331 # Test for #6430, based on the test case from #1901
332
333 cols = [['test'] * 3, [''] * 3]
334 rec = np.rec.fromarrays(cols)
335 assert_equal(rec['f0'], ['test', 'test', 'test'])
336 assert_equal(rec['f1'], ['', '', ''])
337
338 dt = np.dtype([('f0', '|S4'), ('f1', '|S')])
339 rec = np.rec.fromarrays(cols, dtype=dt)
340 assert_equal(rec.itemsize, 4)
341 assert_equal(rec['f0'], [b'test', b'test', b'test'])
342 assert_equal(rec['f1'], [b'', b'', b''])
343
344
345class TestPathUsage:
346 # Test that pathlib.Path can be used
347 def test_tofile_fromfile(self):
348 with temppath(suffix='.bin') as path:
349 path = Path(path)
350 np.random.seed(123)
351 a = np.random.rand(10).astype('f8,i4,S5')
352 a[5] = (0.5, 10, 'abcde')
353 with path.open("wb") as fd:
354 a.tofile(fd)
355 x = np._core.records.fromfile(
356 path, formats='f8,i4,S5', shape=10
357 )
358 assert_array_equal(x, a)
359
360
361class TestRecord:
362 def _create_data(self):
363 return np.rec.fromrecords([(1, 2, 3), (4, 5, 6)],
364 dtype=[("col1", "<i4"),
365 ("col2", "<i4"),
366 ("col3", "<i4")])
367
368 def test_assignment1(self):
369 a = self._create_data()
370 assert_equal(a.col1[0], 1)
371 a[0].col1 = 0
372 assert_equal(a.col1[0], 0)
373
374 def test_assignment2(self):
375 a = self._create_data()
376 assert_equal(a.col1[0], 1)
377 a.col1[0] = 0
378 assert_equal(a.col1[0], 0)
379
380 def test_invalid_assignment(self):
381 a = self._create_data()
382
383 def assign_invalid_column(x):
384 x[0].col5 = 1
385
386 assert_raises(AttributeError, assign_invalid_column, a)
387
388 def test_nonwriteable_setfield(self):
389 # gh-8171
390 r = np.rec.array([(0,), (1,)], dtype=[('f', 'i4')])
391 r.flags.writeable = False
392 with assert_raises(ValueError):
393 r.f = [2, 3]
394 with assert_raises(ValueError):
395 r.setfield([2, 3], *r.dtype.fields['f'])
396
397 def test_out_of_order_fields(self):
398 # names in the same order, padding added to descr
399 x = self._create_data()[['col1', 'col2']]
400 assert_equal(x.dtype.names, ('col1', 'col2'))
401 assert_equal(x.dtype.descr,
402 [('col1', '<i4'), ('col2', '<i4'), ('', '|V4')])
403
404 # names change order to match indexing, as of 1.14 - descr can't
405 # represent that
406 y = self._create_data()[['col2', 'col1']]
407 assert_equal(y.dtype.names, ('col2', 'col1'))
408 assert_raises(ValueError, lambda: y.dtype.descr)
409
410 def test_pickle_1(self):
411 # Issue #1529
412 a = np.array([(1, [])], dtype=[('a', np.int32), ('b', np.int32, 0)])
413 for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
414 assert_equal(a, pickle.loads(pickle.dumps(a, protocol=proto)))
415 assert_equal(a[0], pickle.loads(pickle.dumps(a[0],
416 protocol=proto)))
417
418 def test_pickle_2(self):
419 a = self._create_data()
420 for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
421 assert_equal(a, pickle.loads(pickle.dumps(a, protocol=proto)))
422 assert_equal(a[0], pickle.loads(pickle.dumps(a[0],
423 protocol=proto)))
424
425 def test_pickle_3(self):
426 # Issue #7140
427 a = self._create_data()
428 for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
429 pa = pickle.loads(pickle.dumps(a[0], protocol=proto))
430 assert_(pa.flags.c_contiguous)
431 assert_(pa.flags.f_contiguous)
432 assert_(pa.flags.writeable)
433 assert_(pa.flags.aligned)
434
435 def test_pickle_void(self):
436 # issue gh-13593
437 dt = np.dtype([('obj', 'O'), ('int', 'i')])
438 a = np.empty(1, dtype=dt)
439 data = (bytearray(b'eman'),)
440 a['obj'] = data
441 a['int'] = 42
442 ctor, args = a[0].__reduce__()
443 # check the constructor is what we expect before interpreting the arguments
444 assert ctor is np._core.multiarray.scalar
445 dtype, obj = args
446 # make sure we did not pickle the address
447 assert not isinstance(obj, bytes)
448
449 assert_raises(RuntimeError, ctor, dtype, 13)
450
451 # Test roundtrip:
452 dump = pickle.dumps(a[0])
453 unpickled = pickle.loads(dump)
454 assert a[0] == unpickled
455
456 # Also check the similar (impossible) "object scalar" path:
457 with assert_raises(TypeError):
458 ctor(np.dtype("O"), data)
459
460 def test_objview_record(self):
461 # https://github.com/numpy/numpy/issues/2599
462 dt = np.dtype([('foo', 'i8'), ('bar', 'O')])
463 r = np.zeros((1, 3), dtype=dt).view(np.recarray)
464 r.foo = np.array([1, 2, 3]) # TypeError?
465
466 # https://github.com/numpy/numpy/issues/3256
467 ra = np.recarray(
468 (2,), dtype=[('x', object), ('y', float), ('z', int)]
469 )
470 ra[['x', 'y']] # TypeError?
471
472 def test_record_scalar_setitem(self):
473 # https://github.com/numpy/numpy/issues/3561
474 rec = np.recarray(1, dtype=[('x', float, 5)])
475 rec[0].x = 1
476 assert_equal(rec[0].x, np.ones(5))
477
478 def test_missing_field(self):
479 # https://github.com/numpy/numpy/issues/4806
480 arr = np.zeros((3,), dtype=[('x', int), ('y', int)])
481 assert_raises(KeyError, lambda: arr[['nofield']])
482
483 def test_fromarrays_nested_structured_arrays(self):
484 arrays = [
485 np.arange(10),
486 np.ones(10, dtype=[('a', '<u2'), ('b', '<f4')]),
487 ]
488 arr = np.rec.fromarrays(arrays) # ValueError?
489
490 @pytest.mark.parametrize('nfields', [0, 1, 2])
491 def test_assign_dtype_attribute(self, nfields):
492 dt = np.dtype([('a', np.uint8), ('b', np.uint8), ('c', np.uint8)][:nfields])
493 data = np.zeros(3, dt).view(np.recarray)
494
495 # the original and resulting dtypes differ on whether they are records
496 assert data.dtype.type == np.record
497 assert dt.type != np.record
498
499 # ensure that the dtype remains a record even when assigned
500 data.dtype = dt
501 assert data.dtype.type == np.record
502
503 @pytest.mark.parametrize('nfields', [0, 1, 2])
504 def test_nested_fields_are_records(self, nfields):
505 """ Test that nested structured types are treated as records too """
506 dt = np.dtype([('a', np.uint8), ('b', np.uint8), ('c', np.uint8)][:nfields])
507 dt_outer = np.dtype([('inner', dt)])
508
509 data = np.zeros(3, dt_outer).view(np.recarray)
510 assert isinstance(data, np.recarray)
511 assert isinstance(data['inner'], np.recarray)
512
513 data0 = data[0]
514 assert isinstance(data0, np.record)
515 assert isinstance(data0['inner'], np.record)
516
517 def test_nested_dtype_padding(self):
518 """ test that trailing padding is preserved """
519 # construct a dtype with padding at the end
520 dt = np.dtype([('a', np.uint8), ('b', np.uint8), ('c', np.uint8)])
521 dt_padded_end = dt[['a', 'b']]
522 assert dt_padded_end.itemsize == dt.itemsize
523
524 dt_outer = np.dtype([('inner', dt_padded_end)])
525
526 data = np.zeros(3, dt_outer).view(np.recarray)
527 assert_equal(data['inner'].dtype, dt_padded_end)
528
529 data0 = data[0]
530 assert_equal(data0['inner'].dtype, dt_padded_end)
531
532
533def test_find_duplicate():
534 l1 = [1, 2, 3, 4, 5, 6]
535 assert_(np.rec.find_duplicate(l1) == [])
536
537 l2 = [1, 2, 1, 4, 5, 6]
538 assert_(np.rec.find_duplicate(l2) == [1])
539
540 l3 = [1, 2, 1, 4, 1, 6, 2, 3]
541 assert_(np.rec.find_duplicate(l3) == [1, 2])
542
543 l3 = [2, 2, 1, 4, 1, 6, 2, 3]
544 assert_(np.rec.find_duplicate(l3) == [2, 1])
545