codekingpro/portable-devtools
114k
1"""Tests suite for mrecords.
2
3:author: Pierre Gerard-Marchant
4:contact: pierregm_at_uga_dot_edu
5
6"""
7import pickle
8
9import numpy as np
10import numpy.ma as ma
11from numpy._core.records import (
12 fromarrays as recfromarrays,
13 fromrecords as recfromrecords,
14 recarray,
15)
16from numpy.ma import masked, nomask
17from numpy.ma.mrecords import (
18 MaskedRecords,
19 addfield,
20 fromarrays,
21 fromrecords,
22 fromtextfile,
23 mrecarray,
24)
25from numpy.ma.testutils import assert_, assert_equal, assert_equal_records
26from numpy.testing import temppath
27
28
29class TestMRecords:
30
31 ilist = [1, 2, 3, 4, 5]
32 flist = [1.1, 2.2, 3.3, 4.4, 5.5]
33 slist = [b'one', b'two', b'three', b'four', b'five']
34 ddtype = [('a', int), ('b', float), ('c', '|S8')]
35 mask = [0, 1, 0, 0, 1]
36 base = ma.array(list(zip(ilist, flist, slist)), mask=mask, dtype=ddtype)
37
38 def test_byview(self):
39 # Test creation by view
40 base = self.base
41 mbase = base.view(mrecarray)
42 assert_equal(mbase.recordmask, base.recordmask)
43 assert_equal_records(mbase._mask, base._mask)
44 assert_(isinstance(mbase._data, recarray))
45 assert_equal_records(mbase._data, base._data.view(recarray))
46 for field in ('a', 'b', 'c'):
47 assert_equal(base[field], mbase[field])
48 assert_equal_records(mbase.view(mrecarray), mbase)
49
50 def test_get(self):
51 # Tests fields retrieval
52 base = self.base.copy()
53 mbase = base.view(mrecarray)
54 # As fields..........
55 for field in ('a', 'b', 'c'):
56 assert_equal(getattr(mbase, field), mbase[field])
57 assert_equal(base[field], mbase[field])
58 # as elements .......
59 mbase_first = mbase[0]
60 assert_(isinstance(mbase_first, mrecarray))
61 assert_equal(mbase_first.dtype, mbase.dtype)
62 assert_equal(mbase_first.tolist(), (1, 1.1, b'one'))
63 # Used to be mask, now it's recordmask
64 assert_equal(mbase_first.recordmask, nomask)
65 assert_equal(mbase_first._mask.item(), (False, False, False))
66 assert_equal(mbase_first['a'], mbase['a'][0])
67 mbase_last = mbase[-1]
68 assert_(isinstance(mbase_last, mrecarray))
69 assert_equal(mbase_last.dtype, mbase.dtype)
70 assert_equal(mbase_last.tolist(), (None, None, None))
71 # Used to be mask, now it's recordmask
72 assert_equal(mbase_last.recordmask, True)
73 assert_equal(mbase_last._mask.item(), (True, True, True))
74 assert_equal(mbase_last['a'], mbase['a'][-1])
75 assert_(mbase_last['a'] is masked)
76 # as slice ..........
77 mbase_sl = mbase[:2]
78 assert_(isinstance(mbase_sl, mrecarray))
79 assert_equal(mbase_sl.dtype, mbase.dtype)
80 # Used to be mask, now it's recordmask
81 assert_equal(mbase_sl.recordmask, [0, 1])
82 assert_equal_records(mbase_sl.mask,
83 np.array([(False, False, False),
84 (True, True, True)],
85 dtype=mbase._mask.dtype))
86 assert_equal_records(mbase_sl, base[:2].view(mrecarray))
87 for field in ('a', 'b', 'c'):
88 assert_equal(getattr(mbase_sl, field), base[:2][field])
89
90 def test_set_fields(self):
91 # Tests setting fields.
92 base = self.base.copy()
93 mbase = base.view(mrecarray)
94 mbase = mbase.copy()
95 mbase.fill_value = (999999, 1e20, 'N/A')
96 # Change the data, the mask should be conserved
97 mbase.a._data[:] = 5
98 assert_equal(mbase['a']._data, [5, 5, 5, 5, 5])
99 assert_equal(mbase['a']._mask, [0, 1, 0, 0, 1])
100 # Change the elements, and the mask will follow
101 mbase.a = 1
102 assert_equal(mbase['a']._data, [1] * 5)
103 assert_equal(ma.getmaskarray(mbase['a']), [0] * 5)
104 # Use to be _mask, now it's recordmask
105 assert_equal(mbase.recordmask, [False] * 5)
106 assert_equal(mbase._mask.tolist(),
107 np.array([(0, 0, 0),
108 (0, 1, 1),
109 (0, 0, 0),
110 (0, 0, 0),
111 (0, 1, 1)],
112 dtype=bool))
113 # Set a field to mask ........................
114 mbase.c = masked
115 # Use to be mask, and now it's still mask !
116 assert_equal(mbase.c.mask, [1] * 5)
117 assert_equal(mbase.c.recordmask, [1] * 5)
118 assert_equal(ma.getmaskarray(mbase['c']), [1] * 5)
119 assert_equal(ma.getdata(mbase['c']), [b'N/A'] * 5)
120 assert_equal(mbase._mask.tolist(),
121 np.array([(0, 0, 1),
122 (0, 1, 1),
123 (0, 0, 1),
124 (0, 0, 1),
125 (0, 1, 1)],
126 dtype=bool))
127 # Set fields by slices .......................
128 mbase = base.view(mrecarray).copy()
129 mbase.a[3:] = 5
130 assert_equal(mbase.a, [1, 2, 3, 5, 5])
131 assert_equal(mbase.a._mask, [0, 1, 0, 0, 0])
132 mbase.b[3:] = masked
133 assert_equal(mbase.b, base['b'])
134 assert_equal(mbase.b._mask, [0, 1, 0, 1, 1])
135 # Set fields globally..........................
136 ndtype = [('alpha', '|S1'), ('num', int)]
137 data = ma.array([('a', 1), ('b', 2), ('c', 3)], dtype=ndtype)
138 rdata = data.view(MaskedRecords)
139 val = ma.array([10, 20, 30], mask=[1, 0, 0])
140
141 rdata['num'] = val
142 assert_equal(rdata.num, val)
143 assert_equal(rdata.num.mask, [1, 0, 0])
144
145 def test_set_fields_mask(self):
146 # Tests setting the mask of a field.
147 base = self.base.copy()
148 # This one has already a mask....
149 mbase = base.view(mrecarray)
150 mbase['a'][-2] = masked
151 assert_equal(mbase.a, [1, 2, 3, 4, 5])
152 assert_equal(mbase.a._mask, [0, 1, 0, 1, 1])
153 # This one has not yet
154 mbase = fromarrays([np.arange(5), np.random.rand(5)],
155 dtype=[('a', int), ('b', float)])
156 mbase['a'][-2] = masked
157 assert_equal(mbase.a, [0, 1, 2, 3, 4])
158 assert_equal(mbase.a._mask, [0, 0, 0, 1, 0])
159
160 def test_set_mask(self):
161 base = self.base.copy()
162 mbase = base.view(mrecarray)
163 # Set the mask to True .......................
164 mbase.mask = masked
165 assert_equal(ma.getmaskarray(mbase['b']), [1] * 5)
166 assert_equal(mbase['a']._mask, mbase['b']._mask)
167 assert_equal(mbase['a']._mask, mbase['c']._mask)
168 assert_equal(mbase._mask.tolist(),
169 np.array([(1, 1, 1)] * 5, dtype=bool))
170 # Delete the mask ............................
171 mbase.mask = nomask
172 assert_equal(ma.getmaskarray(mbase['c']), [0] * 5)
173 assert_equal(mbase._mask.tolist(),
174 np.array([(0, 0, 0)] * 5, dtype=bool))
175
176 def test_set_mask_fromarray(self):
177 base = self.base.copy()
178 mbase = base.view(mrecarray)
179 # Sets the mask w/ an array
180 mbase.mask = [1, 0, 0, 0, 1]
181 assert_equal(mbase.a.mask, [1, 0, 0, 0, 1])
182 assert_equal(mbase.b.mask, [1, 0, 0, 0, 1])
183 assert_equal(mbase.c.mask, [1, 0, 0, 0, 1])
184 # Yay, once more !
185 mbase.mask = [0, 0, 0, 0, 1]
186 assert_equal(mbase.a.mask, [0, 0, 0, 0, 1])
187 assert_equal(mbase.b.mask, [0, 0, 0, 0, 1])
188 assert_equal(mbase.c.mask, [0, 0, 0, 0, 1])
189
190 def test_set_mask_fromfields(self):
191 mbase = self.base.copy().view(mrecarray)
192
193 nmask = np.array(
194 [(0, 1, 0), (0, 1, 0), (1, 0, 1), (1, 0, 1), (0, 0, 0)],
195 dtype=[('a', bool), ('b', bool), ('c', bool)])
196 mbase.mask = nmask
197 assert_equal(mbase.a.mask, [0, 0, 1, 1, 0])
198 assert_equal(mbase.b.mask, [1, 1, 0, 0, 0])
199 assert_equal(mbase.c.mask, [0, 0, 1, 1, 0])
200 # Reinitialize and redo
201 mbase.mask = False
202 mbase.fieldmask = nmask
203 assert_equal(mbase.a.mask, [0, 0, 1, 1, 0])
204 assert_equal(mbase.b.mask, [1, 1, 0, 0, 0])
205 assert_equal(mbase.c.mask, [0, 0, 1, 1, 0])
206
207 def test_set_elements(self):
208 base = self.base.copy()
209 # Set an element to mask .....................
210 mbase = base.view(mrecarray).copy()
211 mbase[-2] = masked
212 assert_equal(
213 mbase._mask.tolist(),
214 np.array([(0, 0, 0), (1, 1, 1), (0, 0, 0), (1, 1, 1), (1, 1, 1)],
215 dtype=bool))
216 # Used to be mask, now it's recordmask!
217 assert_equal(mbase.recordmask, [0, 1, 0, 1, 1])
218 # Set slices .................................
219 mbase = base.view(mrecarray).copy()
220 mbase[:2] = (5, 5, 5)
221 assert_equal(mbase.a._data, [5, 5, 3, 4, 5])
222 assert_equal(mbase.a._mask, [0, 0, 0, 0, 1])
223 assert_equal(mbase.b._data, [5., 5., 3.3, 4.4, 5.5])
224 assert_equal(mbase.b._mask, [0, 0, 0, 0, 1])
225 assert_equal(mbase.c._data,
226 [b'5', b'5', b'three', b'four', b'five'])
227 assert_equal(mbase.b._mask, [0, 0, 0, 0, 1])
228
229 mbase = base.view(mrecarray).copy()
230 mbase[:2] = masked
231 assert_equal(mbase.a._data, [1, 2, 3, 4, 5])
232 assert_equal(mbase.a._mask, [1, 1, 0, 0, 1])
233 assert_equal(mbase.b._data, [1.1, 2.2, 3.3, 4.4, 5.5])
234 assert_equal(mbase.b._mask, [1, 1, 0, 0, 1])
235 assert_equal(mbase.c._data,
236 [b'one', b'two', b'three', b'four', b'five'])
237 assert_equal(mbase.b._mask, [1, 1, 0, 0, 1])
238
239 def test_setslices_hardmask(self):
240 # Tests setting slices w/ hardmask.
241 base = self.base.copy()
242 mbase = base.view(mrecarray)
243 mbase.harden_mask()
244 try:
245 mbase[-2:] = (5, 5, 5)
246 assert_equal(mbase.a._data, [1, 2, 3, 5, 5])
247 assert_equal(mbase.b._data, [1.1, 2.2, 3.3, 5, 5.5])
248 assert_equal(mbase.c._data,
249 [b'one', b'two', b'three', b'5', b'five'])
250 assert_equal(mbase.a._mask, [0, 1, 0, 0, 1])
251 assert_equal(mbase.b._mask, mbase.a._mask)
252 assert_equal(mbase.b._mask, mbase.c._mask)
253 except NotImplementedError:
254 # OK, not implemented yet...
255 pass
256 except AssertionError:
257 raise
258 else:
259 raise Exception("Flexible hard masks should be supported !")
260 # Not using a tuple should crash
261 try:
262 mbase[-2:] = 3
263 except (NotImplementedError, TypeError):
264 pass
265 else:
266 raise TypeError("Should have expected a readable buffer object!")
267
268 def test_hardmask(self):
269 # Test hardmask
270 base = self.base.copy()
271 mbase = base.view(mrecarray)
272 mbase.harden_mask()
273 assert_(mbase._hardmask)
274 mbase.mask = nomask
275 assert_equal_records(mbase._mask, base._mask)
276 mbase.soften_mask()
277 assert_(not mbase._hardmask)
278 mbase.mask = nomask
279 # So, the mask of a field is no longer set to nomask...
280 assert_equal_records(mbase._mask,
281 ma.make_mask_none(base.shape, base.dtype))
282 assert_(ma.make_mask(mbase['b']._mask) is nomask)
283 assert_equal(mbase['a']._mask, mbase['b']._mask)
284
285 def test_pickling(self):
286 # Test pickling
287 base = self.base.copy()
288 mrec = base.view(mrecarray)
289 for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
290 _ = pickle.dumps(mrec, protocol=proto)
291 mrec_ = pickle.loads(_)
292 assert_equal(mrec_.dtype, mrec.dtype)
293 assert_equal_records(mrec_._data, mrec._data)
294 assert_equal(mrec_._mask, mrec._mask)
295 assert_equal_records(mrec_._mask, mrec._mask)
296
297 def test_filled(self):
298 # Test filling the array
299 _a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
300 _b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
301 _c = ma.array(['one', 'two', 'three'], mask=[0, 0, 1], dtype='|S8')
302 ddtype = [('a', int), ('b', float), ('c', '|S8')]
303 mrec = fromarrays([_a, _b, _c], dtype=ddtype,
304 fill_value=(99999, 99999., 'N/A'))
305 mrecfilled = mrec.filled()
306 assert_equal(mrecfilled['a'], np.array((1, 2, 99999), dtype=int))
307 assert_equal(mrecfilled['b'], np.array((1.1, 2.2, 99999.),
308 dtype=float))
309 assert_equal(mrecfilled['c'], np.array(('one', 'two', 'N/A'),
310 dtype='|S8'))
311
312 def test_tolist(self):
313 # Test tolist.
314 _a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
315 _b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
316 _c = ma.array(['one', 'two', 'three'], mask=[1, 0, 0], dtype='|S8')
317 ddtype = [('a', int), ('b', float), ('c', '|S8')]
318 mrec = fromarrays([_a, _b, _c], dtype=ddtype,
319 fill_value=(99999, 99999., 'N/A'))
320
321 assert_equal(mrec.tolist(),
322 [(1, 1.1, None), (2, 2.2, b'two'),
323 (None, None, b'three')])
324
325 def test_withnames(self):
326 # Test the creation w/ format and names
327 x = mrecarray(1, formats=float, names='base')
328 x[0]['base'] = 10
329 assert_equal(x['base'][0], 10)
330
331 def test_exotic_formats(self):
332 # Test that 'exotic' formats are processed properly
333 easy = mrecarray(1, dtype=[('i', int), ('s', '|S8'), ('f', float)])
334 easy[0] = masked
335 assert_equal(easy.filled(1).item(), (1, b'1', 1.))
336
337 solo = mrecarray(1, dtype=[('f0', '<f8', (2, 2))])
338 solo[0] = masked
339 assert_equal(solo.filled(1).item(),
340 np.array((1,), dtype=solo.dtype).item())
341
342 mult = mrecarray(2, dtype="i4, (2,3)float, float")
343 mult[0] = masked
344 mult[1] = (1, 1, 1)
345 mult.filled(0)
346 assert_equal_records(mult.filled(0),
347 np.array([(0, 0, 0), (1, 1, 1)],
348 dtype=mult.dtype))
349
350
351class TestView:
352
353 def _create_data(self):
354 a, b = (np.arange(10), np.random.rand(10))
355 ndtype = [('a', float), ('b', float)]
356 arr = np.array(list(zip(a, b)), dtype=ndtype)
357
358 mrec = fromarrays([a, b], dtype=ndtype, fill_value=(-9., -99.))
359 mrec.mask[3] = (False, True)
360 return mrec, a, b, arr
361
362 def test_view_by_itself(self):
363 mrec = self._create_data()[0]
364 test = mrec.view()
365 assert_(isinstance(test, MaskedRecords))
366 assert_equal_records(test, mrec)
367 assert_equal_records(test._mask, mrec._mask)
368
369 def test_view_simple_dtype(self):
370 mrec, a, b, _ = self._create_data()
371 ntype = (float, 2)
372 test = mrec.view(ntype)
373 assert_(isinstance(test, ma.MaskedArray))
374 assert_equal(test, np.array(list(zip(a, b)), dtype=float))
375 assert_(test[3, 1] is ma.masked)
376
377 def test_view_flexible_type(self):
378 mrec, _, _, arr = self._create_data()
379 alttype = [('A', float), ('B', float)]
380 test = mrec.view(alttype)
381 assert_(isinstance(test, MaskedRecords))
382 assert_equal_records(test, arr.view(alttype))
383 assert_(test['B'][3] is masked)
384 assert_equal(test.dtype, np.dtype(alttype))
385 assert_(test._fill_value is None)
386
387
388##############################################################################
389class TestMRecordsImport:
390
391 _a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
392 _b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
393 _c = ma.array([b'one', b'two', b'three'],
394 mask=[0, 0, 1], dtype='|S8')
395 ddtype = [('a', int), ('b', float), ('c', '|S8')]
396 mrec = fromarrays([_a, _b, _c], dtype=ddtype,
397 fill_value=(b'99999', b'99999.',
398 b'N/A'))
399 nrec = recfromarrays((_a._data, _b._data, _c._data), dtype=ddtype)
400 data = (mrec, nrec, ddtype)
401
402 def test_fromarrays(self):
403 _a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
404 _b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
405 _c = ma.array(['one', 'two', 'three'], mask=[0, 0, 1], dtype='|S8')
406 (mrec, nrec, _) = self.data
407 for (f, l) in zip(('a', 'b', 'c'), (_a, _b, _c)):
408 assert_equal(getattr(mrec, f)._mask, l._mask)
409 # One record only
410 _x = ma.array([1, 1.1, 'one'], mask=[1, 0, 0], dtype=object)
411 assert_equal_records(fromarrays(_x, dtype=mrec.dtype), mrec[0])
412
413 def test_fromrecords(self):
414 # Test construction from records.
415 (mrec, nrec, ddtype) = self.data
416 # ......
417 palist = [(1, 'abc', 3.7000002861022949, 0),
418 (2, 'xy', 6.6999998092651367, 1),
419 (0, ' ', 0.40000000596046448, 0)]
420 pa = recfromrecords(palist, names='c1, c2, c3, c4')
421 mpa = fromrecords(palist, names='c1, c2, c3, c4')
422 assert_equal_records(pa, mpa)
423 # .....
424 _mrec = fromrecords(nrec)
425 assert_equal(_mrec.dtype, mrec.dtype)
426 for field in _mrec.dtype.names:
427 assert_equal(getattr(_mrec, field), getattr(mrec._data, field))
428
429 _mrec = fromrecords(nrec.tolist(), names='c1,c2,c3')
430 assert_equal(_mrec.dtype, [('c1', int), ('c2', float), ('c3', '|S5')])
431 for (f, n) in zip(('c1', 'c2', 'c3'), ('a', 'b', 'c')):
432 assert_equal(getattr(_mrec, f), getattr(mrec._data, n))
433
434 _mrec = fromrecords(mrec)
435 assert_equal(_mrec.dtype, mrec.dtype)
436 assert_equal_records(_mrec._data, mrec.filled())
437 assert_equal_records(_mrec._mask, mrec._mask)
438
439 def test_fromrecords_wmask(self):
440 # Tests construction from records w/ mask.
441 (mrec, nrec, ddtype) = self.data
442
443 _mrec = fromrecords(nrec.tolist(), dtype=ddtype, mask=[0, 1, 0,])
444 assert_equal_records(_mrec._data, mrec._data)
445 assert_equal(_mrec._mask.tolist(), [(0, 0, 0), (1, 1, 1), (0, 0, 0)])
446
447 _mrec = fromrecords(nrec.tolist(), dtype=ddtype, mask=True)
448 assert_equal_records(_mrec._data, mrec._data)
449 assert_equal(_mrec._mask.tolist(), [(1, 1, 1), (1, 1, 1), (1, 1, 1)])
450
451 _mrec = fromrecords(nrec.tolist(), dtype=ddtype, mask=mrec._mask)
452 assert_equal_records(_mrec._data, mrec._data)
453 assert_equal(_mrec._mask.tolist(), mrec._mask.tolist())
454
455 _mrec = fromrecords(nrec.tolist(), dtype=ddtype,
456 mask=mrec._mask.tolist())
457 assert_equal_records(_mrec._data, mrec._data)
458 assert_equal(_mrec._mask.tolist(), mrec._mask.tolist())
459
460 def test_fromtextfile(self):
461 # Tests reading from a text file.
462 fcontent = (
463"""#
464'One (S)','Two (I)','Three (F)','Four (M)','Five (-)','Six (C)'
465'strings',1,1.0,'mixed column',,1
466'with embedded "double quotes"',2,2.0,1.0,,1
467'strings',3,3.0E5,3,,1
468'strings',4,-1e-10,,,1
469""")
470 with temppath() as path:
471 with open(path, 'w') as f:
472 f.write(fcontent)
473 mrectxt = fromtextfile(path, delimiter=',', varnames='ABCDEFG')
474 assert_(isinstance(mrectxt, MaskedRecords))
475 assert_equal(mrectxt.F, [1, 1, 1, 1])
476 assert_equal(mrectxt.E._mask, [1, 1, 1, 1])
477 assert_equal(mrectxt.C, [1, 2, 3.e+5, -1e-10])
478
479 def test_addfield(self):
480 # Tests addfield
481 (mrec, nrec, ddtype) = self.data
482 (d, m) = ([100, 200, 300], [1, 0, 0])
483 mrec = addfield(mrec, ma.array(d, mask=m))
484 assert_equal(mrec.f3, d)
485 assert_equal(mrec.f3._mask, m)
486
487
488def test_record_array_with_object_field():
489 # Trac #1839
490 y = ma.masked_array(
491 [(1, '2'), (3, '4')],
492 mask=[(0, 0), (0, 1)],
493 dtype=[('a', int), ('b', object)])
494 # getting an item used to fail
495 y[1]
496