codekingpro/portable-devtools

1
2import numpy as np
3from numpy.testing import assert_, assert_array_equal, assert_equal
4
5
6def buffer_length(arr):
7    if isinstance(arr, str):
8        if not arr:
9            charmax = 0
10        else:
11            charmax = max(ord(c) for c in arr)
12        if charmax < 256:
13            size = 1
14        elif charmax < 65536:
15            size = 2
16        else:
17            size = 4
18        return size * len(arr)
19    v = memoryview(arr)
20    if v.shape is None:
21        return len(v) * v.itemsize
22    else:
23        return np.prod(v.shape) * v.itemsize
24
25
26# In both cases below we need to make sure that the byte swapped value (as
27# UCS4) is still a valid unicode:
28# Value that can be represented in UCS2 interpreters
29ucs2_value = '\u0900'
30# Value that cannot be represented in UCS2 interpreters (but can in UCS4)
31ucs4_value = '\U00100900'
32
33
34def test_string_cast():
35    str_arr = np.array(["1234", "1234\0\0"], dtype='S')
36    uni_arr1 = str_arr.astype('>U')
37    uni_arr2 = str_arr.astype('<U')
38
39    assert_array_equal(str_arr != uni_arr1, np.ones(2, dtype=bool))
40    assert_array_equal(uni_arr1 != str_arr, np.ones(2, dtype=bool))
41    assert_array_equal(str_arr == uni_arr1, np.zeros(2, dtype=bool))
42    assert_array_equal(uni_arr1 == str_arr, np.zeros(2, dtype=bool))
43
44    assert_array_equal(uni_arr1, uni_arr2)
45
46
47############################################################
48#    Creation tests
49############################################################
50
51class CreateZeros:
52    """Check the creation of zero-valued arrays"""
53
54    def content_check(self, ua, ua_scalar, nbytes):
55
56        # Check the length of the unicode base type
57        assert_(int(ua.dtype.str[2:]) == self.ulen)
58        # Check the length of the data buffer
59        assert_(buffer_length(ua) == nbytes)
60        # Small check that data in array element is ok
61        assert_(ua_scalar == '')
62        # Encode to ascii and double check
63        assert_(ua_scalar.encode('ascii') == b'')
64        # Check buffer lengths for scalars
65        assert_(buffer_length(ua_scalar) == 0)
66
67    def test_zeros0D(self):
68        # Check creation of 0-dimensional objects
69        ua = np.zeros((), dtype=f'U{self.ulen}')
70        self.content_check(ua, ua[()], 4 * self.ulen)
71
72    def test_zerosSD(self):
73        # Check creation of single-dimensional objects
74        ua = np.zeros((2,), dtype=f'U{self.ulen}')
75        self.content_check(ua, ua[0], 4 * self.ulen * 2)
76        self.content_check(ua, ua[1], 4 * self.ulen * 2)
77
78    def test_zerosMD(self):
79        # Check creation of multi-dimensional objects
80        ua = np.zeros((2, 3, 4), dtype=f'U{self.ulen}')
81        self.content_check(ua, ua[0, 0, 0], 4 * self.ulen * 2 * 3 * 4)
82        self.content_check(ua, ua[-1, -1, -1], 4 * self.ulen * 2 * 3 * 4)
83
84
85class TestCreateZeros_1(CreateZeros):
86    """Check the creation of zero-valued arrays (size 1)"""
87    ulen = 1
88
89
90class TestCreateZeros_2(CreateZeros):
91    """Check the creation of zero-valued arrays (size 2)"""
92    ulen = 2
93
94
95class TestCreateZeros_1009(CreateZeros):
96    """Check the creation of zero-valued arrays (size 1009)"""
97    ulen = 1009
98
99
100class CreateValues:
101    """Check the creation of unicode arrays with values"""
102
103    def content_check(self, ua, ua_scalar, nbytes):
104
105        # Check the length of the unicode base type
106        assert_(int(ua.dtype.str[2:]) == self.ulen)
107        # Check the length of the data buffer
108        assert_(buffer_length(ua) == nbytes)
109        # Small check that data in array element is ok
110        assert_(ua_scalar == self.ucs_value * self.ulen)
111        # Encode to UTF-8 and double check
112        assert_(ua_scalar.encode('utf-8') ==
113                        (self.ucs_value * self.ulen).encode('utf-8'))
114        # Check buffer lengths for scalars
115        if self.ucs_value == ucs4_value:
116            # In UCS2, the \U0010FFFF will be represented using a
117            # surrogate *pair*
118            assert_(buffer_length(ua_scalar) == 2 * 2 * self.ulen)
119        else:
120            # In UCS2, the \uFFFF will be represented using a
121            # regular 2-byte word
122            assert_(buffer_length(ua_scalar) == 2 * self.ulen)
123
124    def test_values0D(self):
125        # Check creation of 0-dimensional objects with values
126        ua = np.array(self.ucs_value * self.ulen, dtype=f'U{self.ulen}')
127        self.content_check(ua, ua[()], 4 * self.ulen)
128
129    def test_valuesSD(self):
130        # Check creation of single-dimensional objects with values
131        ua = np.array([self.ucs_value * self.ulen] * 2, dtype=f'U{self.ulen}')
132        self.content_check(ua, ua[0], 4 * self.ulen * 2)
133        self.content_check(ua, ua[1], 4 * self.ulen * 2)
134
135    def test_valuesMD(self):
136        # Check creation of multi-dimensional objects with values
137        data = [[[self.ucs_value * self.ulen] * 2] * 3] * 4
138        ua = np.array(data, dtype=f'U{self.ulen}')
139        self.content_check(ua, ua[0, 0, 0], 4 * self.ulen * 2 * 3 * 4)
140        self.content_check(ua, ua[-1, -1, -1], 4 * self.ulen * 2 * 3 * 4)
141
142
143class TestCreateValues_1_UCS2(CreateValues):
144    """Check the creation of valued arrays (size 1, UCS2 values)"""
145    ulen = 1
146    ucs_value = ucs2_value
147
148
149class TestCreateValues_1_UCS4(CreateValues):
150    """Check the creation of valued arrays (size 1, UCS4 values)"""
151    ulen = 1
152    ucs_value = ucs4_value
153
154
155class TestCreateValues_2_UCS2(CreateValues):
156    """Check the creation of valued arrays (size 2, UCS2 values)"""
157    ulen = 2
158    ucs_value = ucs2_value
159
160
161class TestCreateValues_2_UCS4(CreateValues):
162    """Check the creation of valued arrays (size 2, UCS4 values)"""
163    ulen = 2
164    ucs_value = ucs4_value
165
166
167class TestCreateValues_1009_UCS2(CreateValues):
168    """Check the creation of valued arrays (size 1009, UCS2 values)"""
169    ulen = 1009
170    ucs_value = ucs2_value
171
172
173class TestCreateValues_1009_UCS4(CreateValues):
174    """Check the creation of valued arrays (size 1009, UCS4 values)"""
175    ulen = 1009
176    ucs_value = ucs4_value
177
178
179############################################################
180#    Assignment tests
181############################################################
182
183class AssignValues:
184    """Check the assignment of unicode arrays with values"""
185
186    def content_check(self, ua, ua_scalar, nbytes):
187
188        # Check the length of the unicode base type
189        assert_(int(ua.dtype.str[2:]) == self.ulen)
190        # Check the length of the data buffer
191        assert_(buffer_length(ua) == nbytes)
192        # Small check that data in array element is ok
193        assert_(ua_scalar == self.ucs_value * self.ulen)
194        # Encode to UTF-8 and double check
195        assert_(ua_scalar.encode('utf-8') ==
196                        (self.ucs_value * self.ulen).encode('utf-8'))
197        # Check buffer lengths for scalars
198        if self.ucs_value == ucs4_value:
199            # In UCS2, the \U0010FFFF will be represented using a
200            # surrogate *pair*
201            assert_(buffer_length(ua_scalar) == 2 * 2 * self.ulen)
202        else:
203            # In UCS2, the \uFFFF will be represented using a
204            # regular 2-byte word
205            assert_(buffer_length(ua_scalar) == 2 * self.ulen)
206
207    def test_values0D(self):
208        # Check assignment of 0-dimensional objects with values
209        ua = np.zeros((), dtype=f'U{self.ulen}')
210        ua[()] = self.ucs_value * self.ulen
211        self.content_check(ua, ua[()], 4 * self.ulen)
212
213    def test_valuesSD(self):
214        # Check assignment of single-dimensional objects with values
215        ua = np.zeros((2,), dtype=f'U{self.ulen}')
216        ua[0] = self.ucs_value * self.ulen
217        self.content_check(ua, ua[0], 4 * self.ulen * 2)
218        ua[1] = self.ucs_value * self.ulen
219        self.content_check(ua, ua[1], 4 * self.ulen * 2)
220
221    def test_valuesMD(self):
222        # Check assignment of multi-dimensional objects with values
223        ua = np.zeros((2, 3, 4), dtype=f'U{self.ulen}')
224        ua[0, 0, 0] = self.ucs_value * self.ulen
225        self.content_check(ua, ua[0, 0, 0], 4 * self.ulen * 2 * 3 * 4)
226        ua[-1, -1, -1] = self.ucs_value * self.ulen
227        self.content_check(ua, ua[-1, -1, -1], 4 * self.ulen * 2 * 3 * 4)
228
229
230class TestAssignValues_1_UCS2(AssignValues):
231    """Check the assignment of valued arrays (size 1, UCS2 values)"""
232    ulen = 1
233    ucs_value = ucs2_value
234
235
236class TestAssignValues_1_UCS4(AssignValues):
237    """Check the assignment of valued arrays (size 1, UCS4 values)"""
238    ulen = 1
239    ucs_value = ucs4_value
240
241
242class TestAssignValues_2_UCS2(AssignValues):
243    """Check the assignment of valued arrays (size 2, UCS2 values)"""
244    ulen = 2
245    ucs_value = ucs2_value
246
247
248class TestAssignValues_2_UCS4(AssignValues):
249    """Check the assignment of valued arrays (size 2, UCS4 values)"""
250    ulen = 2
251    ucs_value = ucs4_value
252
253
254class TestAssignValues_1009_UCS2(AssignValues):
255    """Check the assignment of valued arrays (size 1009, UCS2 values)"""
256    ulen = 1009
257    ucs_value = ucs2_value
258
259
260class TestAssignValues_1009_UCS4(AssignValues):
261    """Check the assignment of valued arrays (size 1009, UCS4 values)"""
262    ulen = 1009
263    ucs_value = ucs4_value
264
265
266############################################################
267#    Byteorder tests
268############################################################
269
270class ByteorderValues:
271    """Check the byteorder of unicode arrays in round-trip conversions"""
272
273    def test_values0D(self):
274        # Check byteorder of 0-dimensional objects
275        ua = np.array(self.ucs_value * self.ulen, dtype=f'U{self.ulen}')
276        ua2 = ua.view(ua.dtype.newbyteorder())
277        # This changes the interpretation of the data region (but not the
278        #  actual data), therefore the returned scalars are not
279        #  the same (they are byte-swapped versions of each other).
280        assert_(ua[()] != ua2[()])
281        ua3 = ua2.view(ua2.dtype.newbyteorder())
282        # Arrays must be equal after the round-trip
283        assert_equal(ua, ua3)
284
285    def test_valuesSD(self):
286        # Check byteorder of single-dimensional objects
287        ua = np.array([self.ucs_value * self.ulen] * 2, dtype=f'U{self.ulen}')
288        ua2 = ua.view(ua.dtype.newbyteorder())
289        assert_((ua != ua2).all())
290        assert_(ua[-1] != ua2[-1])
291        ua3 = ua2.view(ua2.dtype.newbyteorder())
292        # Arrays must be equal after the round-trip
293        assert_equal(ua, ua3)
294
295    def test_valuesMD(self):
296        # Check byteorder of multi-dimensional objects
297        ua = np.array([[[self.ucs_value * self.ulen] * 2] * 3] * 4,
298                      dtype=f'U{self.ulen}')
299        ua2 = ua.view(ua.dtype.newbyteorder())
300        assert_((ua != ua2).all())
301        assert_(ua[-1, -1, -1] != ua2[-1, -1, -1])
302        ua3 = ua2.view(ua2.dtype.newbyteorder())
303        # Arrays must be equal after the round-trip
304        assert_equal(ua, ua3)
305
306    def test_values_cast(self):
307        # Check byteorder of when casting the array for a strided and
308        # contiguous array:
309        test1 = np.array([self.ucs_value * self.ulen] * 2, dtype=f'U{self.ulen}')
310        test2 = np.repeat(test1, 2)[::2]
311        for ua in (test1, test2):
312            ua2 = ua.astype(dtype=ua.dtype.newbyteorder())
313            assert_((ua == ua2).all())
314            assert_(ua[-1] == ua2[-1])
315            ua3 = ua2.astype(dtype=ua.dtype)
316            # Arrays must be equal after the round-trip
317            assert_equal(ua, ua3)
318
319    def test_values_updowncast(self):
320        # Check byteorder of when casting the array to a longer and shorter
321        # string length for strided and contiguous arrays
322        test1 = np.array([self.ucs_value * self.ulen] * 2, dtype=f'U{self.ulen}')
323        test2 = np.repeat(test1, 2)[::2]
324        for ua in (test1, test2):
325            # Cast to a longer type with zero padding
326            longer_type = np.dtype(f'U{self.ulen + 1}').newbyteorder()
327            ua2 = ua.astype(dtype=longer_type)
328            assert_((ua == ua2).all())
329            assert_(ua[-1] == ua2[-1])
330            # Cast back again with truncating:
331            ua3 = ua2.astype(dtype=ua.dtype)
332            # Arrays must be equal after the round-trip
333            assert_equal(ua, ua3)
334
335
336class TestByteorder_1_UCS2(ByteorderValues):
337    """Check the byteorder in unicode (size 1, UCS2 values)"""
338    ulen = 1
339    ucs_value = ucs2_value
340
341
342class TestByteorder_1_UCS4(ByteorderValues):
343    """Check the byteorder in unicode (size 1, UCS4 values)"""
344    ulen = 1
345    ucs_value = ucs4_value
346
347
348class TestByteorder_2_UCS2(ByteorderValues):
349    """Check the byteorder in unicode (size 2, UCS2 values)"""
350    ulen = 2
351    ucs_value = ucs2_value
352
353
354class TestByteorder_2_UCS4(ByteorderValues):
355    """Check the byteorder in unicode (size 2, UCS4 values)"""
356    ulen = 2
357    ucs_value = ucs4_value
358
359
360class TestByteorder_1009_UCS2(ByteorderValues):
361    """Check the byteorder in unicode (size 1009, UCS2 values)"""
362    ulen = 1009
363    ucs_value = ucs2_value
364
365
366class TestByteorder_1009_UCS4(ByteorderValues):
367    """Check the byteorder in unicode (size 1009, UCS4 values)"""
368    ulen = 1009
369    ucs_value = ucs4_value
370