Python numpy.unicode_() Examples

def test_join(self):
        if sys.version_info[0] >= 3:
            # NOTE: list(b'123') == [49, 50, 51]
            #       so that b','.join(b'123') results to an error on Py3
            A0 = self.A.decode('ascii')
        else:
            A0 = self.A

        A = np.char.join([',', '#'], A0)
        if sys.version_info[0] >= 3:
            assert_(issubclass(A.dtype.type, np.unicode_))
        else:
            assert_(issubclass(A.dtype.type, np.string_))
        tgt = np.array([[' ,a,b,c, ', ''],
                        ['1,2,3,4,5', 'M#i#x#e#d#C#a#s#e'],
                        ['1,2,3, ,\t, ,3,4,5, ,\x00, ', 'U#P#P#E#R']])
        assert_array_equal(np.char.join([',', '#'], A0), tgt) 

def test_rstrip(self):
        assert_(issubclass(self.A.rstrip().dtype.type, np.string_))

        tgt = asbytes_nested([[' abc', ''],
                              ['12345', 'MixedCase'],
                              ['123 \t 345', 'UPPER']])
        assert_array_equal(self.A.rstrip(), tgt)

        tgt = asbytes_nested([[' abc ', ''],
                              ['1234', 'MixedCase'],
                              ['123 \t 345 \x00', 'UPP']
                              ])
        assert_array_equal(self.A.rstrip(asbytes_nested(['5', 'ER'])), tgt)

        tgt = [[sixu(' \u03a3'), ''],
               ['12345', 'MixedCase'],
               ['123 \t 345', 'UPPER']]
        assert_(issubclass(self.B.rstrip().dtype.type, np.unicode_))
        assert_array_equal(self.B.rstrip(), tgt) 

def test_lstrip(self):
        tgt = asbytes_nested([['abc ', ''],
                              ['12345', 'MixedCase'],
                              ['123 \t 345 \0 ', 'UPPER']])
        assert_(issubclass(self.A.lstrip().dtype.type, np.string_))
        assert_array_equal(self.A.lstrip(), tgt)

        tgt = asbytes_nested([[' abc', ''],
                              ['2345', 'ixedCase'],
                              ['23 \t 345 \x00', 'UPPER']])
        assert_array_equal(self.A.lstrip(asbytes_nested(['1', 'M'])), tgt)

        tgt = [[sixu('\u03a3 '), ''],
               ['12345', 'MixedCase'],
               ['123 \t 345 \0 ', 'UPPER']]
        assert_(issubclass(self.B.lstrip().dtype.type, np.unicode_))
        assert_array_equal(self.B.lstrip(), tgt) 

def test_replace(self):
        R = self.A.replace(asbytes_nested(['3', 'a']),
                           asbytes_nested(['##########', '@']))
        tgt = asbytes_nested([[' abc ', ''],
                              ['12##########45', 'MixedC@se'],
                              ['12########## \t ##########45 \x00', 'UPPER']])
        assert_(issubclass(R.dtype.type, np.string_))
        assert_array_equal(R, tgt)

        if sys.version_info[0] < 3:
            # NOTE: b'abc'.replace(b'a', 'b') is not allowed on Py3
            R = self.A.replace(asbytes('a'), sixu('\u03a3'))
            tgt = [[sixu(' \u03a3bc '), ''],
                   ['12345', sixu('MixedC\u03a3se')],
                   ['123 \t 345 \x00', 'UPPER']]
            assert_(issubclass(R.dtype.type, np.unicode_))
            assert_array_equal(R, tgt) 

def test_char_radd(self):
        # GH issue 9620, reached gentype_add and raise TypeError
        np_s = np.string_('abc')
        np_u = np.unicode_('abc')
        s = b'def'
        u = u'def'
        assert_(np_s.__radd__(np_s) is NotImplemented)
        assert_(np_s.__radd__(np_u) is NotImplemented)
        assert_(np_s.__radd__(s) is NotImplemented)
        assert_(np_s.__radd__(u) is NotImplemented)
        assert_(np_u.__radd__(np_s) is NotImplemented)
        assert_(np_u.__radd__(np_u) is NotImplemented)
        assert_(np_u.__radd__(s) is NotImplemented)
        assert_(np_u.__radd__(u) is NotImplemented)
        assert_(s + np_s == b'defabc')
        assert_(u + np_u == u'defabc')


        class Mystr(str, np.generic):
            # would segfault
            pass

        ret = s + Mystr('abc')
        assert_(type(ret) is type(s)) 

def test_void_scalar_structured_data(self):
        dt = np.dtype([('name', np.unicode_, 16), ('grades', np.float64, (2,))])
        x = np.array(('ndarray_scalar', (1.2, 3.0)), dtype=dt)[()]
        assert_(isinstance(x, np.void))
        mv_x = memoryview(x)
        expected_size = 16 * np.dtype((np.unicode_, 1)).itemsize
        expected_size += 2 * np.dtype((np.float64, 1)).itemsize
        assert_equal(mv_x.itemsize, expected_size)
        assert_equal(mv_x.ndim, 0)
        assert_equal(mv_x.shape, ())
        assert_equal(mv_x.strides, ())
        assert_equal(mv_x.suboffsets, ())

        # check scalar format string against ndarray format string
        a = np.array([('Sarah', (8.0, 7.0)), ('John', (6.0, 7.0))], dtype=dt)
        assert_(isinstance(a, np.ndarray))
        mv_a = memoryview(a)
        assert_equal(mv_x.itemsize, mv_a.itemsize)
        assert_equal(mv_x.format, mv_a.format) 

def test_join(self):
        if sys.version_info[0] >= 3:
            # NOTE: list(b'123') == [49, 50, 51]
            #       so that b','.join(b'123') results to an error on Py3
            A0 = self.A.decode('ascii')
        else:
            A0 = self.A

        A = np.char.join([',', '#'], A0)
        if sys.version_info[0] >= 3:
            assert_(issubclass(A.dtype.type, np.unicode_))
        else:
            assert_(issubclass(A.dtype.type, np.string_))
        assert_array_equal(np.char.join([',', '#'], A0),
                           [
            [' ,a,b,c, ', ''],
            ['1,2,3,4,5', 'M#i#x#e#d#C#a#s#e'],
            ['1,2,3, ,\t, ,3,4,5, ,\x00, ', 'U#P#P#E#R']]) 

def _ensure_supported_dtypes(array):
        # We only support these types for now, as we need to read them in Java
        if array.dtype.kind == 'i':
            array = array.astype('<i8')
        elif array.dtype.kind == 'f':
            array = array.astype('<f8')
        elif array.dtype.kind in ('O', 'U', 'S'):
            if array.dtype.kind == 'O' and infer_dtype(array) not in ['unicode', 'string', 'bytes']:
                # `string` in python2 and `bytes` in python3
                raise UnhandledDtypeException("Casting object column to string failed")
            try:
                array = array.astype(np.unicode_)
            except (UnicodeDecodeError, SystemError):
                # `UnicodeDecodeError` in python2 and `SystemError` in python3
                array = np.array([s.decode('utf-8') for s in array])
            except:
                raise UnhandledDtypeException("Only unicode and utf8 strings are supported.")
        else:
            raise UnhandledDtypeException("Unsupported dtype '%s' - only int64, float64 and U are supported" % array.dtype)
        # Everything is little endian in tickstore
        if array.dtype.byteorder != '<':
            array = array.astype(array.dtype.newbyteorder('<'))
        return array 

def test_join(self):
        if sys.version_info[0] >= 3:
            # NOTE: list(b'123') == [49, 50, 51]
            #       so that b','.join(b'123') results to an error on Py3
            A0 = self.A.decode('ascii')
        else:
            A0 = self.A

        A = np.char.join([',', '#'], A0)
        if sys.version_info[0] >= 3:
            assert_(issubclass(A.dtype.type, np.unicode_))
        else:
            assert_(issubclass(A.dtype.type, np.string_))
        tgt = np.array([[' ,a,b,c, ', ''],
                        ['1,2,3,4,5', 'M#i#x#e#d#C#a#s#e'],
                        ['1,2,3, ,\t, ,3,4,5, ,\x00, ', 'U#P#P#E#R']])
        assert_array_equal(np.char.join([',', '#'], A0), tgt) 

def test_lstrip(self):
        tgt = [[b'abc ', b''],
               [b'12345', b'MixedCase'],
               [b'123 \t 345 \0 ', b'UPPER']]
        assert_(issubclass(self.A.lstrip().dtype.type, np.string_))
        assert_array_equal(self.A.lstrip(), tgt)

        tgt = [[b' abc', b''],
               [b'2345', b'ixedCase'],
               [b'23 \t 345 \x00', b'UPPER']]
        assert_array_equal(self.A.lstrip([b'1', b'M']), tgt)

        tgt = [[u'\u03a3 ', ''],
               ['12345', 'MixedCase'],
               ['123 \t 345 \0 ', 'UPPER']]
        assert_(issubclass(self.B.lstrip().dtype.type, np.unicode_))
        assert_array_equal(self.B.lstrip(), tgt) 

def test_replace(self):
        R = self.A.replace([b'3', b'a'],
                           [b'##########', b'@'])
        tgt = [[b' abc ', b''],
               [b'12##########45', b'MixedC@se'],
               [b'12########## \t ##########45 \x00', b'UPPER']]
        assert_(issubclass(R.dtype.type, np.string_))
        assert_array_equal(R, tgt)

        if sys.version_info[0] < 3:
            # NOTE: b'abc'.replace(b'a', 'b') is not allowed on Py3
            R = self.A.replace(b'a', u'\u03a3')
            tgt = [[u' \u03a3bc ', ''],
                   ['12345', u'MixedC\u03a3se'],
                   ['123 \t 345 \x00', 'UPPER']]
            assert_(issubclass(R.dtype.type, np.unicode_))
            assert_array_equal(R, tgt) 

def test_strip(self):
        tgt = [[b'abc', b''],
               [b'12345', b'MixedCase'],
               [b'123 \t 345', b'UPPER']]
        assert_(issubclass(self.A.strip().dtype.type, np.string_))
        assert_array_equal(self.A.strip(), tgt)

        tgt = [[b' abc ', b''],
               [b'234', b'ixedCas'],
               [b'23 \t 345 \x00', b'UPP']]
        assert_array_equal(self.A.strip([b'15', b'EReM']), tgt)

        tgt = [[u'\u03a3', ''],
               ['12345', 'MixedCase'],
               ['123 \t 345', 'UPPER']]
        assert_(issubclass(self.B.strip().dtype.type, np.unicode_))
        assert_array_equal(self.B.strip(), tgt) 

def test_select_dtypes_str_raises(self):
        df = DataFrame({'a': list('abc'),
                        'g': list(u('abc')),
                        'b': list(range(1, 4)),
                        'c': np.arange(3, 6).astype('u1'),
                        'd': np.arange(4.0, 7.0, dtype='float64'),
                        'e': [True, False, True],
                        'f': pd.date_range('now', periods=3).values})
        string_dtypes = set((str, 'str', np.string_, 'S1',
                             'unicode', np.unicode_, 'U1'))
        try:
            string_dtypes.add(unicode)
        except NameError:
            pass
        for dt in string_dtypes:
            with tm.assert_raises_regex(TypeError,
                                        'string dtypes are not allowed'):
                df.select_dtypes(include=[dt])
            with tm.assert_raises_regex(TypeError,
                                        'string dtypes are not allowed'):
                df.select_dtypes(exclude=[dt]) 

def test_char_radd(self):
        # GH issue 9620, reached gentype_add and raise TypeError
        np_s = np.string_('abc')
        np_u = np.unicode_('abc')
        s = b'def'
        u = u'def'
        assert_(np_s.__radd__(np_s) is NotImplemented)
        assert_(np_s.__radd__(np_u) is NotImplemented)
        assert_(np_s.__radd__(s) is NotImplemented)
        assert_(np_s.__radd__(u) is NotImplemented)
        assert_(np_u.__radd__(np_s) is NotImplemented)
        assert_(np_u.__radd__(np_u) is NotImplemented)
        assert_(np_u.__radd__(s) is NotImplemented)
        assert_(np_u.__radd__(u) is NotImplemented)
        assert_(s + np_s == b'defabc')
        assert_(u + np_u == u'defabc')


        class Mystr(str, np.generic):
            # would segfault
            pass

        ret = s + Mystr('abc')
        assert_(type(ret) is type(s)) 

def test_from_unicode_array(self):
        A = np.array([['abc', u'Sigma \u03a3'],
                      ['long   ', '0123456789']])
        assert_equal(A.dtype.type, np.unicode_)
        B = np.char.array(A)
        assert_array_equal(B, A)
        assert_equal(B.dtype, A.dtype)
        assert_equal(B.shape, A.shape)
        B = np.char.array(A, **kw_unicode_true)
        assert_array_equal(B, A)
        assert_equal(B.dtype, A.dtype)
        assert_equal(B.shape, A.shape)

        def fail():
            np.char.array(A, **kw_unicode_false)

        assert_raises(UnicodeEncodeError, fail) 

def test_from_unicode_array(self):
        A = np.array([['abc', sixu('Sigma \u03a3')],
                      ['long   ', '0123456789']])
        assert_equal(A.dtype.type, np.unicode_)
        B = np.char.array(A)
        assert_array_equal(B, A)
        assert_equal(B.dtype, A.dtype)
        assert_equal(B.shape, A.shape)
        B = np.char.array(A, **kw_unicode_true)
        assert_array_equal(B, A)
        assert_equal(B.dtype, A.dtype)
        assert_equal(B.shape, A.shape)

        def fail():
            np.char.array(A, **kw_unicode_false)

        self.assertRaises(UnicodeEncodeError, fail) 

def test_strip(self):
        tgt = [[b'abc', b''],
               [b'12345', b'MixedCase'],
               [b'123 \t 345', b'UPPER']]
        assert_(issubclass(self.A.strip().dtype.type, np.string_))
        assert_array_equal(self.A.strip(), tgt)

        tgt = [[b' abc ', b''],
               [b'234', b'ixedCas'],
               [b'23 \t 345 \x00', b'UPP']]
        assert_array_equal(self.A.strip([b'15', b'EReM']), tgt)

        tgt = [[u'\u03a3', ''],
               ['12345', 'MixedCase'],
               ['123 \t 345', 'UPPER']]
        assert_(issubclass(self.B.strip().dtype.type, np.unicode_))
        assert_array_equal(self.B.strip(), tgt) 

def test_rstrip(self):
        assert_(issubclass(self.A.rstrip().dtype.type, np.string_))

        tgt = [[b' abc', b''],
               [b'12345', b'MixedCase'],
               [b'123 \t 345', b'UPPER']]
        assert_array_equal(self.A.rstrip(), tgt)

        tgt = [[b' abc ', b''],
               [b'1234', b'MixedCase'],
               [b'123 \t 345 \x00', b'UPP']
               ]
        assert_array_equal(self.A.rstrip([b'5', b'ER']), tgt)

        tgt = [[u' \u03a3', ''],
               ['12345', 'MixedCase'],
               ['123 \t 345', 'UPPER']]
        assert_(issubclass(self.B.rstrip().dtype.type, np.unicode_))
        assert_array_equal(self.B.rstrip(), tgt) 

def test_rstrip(self):
        assert_(issubclass(self.A.rstrip().dtype.type, np.string_))

        tgt = [[b' abc', b''],
               [b'12345', b'MixedCase'],
               [b'123 \t 345', b'UPPER']]
        assert_array_equal(self.A.rstrip(), tgt)

        tgt = [[b' abc ', b''],
               [b'1234', b'MixedCase'],
               [b'123 \t 345 \x00', b'UPP']
               ]
        assert_array_equal(self.A.rstrip([b'5', b'ER']), tgt)

        tgt = [[u' \u03a3', ''],
               ['12345', 'MixedCase'],
               ['123 \t 345', 'UPPER']]
        assert_(issubclass(self.B.rstrip().dtype.type, np.unicode_))
        assert_array_equal(self.B.rstrip(), tgt) 

def test_replace(self):
        R = self.A.replace([b'3', b'a'],
                           [b'##########', b'@'])
        tgt = [[b' abc ', b''],
               [b'12##########45', b'MixedC@se'],
               [b'12########## \t ##########45 \x00', b'UPPER']]
        assert_(issubclass(R.dtype.type, np.string_))
        assert_array_equal(R, tgt)

        if sys.version_info[0] < 3:
            # NOTE: b'abc'.replace(b'a', 'b') is not allowed on Py3
            R = self.A.replace(b'a', u'\u03a3')
            tgt = [[u' \u03a3bc ', ''],
                   ['12345', u'MixedC\u03a3se'],
                   ['123 \t 345 \x00', 'UPPER']]
            assert_(issubclass(R.dtype.type, np.unicode_))
            assert_array_equal(R, tgt) 

def nan_filter(self):
        """Populates Target List and filters out values which are nan
        
        """
        
        # filter out nan values in numerical attributes
        for att in self.catalog_atts:
            if ('close' in att) or ('bright' in att):
                continue
            if getattr(self, att).shape[0] == 0:
                pass
            elif (type(getattr(self, att)[0]) == str) or (type(getattr(self, att)[0]) == bytes):
                # FIXME: intent here unclear: 
                #   note float('nan') is an IEEE NaN, getattr(.) is a str, and != on NaNs is special
                i = np.where(getattr(self, att) != float('nan'))[0]
                self.revise_lists(i)
            # exclude non-numerical types
            elif type(getattr(self, att)[0]) not in (np.unicode_, np.string_, np.bool_, bytes):
                if att == 'coords':
                    i1 = np.where(~np.isnan(self.coords.ra.to('deg').value))[0]
                    i2 = np.where(~np.isnan(self.coords.dec.to('deg').value))[0]
                    i = np.intersect1d(i1,i2)
                else:
                    i = np.where(~np.isnan(getattr(self, att)))[0]
                self.revise_lists(i) 

def test_unicode_upconvert(self):
        A = np.char.array(['abc'])
        B = np.char.array([u'\u03a3'])
        assert_(issubclass((A + B).dtype.type, np.unicode_)) 

def GetNumpyAppendFn(dtype):
  # numpy dtype for strings are variable length. We can not compare
  # dtype with a single constant (np.string does not exist) to decide
  # dtype is a "string" type. We need to compare the dtype.type to be
  # sure it's a string type.
  if dtype.type == np.string_ or dtype.type == np.unicode_:
    if _FAST_TENSOR_UTIL_AVAILABLE:
      return fast_tensor_util.AppendObjectArrayToTensorProto
    else:
      return SlowAppendObjectArrayToTensorProto
  return GetFromNumpyDTypeDict(_NP_TO_APPEND_FN, dtype) 

def test_pickle_py2_bytes_encoding(self):
        # Check that arrays and scalars pickled on Py2 are
        # unpickleable on Py3 using encoding='bytes'

        test_data = [
            # (original, py2_pickle)
            (np.unicode_('\u6f2c'),
             asbytes("cnumpy.core.multiarray\nscalar\np0\n(cnumpy\ndtype\np1\n"
                     "(S'U1'\np2\nI0\nI1\ntp3\nRp4\n(I3\nS'<'\np5\nNNNI4\nI4\n"
                     "I0\ntp6\nbS',o\\x00\\x00'\np7\ntp8\nRp9\n.")),

            (np.array([9e123], dtype=np.float64),
             asbytes("cnumpy.core.multiarray\n_reconstruct\np0\n(cnumpy\nndarray\n"
                     "p1\n(I0\ntp2\nS'b'\np3\ntp4\nRp5\n(I1\n(I1\ntp6\ncnumpy\ndtype\n"
                     "p7\n(S'f8'\np8\nI0\nI1\ntp9\nRp10\n(I3\nS'<'\np11\nNNNI-1\nI-1\n"
                     "I0\ntp12\nbI00\nS'O\\x81\\xb7Z\\xaa:\\xabY'\np13\ntp14\nb.")),

            (np.array([(9e123,)], dtype=[('name', float)]),
             asbytes("cnumpy.core.multiarray\n_reconstruct\np0\n(cnumpy\nndarray\np1\n"
                     "(I0\ntp2\nS'b'\np3\ntp4\nRp5\n(I1\n(I1\ntp6\ncnumpy\ndtype\np7\n"
                     "(S'V8'\np8\nI0\nI1\ntp9\nRp10\n(I3\nS'|'\np11\nN(S'name'\np12\ntp13\n"
                     "(dp14\ng12\n(g7\n(S'f8'\np15\nI0\nI1\ntp16\nRp17\n(I3\nS'<'\np18\nNNNI-1\n"
                     "I-1\nI0\ntp19\nbI0\ntp20\nsI8\nI1\nI0\ntp21\n"
                     "bI00\nS'O\\x81\\xb7Z\\xaa:\\xabY'\np22\ntp23\nb.")),
        ]

        if sys.version_info[:2] >= (3, 4):
            # encoding='bytes' was added in Py3.4
            for original, data in test_data:
                result = pickle.loads(data, encoding='bytes')
                assert_equal(result, original)

                if isinstance(result, np.ndarray) and result.dtype.names:
                    for name in result.dtype.names:
                        assert_(isinstance(name, str)) 

def test_char_repeat(self):
        np_s = np.string_('abc')
        np_u = np.unicode_('abc')
        np_i = np.int(5)
        res_np = np_s * np_i
        res_s = b'abc' * 5
        assert_(res_np == res_s) 

def _getconv(dtype):
    """ Find the correct dtype converter. Adapted from matplotlib """

    def floatconv(x):
        x.lower()
        if '0x' in x:
            return float.fromhex(x)
        return float(x)

    typ = dtype.type
    if issubclass(typ, np.bool_):
        return lambda x: bool(int(x))
    if issubclass(typ, np.uint64):
        return np.uint64
    if issubclass(typ, np.int64):
        return np.int64
    if issubclass(typ, np.integer):
        return lambda x: int(float(x))
    elif issubclass(typ, np.longdouble):
        return np.longdouble
    elif issubclass(typ, np.floating):
        return floatconv
    elif issubclass(typ, complex):
        return lambda x: complex(asstr(x))
    elif issubclass(typ, np.bytes_):
        return asbytes
    elif issubclass(typ, np.unicode_):
        return asunicode
    else:
        return asstr

# amount of lines loadtxt reads in one chunk, can be overriden for testing 

def setup(self):
        TestComparisons.setup(self)
        self.B = np.array([['efg', '123  '],
                           ['051', 'tuv']], np.unicode_).view(np.chararray) 

def test_unicode_string_comparison(self,level=rlevel):
        # Ticket #190
        a = np.array('hello', np.unicode_)
        b = np.array('world')
        a == b 

def setup(self):
        TestComparisons.setup(self)
        self.A = np.array([['abc', '123'],
                           ['789', 'xyz']], np.unicode_).view(np.chararray) 

def test_lower(self):
        assert_(issubclass(self.A.lower().dtype.type, np.string_))
        assert_array_equal(self.A.lower(), asbytes_nested([
                [' abc ', ''],
                ['12345', 'mixedcase'],
                ['123 \t 345 \0 ', 'upper']]))
        assert_(issubclass(self.B.lower().dtype.type, np.unicode_))
        assert_array_equal(self.B.lower(), [
                [sixu(' \u03c3 '), sixu('')],
                [sixu('12345'), sixu('mixedcase')],
                [sixu('123 \t 345 \0 '), sixu('upper')]])