Python pandas.compat.u_safe() Examples

def normalize(self, form):
        """Return the Unicode normal form for the strings in the Series/Index.
        For more information on the forms, see the
        :func:`unicodedata.normalize`.

        Parameters
        ----------
        form : {'NFC', 'NFKC', 'NFD', 'NFKD'}
            Unicode form

        Returns
        -------
        normalized : Series/Index of objects
        """
        import unicodedata
        f = lambda x: unicodedata.normalize(form, compat.u_safe(x))
        result = _na_map(f, self._data)
        return self._wrap_result(result) 

def normalize(self, form):
        """Return the Unicode normal form for the strings in the Series/Index.
        For more information on the forms, see the
        :func:`unicodedata.normalize`.

        Parameters
        ----------
        form : {'NFC', 'NFKC', 'NFD', 'NFKD'}
            Unicode form

        Returns
        -------
        normalized : Series/Index of objects
        """
        import unicodedata
        f = lambda x: unicodedata.normalize(form, compat.u_safe(x))
        result = _na_map(f, self._data)
        return self._wrap_result(result) 

def validate_col(self, itemsize=None):
        """ validate this column: return the compared against itemsize """

        # validate this column for string truncation (or reset to the max size)
        if _ensure_decoded(self.kind) == u('string'):
            c = self.col
            if c is not None:
                if itemsize is None:
                    itemsize = self.itemsize
                if c.itemsize < itemsize:
                    raise ValueError(
                        "Trying to store a string with len [%s] in [%s] "
                        "column but\nthis column has a limit of [%s]!\n"
                        "Consider using min_itemsize to preset the sizes on "
                        "these columns" % (itemsize, self.cname, c.itemsize))
                return c.itemsize

        return None 

def normalize(self, form):
        """
        Return the Unicode normal form for the strings in the Series/Index.
        For more information on the forms, see the
        :func:`unicodedata.normalize`.

        Parameters
        ----------
        form : {'NFC', 'NFKC', 'NFD', 'NFKD'}
            Unicode form

        Returns
        -------
        normalized : Series/Index of objects
        """
        import unicodedata
        f = lambda x: unicodedata.normalize(form, compat.u_safe(x))
        result = _na_map(f, self._parent)
        return self._wrap_result(result) 

def _unconvert_index(data, kind, encoding=None):
    kind = _ensure_decoded(kind)
    if kind == u('datetime64'):
        index = DatetimeIndex(data)
    elif kind == u('datetime'):
        index = np.array([datetime.fromtimestamp(v) for v in data],
                         dtype=object)
    elif kind == u('date'):
        try:
            index = np.array(
                [date.fromordinal(v) for v in data], dtype=object)
        except (ValueError):
            index = np.array(
                [date.fromtimestamp(v) for v in data], dtype=object)
    elif kind in (u('integer'), u('float')):
        index = np.array(data)
    elif kind in (u('string')):
        index = _unconvert_string_array(data, nan_rep=None, encoding=encoding)
    elif kind == u('object'):
        index = np.array(data[0])
    else:  # pragma: no cover
        raise ValueError('unrecognized index type %s' % kind)
    return index 

def _unconvert_index(data, kind, encoding=None):
    kind = _ensure_decoded(kind)
    if kind == u('datetime64'):
        index = DatetimeIndex(data)
    elif kind == u('timedelta64'):
        index = TimedeltaIndex(data)
    elif kind == u('datetime'):
        index = np.asarray([datetime.fromtimestamp(v) for v in data],
                           dtype=object)
    elif kind == u('date'):
        try:
            index = np.asarray(
                [date.fromordinal(v) for v in data], dtype=object)
        except (ValueError):
            index = np.asarray(
                [date.fromtimestamp(v) for v in data], dtype=object)
    elif kind in (u('integer'), u('float')):
        index = np.asarray(data)
    elif kind in (u('string')):
        index = _unconvert_string_array(data, nan_rep=None, encoding=encoding)
    elif kind == u('object'):
        index = np.asarray(data[0])
    else:  # pragma: no cover
        raise ValueError('unrecognized index type %s' % kind)
    return index 

def set_kind(self):
        # set my kind if we can
        if self.dtype is not None:
            dtype = _ensure_decoded(self.dtype)
            if dtype.startswith(u('string')) or dtype.startswith(u('bytes')):
                self.kind = 'string'
            elif dtype.startswith(u('float')):
                self.kind = 'float'
            elif dtype.startswith(u('int')) or dtype.startswith(u('uint')):
                self.kind = 'integer'
            elif dtype.startswith(u('date')):
                self.kind = 'datetime'
            elif dtype.startswith(u('timedelta')):
                self.kind = 'timedelta'
            elif dtype.startswith(u('bool')):
                self.kind = 'bool'
            else:
                raise AssertionError(
                    "cannot interpret dtype of [%s] in [%s]" % (dtype, self))

            # set my typ if we need
            if self.typ is None:
                self.typ = getattr(self.description, self.cname, None) 

def validate_col(self, itemsize=None):
        """ validate this column: return the compared against itemsize """

        # validate this column for string truncation (or reset to the max size)
        if _ensure_decoded(self.kind) == u('string'):
            c = self.col
            if c is not None:
                if itemsize is None:
                    itemsize = self.itemsize
                if c.itemsize < itemsize:
                    raise ValueError(
                        "Trying to store a string with len [%s] in [%s] "
                        "column but\nthis column has a limit of [%s]!\n"
                        "Consider using min_itemsize to preset the sizes on "
                        "these columns" % (itemsize, self.cname, c.itemsize))
                return c.itemsize

        return None 

def normalize(self, form):
        """Return the Unicode normal form for the strings in the Series/Index.
        For more information on the forms, see the
        :func:`unicodedata.normalize`.

        Parameters
        ----------
        form : {'NFC', 'NFKC', 'NFD', 'NFKD'}
            Unicode form

        Returns
        -------
        normalized : Series/Index of objects
        """
        import unicodedata
        f = lambda x: unicodedata.normalize(form, compat.u_safe(x))
        result = _na_map(f, self._data)
        return self._wrap_result(result) 

def validate_col(self, itemsize=None):
        """ validate this column: return the compared against itemsize """

        # validate this column for string truncation (or reset to the max size)
        if _ensure_decoded(self.kind) == u('string'):
            c = self.col
            if c is not None:
                if itemsize is None:
                    itemsize = self.itemsize
                if c.itemsize < itemsize:
                    raise ValueError(
                        "Trying to store a string with len [%s] in [%s] "
                        "column but\nthis column has a limit of [%s]!\n"
                        "Consider using min_itemsize to preset the sizes on "
                        "these columns" % (itemsize, self.cname, c.itemsize))
                return c.itemsize

        return None 

def normalize(self, form):
        """
        Return the Unicode normal form for the strings in the Series/Index.
        For more information on the forms, see the
        :func:`unicodedata.normalize`.

        Parameters
        ----------
        form : {'NFC', 'NFKC', 'NFD', 'NFKD'}
            Unicode form

        Returns
        -------
        normalized : Series/Index of objects
        """
        import unicodedata
        f = lambda x: unicodedata.normalize(form, compat.u_safe(x))
        result = _na_map(f, self._parent)
        return self._wrap_result(result) 

def validate_col(self, itemsize=None):
        """ validate this column: return the compared against itemsize """

        # validate this column for string truncation (or reset to the max size)
        if _ensure_decoded(self.kind) == u('string'):
            c = self.col
            if c is not None:
                if itemsize is None:
                    itemsize = self.itemsize
                if c.itemsize < itemsize:
                    raise ValueError(
                        "Trying to store a string with len [%s] in [%s] "
                        "column but\nthis column has a limit of [%s]!\n"
                        "Consider using min_itemsize to preset the sizes on "
                        "these columns" % (itemsize, self.cname, c.itemsize))
                return c.itemsize

        return None 

def _unconvert_index(data, kind, encoding=None):
    kind = _ensure_decoded(kind)
    if kind == u('datetime64'):
        index = DatetimeIndex(data)
    elif kind == u('timedelta64'):
        index = TimedeltaIndex(data)
    elif kind == u('datetime'):
        index = np.asarray([datetime.fromtimestamp(v) for v in data],
                           dtype=object)
    elif kind == u('date'):
        try:
            index = np.asarray(
                [date.fromordinal(v) for v in data], dtype=object)
        except (ValueError):
            index = np.asarray(
                [date.fromtimestamp(v) for v in data], dtype=object)
    elif kind in (u('integer'), u('float')):
        index = np.asarray(data)
    elif kind in (u('string')):
        index = _unconvert_string_array(data, nan_rep=None, encoding=encoding)
    elif kind == u('object'):
        index = np.asarray(data[0])
    else:  # pragma: no cover
        raise ValueError('unrecognized index type %s' % kind)
    return index 

def is_exists(self):
        """ has this table been created """
        return u('table') in self.group 

def maybe_set_size(self, min_itemsize=None, **kwargs):
        """ maybe set a string col itemsize:
               min_itemsize can be an integer or a dict with this columns name
               with an integer size """
        if _ensure_decoded(self.kind) == u('string'):

            if isinstance(min_itemsize, dict):
                min_itemsize = min_itemsize.get(self.name)

            if min_itemsize is not None and self.typ.itemsize < min_itemsize:
                self.typ = _tables(
                ).StringCol(itemsize=min_itemsize, pos=self.pos) 

def _need_convert(kind):
    kind = _ensure_decoded(kind)
    if kind in (u('datetime'), u('datetime64'), u('string')):
        return True
    return False 

def __init__(self, values=None, kind=None, typ=None,
                 cname=None, data=None, meta=None, metadata=None,
                 block=None, **kwargs):
        super(DataCol, self).__init__(values=values, kind=kind, typ=typ,
                                      cname=cname, **kwargs)
        self.dtype = None
        self.dtype_attr = u("%s_dtype" % self.name)
        self.meta = meta
        self.meta_attr = u("%s_meta" % self.name)
        self.set_data(data)
        self.set_metadata(metadata) 

def _unconvert_index_legacy(data, kind, legacy=False, encoding=None):
    kind = _ensure_decoded(kind)
    if kind == u('datetime'):
        index = to_datetime(data)
    elif kind in (u('integer')):
        index = np.asarray(data, dtype=object)
    elif kind in (u('string')):
        index = _unconvert_string_array(data, nan_rep=None, encoding=encoding)
    else:  # pragma: no cover
        raise ValueError('unrecognized index type %s' % kind)
    return index 

def read_array(self, key, start=None, stop=None):
        """ read an array for the specified node (off of group """
        import tables
        node = getattr(self.group, key)
        attrs = node._v_attrs

        transposed = getattr(attrs, 'transposed', False)

        if isinstance(node, tables.VLArray):
            ret = node[0][start:stop]
        else:
            dtype = getattr(attrs, 'value_type', None)
            shape = getattr(attrs, 'shape', None)

            if shape is not None:
                # length 0 axis
                ret = np.empty(shape, dtype=dtype)
            else:
                ret = node[start:stop]

            if dtype == u('datetime64'):

                # reconstruct a timezone if indicated
                ret = _set_tz(ret, getattr(attrs, 'tz', None), coerce=True)

            elif dtype == u('timedelta64'):
                ret = np.asarray(ret, dtype='m8[ns]')

        if transposed:
            return ret.T
        else:
            return ret 

def read_index(self, key, **kwargs):
        variety = _ensure_decoded(getattr(self.attrs, '%s_variety' % key))

        if variety == u('multi'):
            return self.read_multi_index(key, **kwargs)
        elif variety == u('block'):
            return self.read_block_index(key, **kwargs)
        elif variety == u('sparseint'):
            return self.read_sparse_intindex(key, **kwargs)
        elif variety == u('regular'):
            _, index = self.read_index_node(getattr(self.group, key), **kwargs)
            return index
        else:  # pragma: no cover
            raise TypeError('unrecognized index variety: %s' % variety) 

def read_index_node(self, node, start=None, stop=None):
        data = node[start:stop]
        # If the index was an empty array write_array_empty() will
        # have written a sentinel. Here we relace it with the original.
        if ('shape' in node._v_attrs and
                self._is_empty_array(getattr(node._v_attrs, 'shape'))):
            data = np.empty(getattr(node._v_attrs, 'shape'),
                            dtype=getattr(node._v_attrs, 'value_type'))
        kind = _ensure_decoded(node._v_attrs.kind)
        name = None

        if 'name' in node._v_attrs:
            name = _ensure_str(node._v_attrs.name)

        index_class = self._alias_to_class(_ensure_decoded(
            getattr(node._v_attrs, 'index_class', '')))
        factory = self._get_index_factory(index_class)

        kwargs = {}
        if u('freq') in node._v_attrs:
            kwargs['freq'] = node._v_attrs['freq']

        if u('tz') in node._v_attrs:
            kwargs['tz'] = node._v_attrs['tz']

        if kind in (u('date'), u('datetime')):
            index = factory(_unconvert_index(data, kind,
                                             encoding=self.encoding),
                            dtype=object, **kwargs)
        else:
            index = factory(_unconvert_index(data, kind,
                                             encoding=self.encoding), **kwargs)

        index.name = name

        return name, index 

def read(self, **kwargs):
        kwargs = self.validate_read(kwargs)
        index = self.read_index('index')
        sp_values = self.read_array('sp_values')
        sp_index = self.read_index('sp_index')
        return SparseSeries(sp_values, index=index, sparse_index=sp_index,
                            kind=self.kind or u('block'),
                            fill_value=self.fill_value,
                            name=self.name) 

def read_array(self, key, start=None, stop=None):
        """ read an array for the specified node (off of group """
        import tables
        node = getattr(self.group, key)
        attrs = node._v_attrs

        transposed = getattr(attrs, 'transposed', False)

        if isinstance(node, tables.VLArray):
            ret = node[0][start:stop]
        else:
            dtype = getattr(attrs, 'value_type', None)
            shape = getattr(attrs, 'shape', None)

            if shape is not None:
                # length 0 axis
                ret = np.empty(shape, dtype=dtype)
            else:
                ret = node[start:stop]

            if dtype == u('datetime64'):

                # reconstruct a timezone if indicated
                ret = _set_tz(ret, getattr(attrs, 'tz', None), coerce=True)

            elif dtype == u('timedelta64'):
                ret = np.asarray(ret, dtype='m8[ns]')

        if transposed:
            return ret.T
        else:
            return ret 

def _unconvert_index_legacy(data, kind, legacy=False, encoding=None):
    kind = _ensure_decoded(kind)
    if kind == u('datetime'):
        index = to_datetime(data)
    elif kind in (u('integer')):
        index = np.asarray(data, dtype=object)
    elif kind in (u('string')):
        index = _unconvert_string_array(data, nan_rep=None, encoding=encoding)
    else:  # pragma: no cover
        raise ValueError('unrecognized index type %s' % kind)
    return index 

def is_exists(self):
        """ has this table been created """
        return u('table') in self.group 

def _need_convert(kind):
    kind = _ensure_decoded(kind)
    if kind in (u('datetime'), u('datetime64'), u('string')):
        return True
    return False 

def read(self, **kwargs):
        kwargs = self.validate_read(kwargs)
        index = self.read_index('index')
        sp_values = self.read_array('sp_values')
        sp_index = self.read_index('sp_index')
        return SparseSeries(sp_values, index=index, sparse_index=sp_index,
                            kind=self.kind or u('block'),
                            fill_value=self.fill_value,
                            name=self.name) 

def read_index_node(self, node, start=None, stop=None):
        data = node[start:stop]
        # If the index was an empty array write_array_empty() will
        # have written a sentinel. Here we relace it with the original.
        if ('shape' in node._v_attrs and
                self._is_empty_array(getattr(node._v_attrs, 'shape'))):
            data = np.empty(getattr(node._v_attrs, 'shape'),
                            dtype=getattr(node._v_attrs, 'value_type'))
        kind = _ensure_decoded(node._v_attrs.kind)
        name = None

        if 'name' in node._v_attrs:
            name = _ensure_str(node._v_attrs.name)

        index_class = self._alias_to_class(_ensure_decoded(
            getattr(node._v_attrs, 'index_class', '')))
        factory = self._get_index_factory(index_class)

        kwargs = {}
        if u('freq') in node._v_attrs:
            kwargs['freq'] = node._v_attrs['freq']

        if u('tz') in node._v_attrs:
            kwargs['tz'] = node._v_attrs['tz']

        if kind in (u('date'), u('datetime')):
            index = factory(_unconvert_index(data, kind,
                                             encoding=self.encoding),
                            dtype=object, **kwargs)
        else:
            index = factory(_unconvert_index(data, kind,
                                             encoding=self.encoding), **kwargs)

        index.name = name

        return name, index 

def groups(self):
        """return a list of all the top-level nodes (that are not themselves a
        pandas storage object)
        """
        _tables()
        self._check_if_open()
        return [
            g for g in self._handle.walk_nodes()
            if (getattr(g._v_attrs, 'pandas_type', None) or
                getattr(g, 'table', None) or
                (isinstance(g, _table_mod.table.Table) and
                 g._v_name != u('table')))
        ] 

def maybe_set_size(self, min_itemsize=None, **kwargs):
        """ maybe set a string col itemsize:
               min_itemsize can be an integer or a dict with this columns name
               with an integer size """
        if _ensure_decoded(self.kind) == u('string'):

            if isinstance(min_itemsize, dict):
                min_itemsize = min_itemsize.get(self.name)

            if min_itemsize is not None and self.typ.itemsize < min_itemsize:
                self.typ = _tables(
                ).StringCol(itemsize=min_itemsize, pos=self.pos)