Python pandas.util.testing.makeCustomDataframe() Examples

def check_basic_frame_series_alignment(self, engine, parser):
        tm.skip_if_no_ne(engine)

        def testit(r_idx_type, c_idx_type, index_name):
            df = mkdf(10, 10, data_gen_f=f, r_idx_type=r_idx_type,
                      c_idx_type=c_idx_type)
            index = getattr(df, index_name)
            s = Series(np.random.randn(5), index[:5])

            res = pd.eval('df + s', engine=engine, parser=parser)
            if r_idx_type == 'dt' or c_idx_type == 'dt':
                if engine == 'numexpr':
                    expected = df.add(s)
                else:
                    expected = df + s
            else:
                expected = df + s
            assert_frame_equal(res, expected)

        args = product(self.lhs_index_types, self.index_types,
                       ('index', 'columns'))
        for r_idx_type, c_idx_type, index_name in args:
            testit(r_idx_type, c_idx_type, index_name) 

def test_ix_empty_list_indexer_is_ok(self):
        with catch_warnings(record=True):
            from pandas.util.testing import makeCustomDataframe as mkdf
            df = mkdf(5, 2)
            # vertical empty
            tm.assert_frame_equal(df.ix[:, []], df.iloc[:, :0],
                                  check_index_type=True,
                                  check_column_type=True)
            # horizontal empty
            tm.assert_frame_equal(df.ix[[], :], df.iloc[:0, :],
                                  check_index_type=True,
                                  check_column_type=True)
            # horizontal empty
            tm.assert_frame_equal(df.ix[[]], df.iloc[:0, :],
                                  check_index_type=True,
                                  check_column_type=True) 

def test_to_csv_cols_reordering(self):
        # GH3454
        import pandas as pd

        chunksize = 5
        N = int(chunksize * 2.5)

        df = mkdf(N, 3)
        cs = df.columns
        cols = [cs[2], cs[0]]

        with ensure_clean() as path:
            df.to_csv(path, columns=cols, chunksize=chunksize)
            rs_c = pd.read_csv(path, index_col=0)

        assert_frame_equal(df[cols], rs_c, check_names=False) 

def test_medium_complex_frame_alignment(self, engine, parser):
        args = product(self.lhs_index_types, self.index_types,
                       self.index_types, self.index_types)

        with warnings.catch_warnings(record=True):
            warnings.simplefilter('always', RuntimeWarning)

            for r1, c1, r2, c2 in args:
                df = mkdf(3, 2, data_gen_f=f, r_idx_type=r1, c_idx_type=c1)
                df2 = mkdf(4, 2, data_gen_f=f, r_idx_type=r2, c_idx_type=c2)
                df3 = mkdf(5, 2, data_gen_f=f, r_idx_type=r2, c_idx_type=c2)
                if should_warn(df.index, df2.index, df3.index):
                    with tm.assert_produces_warning(RuntimeWarning):
                        res = pd.eval('df + df2 + df3', engine=engine,
                                      parser=parser)
                else:
                    res = pd.eval('df + df2 + df3',
                                  engine=engine, parser=parser)
                assert_frame_equal(res, df + df2 + df3) 

def test_basic_frame_alignment(self, engine, parser):
        args = product(self.lhs_index_types, self.index_types,
                       self.index_types)
        with warnings.catch_warnings(record=True):
            warnings.simplefilter('always', RuntimeWarning)
            for lr_idx_type, rr_idx_type, c_idx_type in args:
                df = mkdf(10, 10, data_gen_f=f, r_idx_type=lr_idx_type,
                          c_idx_type=c_idx_type)
                df2 = mkdf(20, 10, data_gen_f=f, r_idx_type=rr_idx_type,
                           c_idx_type=c_idx_type)
                # only warns if not monotonic and not sortable
                if should_warn(df.index, df2.index):
                    with tm.assert_produces_warning(RuntimeWarning):
                        res = pd.eval('df + df2', engine=engine, parser=parser)
                else:
                    res = pd.eval('df + df2', engine=engine, parser=parser)
                assert_frame_equal(res, df + df2) 

def test_header_multi_index(all_parsers):
    parser = all_parsers
    expected = tm.makeCustomDataframe(
        5, 3, r_idx_nlevels=2, c_idx_nlevels=4)

    data = """\
C0,,C_l0_g0,C_l0_g1,C_l0_g2

C1,,C_l1_g0,C_l1_g1,C_l1_g2
C2,,C_l2_g0,C_l2_g1,C_l2_g2
C3,,C_l3_g0,C_l3_g1,C_l3_g2
R0,R1,,,
R_l0_g0,R_l1_g0,R0C0,R0C1,R0C2
R_l0_g1,R_l1_g1,R1C0,R1C1,R1C2
R_l0_g2,R_l1_g2,R2C0,R2C1,R2C2
R_l0_g3,R_l1_g3,R3C0,R3C1,R3C2
R_l0_g4,R_l1_g4,R4C0,R4C1,R4C2
"""
    result = parser.read_csv(StringIO(data), header=[0, 1, 2, 3],
                             index_col=[0, 1])
    tm.assert_frame_equal(result, expected) 

def test_slice_locs_with_type_mismatch(self):
        df = tm.makeTimeDataFrame()
        stacked = df.stack()
        idx = stacked.index
        tm.assert_raises_regex(TypeError, '^Level type mismatch',
                               idx.slice_locs, (1, 3))
        tm.assert_raises_regex(TypeError, '^Level type mismatch',
                               idx.slice_locs,
                               df.index[5] + timedelta(
                                   seconds=30), (5, 2))
        df = tm.makeCustomDataframe(5, 5)
        stacked = df.stack()
        idx = stacked.index
        with tm.assert_raises_regex(TypeError, '^Level type mismatch'):
            idx.slice_locs(timedelta(seconds=30))
        # TODO: Try creating a UnicodeDecodeError in exception message
        with tm.assert_raises_regex(TypeError, '^Level type mismatch'):
            idx.slice_locs(df.index[1], (16, "a")) 

def test_slice_locs_with_type_mismatch():
    df = tm.makeTimeDataFrame()
    stacked = df.stack()
    idx = stacked.index
    with pytest.raises(TypeError, match='^Level type mismatch'):
        idx.slice_locs((1, 3))
    with pytest.raises(TypeError, match='^Level type mismatch'):
        idx.slice_locs(df.index[5] + timedelta(seconds=30), (5, 2))
    df = tm.makeCustomDataframe(5, 5)
    stacked = df.stack()
    idx = stacked.index
    with pytest.raises(TypeError, match='^Level type mismatch'):
        idx.slice_locs(timedelta(seconds=30))
    # TODO: Try creating a UnicodeDecodeError in exception message
    with pytest.raises(TypeError, match='^Level type mismatch'):
        idx.slice_locs(df.index[1], (16, "a")) 

def test_slice_locs_with_type_mismatch(self):
        df = tm.makeTimeDataFrame()
        stacked = df.stack()
        idx = stacked.index
        tm.assert_raises_regex(TypeError, '^Level type mismatch',
                               idx.slice_locs, (1, 3))
        tm.assert_raises_regex(TypeError, '^Level type mismatch',
                               idx.slice_locs,
                               df.index[5] + timedelta(
                                   seconds=30), (5, 2))
        df = tm.makeCustomDataframe(5, 5)
        stacked = df.stack()
        idx = stacked.index
        with tm.assert_raises_regex(TypeError, '^Level type mismatch'):
            idx.slice_locs(timedelta(seconds=30))
        # TODO: Try creating a UnicodeDecodeError in exception message
        with tm.assert_raises_regex(TypeError, '^Level type mismatch'):
            idx.slice_locs(df.index[1], (16, "a")) 

def test_header_multi_index(all_parsers):
    parser = all_parsers
    expected = tm.makeCustomDataframe(
        5, 3, r_idx_nlevels=2, c_idx_nlevels=4)

    data = """\
C0,,C_l0_g0,C_l0_g1,C_l0_g2

C1,,C_l1_g0,C_l1_g1,C_l1_g2
C2,,C_l2_g0,C_l2_g1,C_l2_g2
C3,,C_l3_g0,C_l3_g1,C_l3_g2
R0,R1,,,
R_l0_g0,R_l1_g0,R0C0,R0C1,R0C2
R_l0_g1,R_l1_g1,R1C0,R1C1,R1C2
R_l0_g2,R_l1_g2,R2C0,R2C1,R2C2
R_l0_g3,R_l1_g3,R3C0,R3C1,R3C2
R_l0_g4,R_l1_g4,R4C0,R4C1,R4C2
"""
    result = parser.read_csv(StringIO(data), header=[0, 1, 2, 3],
                             index_col=[0, 1])
    tm.assert_frame_equal(result, expected) 

def test_to_csv_cols_reordering(self):
        # GH3454
        import pandas as pd

        chunksize = 5
        N = int(chunksize * 2.5)

        df = mkdf(N, 3)
        cs = df.columns
        cols = [cs[2], cs[0]]

        with ensure_clean() as path:
            df.to_csv(path, columns=cols, chunksize=chunksize)
            rs_c = pd.read_csv(path, index_col=0)

        assert_frame_equal(df[cols], rs_c, check_names=False) 

def test_ix_empty_list_indexer_is_ok(self):
        with catch_warnings(record=True):
            from pandas.util.testing import makeCustomDataframe as mkdf
            df = mkdf(5, 2)
            # vertical empty
            tm.assert_frame_equal(df.ix[:, []], df.iloc[:, :0],
                                  check_index_type=True,
                                  check_column_type=True)
            # horizontal empty
            tm.assert_frame_equal(df.ix[[], :], df.iloc[:0, :],
                                  check_index_type=True,
                                  check_column_type=True)
            # horizontal empty
            tm.assert_frame_equal(df.ix[[]], df.iloc[:0, :],
                                  check_index_type=True,
                                  check_column_type=True) 

def test_medium_complex_frame_alignment(self, engine, parser):
        args = product(self.lhs_index_types, self.index_types,
                       self.index_types, self.index_types)

        with warnings.catch_warnings(record=True):
            warnings.simplefilter('always', RuntimeWarning)

            for r1, c1, r2, c2 in args:
                df = mkdf(3, 2, data_gen_f=f, r_idx_type=r1, c_idx_type=c1)
                df2 = mkdf(4, 2, data_gen_f=f, r_idx_type=r2, c_idx_type=c2)
                df3 = mkdf(5, 2, data_gen_f=f, r_idx_type=r2, c_idx_type=c2)
                if should_warn(df.index, df2.index, df3.index):
                    with tm.assert_produces_warning(RuntimeWarning):
                        res = pd.eval('df + df2 + df3', engine=engine,
                                      parser=parser)
                else:
                    res = pd.eval('df + df2 + df3',
                                  engine=engine, parser=parser)
                assert_frame_equal(res, df + df2 + df3) 

def test_basic_frame_alignment(self, engine, parser):
        args = product(self.lhs_index_types, self.index_types,
                       self.index_types)
        with warnings.catch_warnings(record=True):
            warnings.simplefilter('always', RuntimeWarning)
            for lr_idx_type, rr_idx_type, c_idx_type in args:
                df = mkdf(10, 10, data_gen_f=f, r_idx_type=lr_idx_type,
                          c_idx_type=c_idx_type)
                df2 = mkdf(20, 10, data_gen_f=f, r_idx_type=rr_idx_type,
                           c_idx_type=c_idx_type)
                # only warns if not monotonic and not sortable
                if should_warn(df.index, df2.index):
                    with tm.assert_produces_warning(RuntimeWarning):
                        res = pd.eval('df + df2', engine=engine, parser=parser)
                else:
                    res = pd.eval('df + df2', engine=engine, parser=parser)
                assert_frame_equal(res, df + df2) 

def test_basic_frame_series_alignment(self, engine, parser):
        def testit(r_idx_type, c_idx_type, index_name):
            df = mkdf(10, 10, data_gen_f=f, r_idx_type=r_idx_type,
                      c_idx_type=c_idx_type)
            index = getattr(df, index_name)
            s = Series(np.random.randn(5), index[:5])

            if should_warn(df.index, s.index):
                with tm.assert_produces_warning(RuntimeWarning):
                    res = pd.eval('df + s', engine=engine, parser=parser)
            else:
                res = pd.eval('df + s', engine=engine, parser=parser)

            if r_idx_type == 'dt' or c_idx_type == 'dt':
                expected = df.add(s) if engine == 'numexpr' else df + s
            else:
                expected = df + s
            assert_frame_equal(res, expected)

        args = product(self.lhs_index_types, self.index_types,
                       ('index', 'columns'))
        with warnings.catch_warnings(record=True):
            warnings.simplefilter('always', RuntimeWarning)
            for r_idx_type, c_idx_type, index_name in args:
                testit(r_idx_type, c_idx_type, index_name) 

def test_slice_locs_with_type_mismatch():
    df = tm.makeTimeDataFrame()
    stacked = df.stack()
    idx = stacked.index
    with pytest.raises(TypeError, match='^Level type mismatch'):
        idx.slice_locs((1, 3))
    with pytest.raises(TypeError, match='^Level type mismatch'):
        idx.slice_locs(df.index[5] + timedelta(seconds=30), (5, 2))
    df = tm.makeCustomDataframe(5, 5)
    stacked = df.stack()
    idx = stacked.index
    with pytest.raises(TypeError, match='^Level type mismatch'):
        idx.slice_locs(timedelta(seconds=30))
    # TODO: Try creating a UnicodeDecodeError in exception message
    with pytest.raises(TypeError, match='^Level type mismatch'):
        idx.slice_locs(df.index[1], (16, "a")) 

def test_ix_empty_list_indexer_is_ok(self):
        with catch_warnings(record=True):
            from pandas.util.testing import makeCustomDataframe as mkdf
            df = mkdf(5, 2)
            # vertical empty
            tm.assert_frame_equal(df.ix[:, []], df.iloc[:, :0],
                                  check_index_type=True,
                                  check_column_type=True)
            # horizontal empty
            tm.assert_frame_equal(df.ix[[], :], df.iloc[:0, :],
                                  check_index_type=True,
                                  check_column_type=True)
            # horizontal empty
            tm.assert_frame_equal(df.ix[[]], df.iloc[:0, :],
                                  check_index_type=True,
                                  check_column_type=True) 

def test_to_csv_cols_reordering(self):
        # GH3454
        import pandas as pd

        chunksize = 5
        N = int(chunksize * 2.5)

        df = mkdf(N, 3)
        cs = df.columns
        cols = [cs[2], cs[0]]

        with ensure_clean() as path:
            df.to_csv(path, columns=cols, chunksize=chunksize)
            rs_c = pd.read_csv(path, index_col=0)

        assert_frame_equal(df[cols], rs_c, check_names=False) 

def setUpClass(cls):
        super(TestClipboard, cls).setUpClass()
        cls.data = {}
        cls.data['string'] = mkdf(5, 3, c_idx_type='s', r_idx_type='i',
                                  c_idx_names=[None], r_idx_names=[None])
        cls.data['int'] = mkdf(5, 3, data_gen_f=lambda *args: randint(2),
                               c_idx_type='s', r_idx_type='i',
                               c_idx_names=[None], r_idx_names=[None])
        cls.data['float'] = mkdf(5, 3,
                                 data_gen_f=lambda r, c: float(r) + 0.01,
                                 c_idx_type='s', r_idx_type='i',
                                 c_idx_names=[None], r_idx_names=[None])
        cls.data['mixed'] = DataFrame({'a': np.arange(1.0, 6.0) + 0.01,
                                       'b': np.arange(1, 6),
                                       'c': list('abcde')})
        # Test GH-5346
        max_rows = get_option('display.max_rows')
        cls.data['longdf'] = mkdf(max_rows+1, 3, data_gen_f=lambda *args: randint(2),
                                  c_idx_type='s', r_idx_type='i',
                                  c_idx_names=[None], r_idx_names=[None])  
        cls.data_types = list(cls.data.keys()) 

def check_series_frame_commutativity(self, engine, parser):
        tm.skip_if_no_ne(engine)
        args = product(self.lhs_index_types, self.index_types, ('+', '*'),
                       ('index', 'columns'))
        for r_idx_type, c_idx_type, op, index_name in args:
            df = mkdf(10, 10, data_gen_f=f, r_idx_type=r_idx_type,
                      c_idx_type=c_idx_type)
            index = getattr(df, index_name)
            s = Series(np.random.randn(5), index[:5])

            lhs = 's {0} df'.format(op)
            rhs = 'df {0} s'.format(op)
            a = pd.eval(lhs, engine=engine, parser=parser)
            b = pd.eval(rhs, engine=engine, parser=parser)

            if r_idx_type != 'dt' and c_idx_type != 'dt':
                if engine == 'numexpr':
                    assert_frame_equal(a, b) 

def test_basic_frame_series_alignment(self, engine, parser):
        def testit(r_idx_type, c_idx_type, index_name):
            df = mkdf(10, 10, data_gen_f=f, r_idx_type=r_idx_type,
                      c_idx_type=c_idx_type)
            index = getattr(df, index_name)
            s = Series(np.random.randn(5), index[:5])

            if should_warn(df.index, s.index):
                with tm.assert_produces_warning(RuntimeWarning):
                    res = pd.eval('df + s', engine=engine, parser=parser)
            else:
                res = pd.eval('df + s', engine=engine, parser=parser)

            if r_idx_type == 'dt' or c_idx_type == 'dt':
                expected = df.add(s) if engine == 'numexpr' else df + s
            else:
                expected = df + s
            assert_frame_equal(res, expected)

        args = product(self.lhs_index_types, self.index_types,
                       ('index', 'columns'))
        with warnings.catch_warnings(record=True):
            warnings.simplefilter('always', RuntimeWarning)
            for r_idx_type, c_idx_type, index_name in args:
                testit(r_idx_type, c_idx_type, index_name) 

def test_basic_period_index_subscript_expression(self):
        df = mkdf(2, 2, data_gen_f=f, c_idx_type='p', r_idx_type='i')
        r = self.eval('df[df < 2 + 3]', local_dict={'df': df})
        e = df[df < 2 + 3]
        assert_frame_equal(r, e) 

def check_medium_complex_frame_alignment(self, engine, parser):
        tm.skip_if_no_ne(engine)
        args = product(self.lhs_index_types, self.index_types,
                       self.index_types, self.index_types)

        for r1, c1, r2, c2 in args:
            df = mkdf(3, 2, data_gen_f=f, r_idx_type=r1, c_idx_type=c1)
            df2 = mkdf(4, 2, data_gen_f=f, r_idx_type=r2, c_idx_type=c2)
            df3 = mkdf(5, 2, data_gen_f=f, r_idx_type=r2, c_idx_type=c2)
            res = pd.eval('df + df2 + df3', engine=engine, parser=parser)
            assert_frame_equal(res, df + df2 + df3) 

def check_basic_frame_alignment(self, engine, parser):
        tm.skip_if_no_ne(engine)
        args = product(self.lhs_index_types, self.index_types,
                       self.index_types)
        for lr_idx_type, rr_idx_type, c_idx_type in args:
            df = mkdf(10, 10, data_gen_f=f, r_idx_type=lr_idx_type,
                      c_idx_type=c_idx_type)
            df2 = mkdf(20, 10, data_gen_f=f, r_idx_type=rr_idx_type,
                       c_idx_type=c_idx_type)
            res = pd.eval('df + df2', engine=engine, parser=parser)
            assert_frame_equal(res, df + df2) 

def test_does_not_convert_mixed_integer(self):
        df = tm.makeCustomDataframe(10, 10, data_gen_f=lambda *args, **kwargs:
                                    randn(), r_idx_type='i', c_idx_type='dt')
        cols = df.columns.join(df.index, how='outer')
        joined = cols.join(df.columns)
        self.assertEqual(cols.dtype, np.dtype('O'))
        self.assertEqual(cols.dtype, joined.dtype)
        assert_array_equal(cols.values, joined.values) 

def test_nested_period_index_subscript_expression(self):
        df = mkdf(2, 2, data_gen_f=f, c_idx_type='p', r_idx_type='i')
        r = self.eval('df[df[df < 2] < 2] + df * 2', local_dict={'df': df})
        e = df[df[df < 2] < 2] + df * 2
        assert_frame_equal(r, e) 

def test_to_html_compat(self):
        df = mkdf(4, 3, data_gen_f=lambda *args: rand(), c_idx_names=False,
                  r_idx_names=False).applymap('{0:.3f}'.format).astype(float)
        out = df.to_html()
        res = self.read_html(out, attrs={'class': 'dataframe'},
                                 index_col=0)[0]
        tm.assert_frame_equal(res, df) 

def test_frame_comparison(self, engine, parser):
        args = product(self.lhs_index_types, repeat=2)
        for r_idx_type, c_idx_type in args:
            df = mkdf(10, 10, data_gen_f=f, r_idx_type=r_idx_type,
                      c_idx_type=c_idx_type)
            res = pd.eval('df < 2', engine=engine, parser=parser)
            assert_frame_equal(res, df < 2)

            df3 = DataFrame(randn(*df.shape), index=df.index,
                            columns=df.columns)
            res = pd.eval('df < df3', engine=engine, parser=parser)
            assert_frame_equal(res, df < df3) 

def test_does_not_convert_mixed_integer(self):
        df = tm.makeCustomDataframe(10, 10,
                                    data_gen_f=lambda *args, **kwargs: randn(),
                                    r_idx_type='i', c_idx_type='td')
        str(df)

        cols = df.columns.join(df.index, how='outer')
        joined = cols.join(df.columns)
        assert cols.dtype == np.dtype('O')
        assert cols.dtype == joined.dtype
        tm.assert_index_equal(cols, joined) 

def check_complex_series_frame_alignment(self, engine, parser):
        tm.skip_if_no_ne(engine)

        import random
        args = product(self.lhs_index_types, self.index_types,
                       self.index_types, self.index_types)
        n = 3
        m1 = 5
        m2 = 2 * m1

        for r1, r2, c1, c2 in args:
            index_name = random.choice(['index', 'columns'])
            obj_name = random.choice(['df', 'df2'])

            df = mkdf(m1, n, data_gen_f=f, r_idx_type=r1, c_idx_type=c1)
            df2 = mkdf(m2, n, data_gen_f=f, r_idx_type=r2, c_idx_type=c2)
            index = getattr(locals().get(obj_name), index_name)
            s = Series(np.random.randn(n), index[:n])

            if r2 == 'dt' or c2 == 'dt':
                if engine == 'numexpr':
                    expected2 = df2.add(s)
                else:
                    expected2 = df2 + s
            else:
                expected2 = df2 + s

            if r1 == 'dt' or c1 == 'dt':
                if engine == 'numexpr':
                    expected = expected2.add(df)
                else:
                    expected = expected2 + df
            else:
                expected = expected2 + df

            res = pd.eval('df2 + s + df', engine=engine, parser=parser)
            assert_equal(res.shape, expected.shape)
            assert_frame_equal(res, expected)