Python pandas.MultiIndex.from_product() Examples

def test_delitem_multiindex(self):
        midx = MultiIndex.from_product([['A', 'B'], [1, 2]])
        df = DataFrame(np.random.randn(4, 4), columns=midx)
        assert len(df.columns) == 4
        assert ('A', ) in df.columns
        assert 'A' in df.columns

        result = df['A']
        assert isinstance(result, DataFrame)
        del df['A']

        assert len(df.columns) == 2

        # A still in the levels, BUT get a KeyError if trying
        # to delete
        assert ('A', ) not in df.columns
        with pytest.raises(KeyError):
            del df[('A',)]

        # behavior of dropped/deleted MultiIndex levels changed from
        # GH 2770 to GH 19027: MultiIndex no longer '.__contains__'
        # levels which are dropped/deleted
        assert 'A' not in df.columns
        with pytest.raises(KeyError):
            del df['A'] 

def take_invalid_kwargs():
    vals = [['A', 'B'],
            [pd.Timestamp('2011-01-01'), pd.Timestamp('2011-01-02')]]
    idx = pd.MultiIndex.from_product(vals, names=['str', 'dt'])
    indices = [1, 2]

    msg = r"take\(\) got an unexpected keyword argument 'foo'"
    with pytest.raises(TypeError, match=msg):
        idx.take(indices, foo=2)

    msg = "the 'out' parameter is not supported"
    with pytest.raises(ValueError, match=msg):
        idx.take(indices, out=indices)

    msg = "the 'mode' parameter is not supported"
    with pytest.raises(ValueError, match=msg):
        idx.take(indices, mode='clip') 

def test_large_multiindex_error():
    # GH12527
    df_below_1000000 = pd.DataFrame(
        1, index=pd.MultiIndex.from_product([[1, 2], range(499999)]),
        columns=['dest'])
    with pytest.raises(KeyError):
        df_below_1000000.loc[(-1, 0), 'dest']
    with pytest.raises(KeyError):
        df_below_1000000.loc[(3, 0), 'dest']
    df_above_1000000 = pd.DataFrame(
        1, index=pd.MultiIndex.from_product([[1, 2], range(500001)]),
        columns=['dest'])
    with pytest.raises(KeyError):
        df_above_1000000.loc[(-1, 0), 'dest']
    with pytest.raises(KeyError):
        df_above_1000000.loc[(3, 0), 'dest'] 

def test_loc_multiindex_indexer_none(self):

        # GH6788
        # multi-index indexer is None (meaning take all)
        attributes = ['Attribute' + str(i) for i in range(1)]
        attribute_values = ['Value' + str(i) for i in range(5)]

        index = MultiIndex.from_product([attributes, attribute_values])
        df = 0.1 * np.random.randn(10, 1 * 5) + 0.5
        df = DataFrame(df, columns=index)
        result = df[attributes]
        tm.assert_frame_equal(result, df)

        # GH 7349
        # loc with a multi-index seems to be doing fallback
        df = DataFrame(np.arange(12).reshape(-1, 1),
                       index=MultiIndex.from_product([[1, 2, 3, 4],
                                                      [1, 2, 3]]))

        expected = df.loc[([1, 2], ), :]
        result = df.loc[[1, 2]]
        tm.assert_frame_equal(result, expected) 

def test_labels_dtypes():

    # GH 8456
    i = MultiIndex.from_tuples([('A', 1), ('A', 2)])
    assert i.codes[0].dtype == 'int8'
    assert i.codes[1].dtype == 'int8'

    i = MultiIndex.from_product([['a'], range(40)])
    assert i.codes[1].dtype == 'int8'
    i = MultiIndex.from_product([['a'], range(400)])
    assert i.codes[1].dtype == 'int16'
    i = MultiIndex.from_product([['a'], range(40000)])
    assert i.codes[1].dtype == 'int32'

    i = pd.MultiIndex.from_product([['a'], range(1000)])
    assert (i.codes[0] >= 0).all()
    assert (i.codes[1] >= 0).all() 

def test_getitem_partial_int(self):
        # GH 12416
        # with single item
        l1 = [10, 20]
        l2 = ['a', 'b']
        df = DataFrame(index=range(2),
                       columns=MultiIndex.from_product([l1, l2]))
        expected = DataFrame(index=range(2),
                             columns=l2)
        result = df[20]
        tm.assert_frame_equal(result, expected)

        # with list
        expected = DataFrame(index=range(2),
                             columns=MultiIndex.from_product([l1[1:], l2]))
        result = df[[20]]
        tm.assert_frame_equal(result, expected)

        # missing item:
        with pytest.raises(KeyError, match='1'):
            df[1]
        with pytest.raises(KeyError, match=r"'\[1\] not in index'"):
            df[[1]] 

def test_multiindex_contains_dropped(self):
        # GH 19027
        # test that dropped MultiIndex levels are not in the MultiIndex
        # despite continuing to be in the MultiIndex's levels
        idx = MultiIndex.from_product([[1, 2], [3, 4]])
        assert 2 in idx
        idx = idx.drop(2)

        # drop implementation keeps 2 in the levels
        assert 2 in idx.levels[0]
        # but it should no longer be in the index itself
        assert 2 not in idx

        # also applies to strings
        idx = MultiIndex.from_product([['a', 'b'], ['c', 'd']])
        assert 'a' in idx
        idx = idx.drop('a')
        assert 'a' in idx.levels[0]
        assert 'a' not in idx 

def test_from_product_iterator():
    # GH 18434
    first = ['foo', 'bar', 'buz']
    second = ['a', 'b', 'c']
    names = ['first', 'second']
    tuples = [('foo', 'a'), ('foo', 'b'), ('foo', 'c'), ('bar', 'a'),
              ('bar', 'b'), ('bar', 'c'), ('buz', 'a'), ('buz', 'b'),
              ('buz', 'c')]
    expected = MultiIndex.from_tuples(tuples, names=names)

    # iterator as input
    result = MultiIndex.from_product(iter([first, second]), names=names)
    tm.assert_index_equal(result, expected)

    # Invalid non-iterable input
    msg = "Input must be a list / sequence of iterables."
    with pytest.raises(TypeError, match=msg):
        MultiIndex.from_product(0) 

def test_rename_axis_mapper(self):
        # GH 19978
        mi = MultiIndex.from_product([['a', 'b', 'c'], [1, 2]],
                                     names=['ll', 'nn'])
        s = Series([i for i in range(len(mi))], index=mi)

        result = s.rename_axis(index={'ll': 'foo'})
        assert result.index.names == ['foo', 'nn']

        result = s.rename_axis(index=str.upper, axis=0)
        assert result.index.names == ['LL', 'NN']

        result = s.rename_axis(index=['foo', 'goo'])
        assert result.index.names == ['foo', 'goo']

        with pytest.raises(TypeError, match='unexpected'):
            s.rename_axis(columns='wrong') 

def test_cat_on_filtered_index(self):
        df = DataFrame(index=MultiIndex.from_product(
            [[2011, 2012], [1, 2, 3]], names=['year', 'month']))

        df = df.reset_index()
        df = df[df.month > 1]

        str_year = df.year.astype('str')
        str_month = df.month.astype('str')
        str_both = str_year.str.cat(str_month, sep=' ')

        assert str_both.loc[1] == '2011 2'

        str_multiple = str_year.str.cat([str_month, str_month], sep=' ')

        assert str_multiple.loc[1] == '2011 2 2' 

def test_from_product_iterator():
    # GH 18434
    first = ['foo', 'bar', 'buz']
    second = ['a', 'b', 'c']
    names = ['first', 'second']
    tuples = [('foo', 'a'), ('foo', 'b'), ('foo', 'c'), ('bar', 'a'),
              ('bar', 'b'), ('bar', 'c'), ('buz', 'a'), ('buz', 'b'),
              ('buz', 'c')]
    expected = MultiIndex.from_tuples(tuples, names=names)

    # iterator as input
    result = MultiIndex.from_product(iter([first, second]), names=names)
    tm.assert_index_equal(result, expected)

    # Invalid non-iterable input
    msg = "Input must be a list / sequence of iterables."
    with pytest.raises(TypeError, match=msg):
        MultiIndex.from_product(0) 

def test_delitem_multiindex(self):
        midx = MultiIndex.from_product([['A', 'B'], [1, 2]])
        df = DataFrame(np.random.randn(4, 4), columns=midx)
        assert len(df.columns) == 4
        assert ('A', ) in df.columns
        assert 'A' in df.columns

        result = df['A']
        assert isinstance(result, DataFrame)
        del df['A']

        assert len(df.columns) == 2

        # A still in the levels, BUT get a KeyError if trying
        # to delete
        assert ('A', ) not in df.columns
        with pytest.raises(KeyError):
            del df[('A',)]

        # behavior of dropped/deleted MultiIndex levels changed from
        # GH 2770 to GH 19027: MultiIndex no longer '.__contains__'
        # levels which are dropped/deleted
        assert 'A' not in df.columns
        with pytest.raises(KeyError):
            del df['A'] 

def test_large_multiindex_error():
    # GH12527
    df_below_1000000 = pd.DataFrame(
        1, index=pd.MultiIndex.from_product([[1, 2], range(499999)]),
        columns=['dest'])
    with pytest.raises(KeyError):
        df_below_1000000.loc[(-1, 0), 'dest']
    with pytest.raises(KeyError):
        df_below_1000000.loc[(3, 0), 'dest']
    df_above_1000000 = pd.DataFrame(
        1, index=pd.MultiIndex.from_product([[1, 2], range(500001)]),
        columns=['dest'])
    with pytest.raises(KeyError):
        df_above_1000000.loc[(-1, 0), 'dest']
    with pytest.raises(KeyError):
        df_above_1000000.loc[(3, 0), 'dest'] 

def take_invalid_kwargs():
    vals = [['A', 'B'],
            [pd.Timestamp('2011-01-01'), pd.Timestamp('2011-01-02')]]
    idx = pd.MultiIndex.from_product(vals, names=['str', 'dt'])
    indices = [1, 2]

    msg = r"take\(\) got an unexpected keyword argument 'foo'"
    with pytest.raises(TypeError, match=msg):
        idx.take(indices, foo=2)

    msg = "the 'out' parameter is not supported"
    with pytest.raises(ValueError, match=msg):
        idx.take(indices, out=indices)

    msg = "the 'mode' parameter is not supported"
    with pytest.raises(ValueError, match=msg):
        idx.take(indices, mode='clip') 

def test_cat_on_filtered_index(self):
        df = DataFrame(index=MultiIndex.from_product(
            [[2011, 2012], [1, 2, 3]], names=['year', 'month']))

        df = df.reset_index()
        df = df[df.month > 1]

        str_year = df.year.astype('str')
        str_month = df.month.astype('str')
        str_both = str_year.str.cat(str_month, sep=' ', join='left')

        assert str_both.loc[1] == '2011 2'

        str_multiple = str_year.str.cat([str_month, str_month],
                                        sep=' ', join='left')

        assert str_multiple.loc[1] == '2011 2 2' 

def test_labels_dtypes():

    # GH 8456
    i = MultiIndex.from_tuples([('A', 1), ('A', 2)])
    assert i.codes[0].dtype == 'int8'
    assert i.codes[1].dtype == 'int8'

    i = MultiIndex.from_product([['a'], range(40)])
    assert i.codes[1].dtype == 'int8'
    i = MultiIndex.from_product([['a'], range(400)])
    assert i.codes[1].dtype == 'int16'
    i = MultiIndex.from_product([['a'], range(40000)])
    assert i.codes[1].dtype == 'int32'

    i = pd.MultiIndex.from_product([['a'], range(1000)])
    assert (i.codes[0] >= 0).all()
    assert (i.codes[1] >= 0).all() 

def test_getitem_partial_int(self):
        # GH 12416
        # with single item
        l1 = [10, 20]
        l2 = ['a', 'b']
        df = DataFrame(index=range(2),
                       columns=MultiIndex.from_product([l1, l2]))
        expected = DataFrame(index=range(2),
                             columns=l2)
        result = df[20]
        tm.assert_frame_equal(result, expected)

        # with list
        expected = DataFrame(index=range(2),
                             columns=MultiIndex.from_product([l1[1:], l2]))
        result = df[[20]]
        tm.assert_frame_equal(result, expected)

        # missing item:
        with pytest.raises(KeyError, match='1'):
            df[1]
        with pytest.raises(KeyError, match=r"'\[1\] not in index'"):
            df[[1]] 

def test_loc_multiindex_indexer_none(self):

        # GH6788
        # multi-index indexer is None (meaning take all)
        attributes = ['Attribute' + str(i) for i in range(1)]
        attribute_values = ['Value' + str(i) for i in range(5)]

        index = MultiIndex.from_product([attributes, attribute_values])
        df = 0.1 * np.random.randn(10, 1 * 5) + 0.5
        df = DataFrame(df, columns=index)
        result = df[attributes]
        tm.assert_frame_equal(result, df)

        # GH 7349
        # loc with a multi-index seems to be doing fallback
        df = DataFrame(np.arange(12).reshape(-1, 1),
                       index=MultiIndex.from_product([[1, 2, 3, 4],
                                                      [1, 2, 3]]))

        expected = df.loc[([1, 2], ), :]
        result = df.loc[[1, 2]]
        tm.assert_frame_equal(result, expected) 

def test_to_sql_index_label_multiindex(self):
        temp_frame = DataFrame({'col1': range(4)},
                               index=MultiIndex.from_product(
                                   [('A0', 'A1'), ('B0', 'B1')]))

        # no index name, defaults to 'level_0' and 'level_1'
        sql.to_sql(temp_frame, 'test_index_label', self.conn)
        frame = sql.read_sql_query('SELECT * FROM test_index_label', self.conn)
        assert frame.columns[0] == 'level_0'
        assert frame.columns[1] == 'level_1'

        # specifying index_label
        sql.to_sql(temp_frame, 'test_index_label', self.conn,
                   if_exists='replace', index_label=['A', 'B'])
        frame = sql.read_sql_query('SELECT * FROM test_index_label', self.conn)
        assert frame.columns[:2].tolist() == ['A', 'B']

        # using the index name
        temp_frame.index.names = ['A', 'B']
        sql.to_sql(temp_frame, 'test_index_label', self.conn,
                   if_exists='replace')
        frame = sql.read_sql_query('SELECT * FROM test_index_label', self.conn)
        assert frame.columns[:2].tolist() == ['A', 'B']

        # has index name, but specifying index_label
        sql.to_sql(temp_frame, 'test_index_label', self.conn,
                   if_exists='replace', index_label=['C', 'D'])
        frame = sql.read_sql_query('SELECT * FROM test_index_label', self.conn)
        assert frame.columns[:2].tolist() == ['C', 'D']

        # wrong length of index_label
        pytest.raises(ValueError, sql.to_sql, temp_frame,
                      'test_index_label', self.conn, if_exists='replace',
                      index_label='C') 

def test_loc_getitem_series(self):
        # GH14730
        # passing a series as a key with a MultiIndex
        index = MultiIndex.from_product([[1, 2, 3], ['A', 'B', 'C']])
        x = Series(index=index, data=range(9), dtype=np.float64)
        y = Series([1, 3])
        expected = Series(
            data=[0, 1, 2, 6, 7, 8],
            index=MultiIndex.from_product([[1, 3], ['A', 'B', 'C']]),
            dtype=np.float64)
        result = x.loc[y]
        tm.assert_series_equal(result, expected)

        result = x.loc[[1, 3]]
        tm.assert_series_equal(result, expected)

        # GH15424
        y1 = Series([1, 3], index=[1, 2])
        result = x.loc[y1]
        tm.assert_series_equal(result, expected)

        empty = Series(data=[], dtype=np.float64)
        expected = Series([], index=MultiIndex(
            levels=index.levels, codes=[[], []], dtype=np.float64))
        result = x.loc[empty]
        tm.assert_series_equal(result, expected) 

def test_multiindex_slicers_datetimelike(self):

        # GH 7429
        # buggy/inconsistent behavior when slicing with datetime-like
        import datetime
        dates = [datetime.datetime(2012, 1, 1, 12, 12, 12) +
                 datetime.timedelta(days=i) for i in range(6)]
        freq = [1, 2]
        index = MultiIndex.from_product(
            [dates, freq], names=['date', 'frequency'])

        df = DataFrame(
            np.arange(6 * 2 * 4, dtype='int64').reshape(
                -1, 4), index=index, columns=list('ABCD'))

        # multi-axis slicing
        idx = pd.IndexSlice
        expected = df.iloc[[0, 2, 4], [0, 1]]
        result = df.loc[(slice(Timestamp('2012-01-01 12:12:12'),
                               Timestamp('2012-01-03 12:12:12')),
                         slice(1, 1)), slice('A', 'B')]
        tm.assert_frame_equal(result, expected)

        result = df.loc[(idx[Timestamp('2012-01-01 12:12:12'):Timestamp(
            '2012-01-03 12:12:12')], idx[1:1]), slice('A', 'B')]
        tm.assert_frame_equal(result, expected)

        result = df.loc[(slice(Timestamp('2012-01-01 12:12:12'),
                               Timestamp('2012-01-03 12:12:12')), 1),
                        slice('A', 'B')]
        tm.assert_frame_equal(result, expected)

        # with strings
        result = df.loc[(slice('2012-01-01 12:12:12', '2012-01-03 12:12:12'),
                         slice(1, 1)), slice('A', 'B')]
        tm.assert_frame_equal(result, expected)

        result = df.loc[(idx['2012-01-01 12:12:12':'2012-01-03 12:12:12'], 1),
                        idx['A', 'B']]
        tm.assert_frame_equal(result, expected) 

def test_from_product_empty_two_levels(first, second):
    names = ['A', 'B']
    result = MultiIndex.from_product([first, second], names=names)
    expected = MultiIndex(levels=[first, second],
                          codes=[[], []], names=names)
    tm.assert_index_equal(result, expected) 

def test_hash_pandas_object(self):

        for obj in [Series([1, 2, 3]),
                    Series([1.0, 1.5, 3.2]),
                    Series([1.0, 1.5, np.nan]),
                    Series([1.0, 1.5, 3.2], index=[1.5, 1.1, 3.3]),
                    Series(['a', 'b', 'c']),
                    Series(['a', np.nan, 'c']),
                    Series(['a', None, 'c']),
                    Series([True, False, True]),
                    Series(),
                    Index([1, 2, 3]),
                    Index([True, False, True]),
                    DataFrame({'x': ['a', 'b', 'c'], 'y': [1, 2, 3]}),
                    DataFrame(),
                    tm.makeMissingDataframe(),
                    tm.makeMixedDataFrame(),
                    tm.makeTimeDataFrame(),
                    tm.makeTimeSeries(),
                    tm.makeTimedeltaIndex(),
                    tm.makePeriodIndex(),
                    Series(tm.makePeriodIndex()),
                    Series(pd.date_range('20130101',
                                         periods=3, tz='US/Eastern')),
                    MultiIndex.from_product(
                        [range(5),
                         ['foo', 'bar', 'baz'],
                         pd.date_range('20130101', periods=2)]),
                    MultiIndex.from_product(
                        [pd.CategoricalIndex(list('aabc')),
                         range(3)])]:
            self.check_equal(obj)
            self.check_not_equal_with_index(obj) 

def test_hash_collisions():
    # non-smoke test that we don't get hash collisions

    index = MultiIndex.from_product([np.arange(1000), np.arange(1000)],
                                    names=['one', 'two'])
    result = index.get_indexer(index.values)
    tm.assert_numpy_array_equal(result, np.arange(
        len(index), dtype='intp'))

    for i in [0, 1, len(index) - 2, len(index) - 1]:
        result = index.get_loc(index[i])
        assert result == i 

def test_duplicate_level_names(names):
    # GH18872, GH19029
    mi = MultiIndex.from_product([[0, 1]] * 3, names=names)
    assert mi.names == names

    # With .rename()
    mi = MultiIndex.from_product([[0, 1]] * 3)
    mi = mi.rename(names)
    assert mi.names == names

    # With .rename(., level=)
    mi.rename(names[1], level=1, inplace=True)
    mi = mi.rename([names[0], names[2]], level=[0, 2])
    assert mi.names == names 

def test_numpy_repeat():
    reps = 2
    numbers = [1, 2, 3]
    names = np.array(['foo', 'bar'])

    m = MultiIndex.from_product([
        numbers, names], names=names)
    expected = MultiIndex.from_product([
        numbers, names.repeat(reps)], names=names)
    tm.assert_index_equal(np.repeat(m, reps), expected)

    msg = "the 'axis' parameter is not supported"
    with pytest.raises(ValueError, match=msg):
        np.repeat(m, reps, axis=1) 

def test_repeat():
    reps = 2
    numbers = [1, 2, 3]
    names = np.array(['foo', 'bar'])

    m = MultiIndex.from_product([
        numbers, names], names=names)
    expected = MultiIndex.from_product([
        numbers, names.repeat(reps)], names=names)
    tm.assert_index_equal(m.repeat(reps), expected) 

def test_repr_roundtrip():

    mi = MultiIndex.from_product([list('ab'), range(3)],
                                 names=['first', 'second'])
    str(mi)

    if PY3:
        tm.assert_index_equal(eval(repr(mi)), mi, exact=True)
    else:
        result = eval(repr(mi))
        # string coerces to unicode
        tm.assert_index_equal(result, mi, exact=False)
        assert mi.get_level_values('first').inferred_type == 'string'
        assert result.get_level_values('first').inferred_type == 'unicode'

    mi_u = MultiIndex.from_product(
        [list(u'ab'), range(3)], names=['first', 'second'])
    result = eval(repr(mi_u))
    tm.assert_index_equal(result, mi_u, exact=True)

    # formatting
    if PY3:
        str(mi)
    else:
        compat.text_type(mi)

    # long format
    mi = MultiIndex.from_product([list('abcdefg'), range(10)],
                                 names=['first', 'second'])

    if PY3:
        tm.assert_index_equal(eval(repr(mi)), mi, exact=True)
    else:
        result = eval(repr(mi))
        # string coerces to unicode
        tm.assert_index_equal(result, mi, exact=False)
        assert mi.get_level_values('first').inferred_type == 'string'
        assert result.get_level_values('first').inferred_type == 'unicode'

    result = eval(repr(mi_u))
    tm.assert_index_equal(result, mi_u, exact=True) 

def test_contains_top_level():
    midx = MultiIndex.from_product([['A', 'B'], [1, 2]])
    assert 'A' in midx
    assert 'A' not in midx._engine 

def test_roundtrip_pickle_with_tz():
    return

    # GH 8367
    # round-trip of timezone
    index = MultiIndex.from_product(
        [[1, 2], ['a', 'b'], date_range('20130101', periods=3,
                                        tz='US/Eastern')
         ], names=['one', 'two', 'three'])
    unpickled = tm.round_trip_pickle(index)
    assert index.equal_levels(unpickled)