Python pandas.testing.assert_frame_equal() Examples

def is_same_as(df, df_to_compare, **kwargs):
    """Asserts that two pd.DataFrames are equal.

    Args:
        df (pd.DataFrame): Any pd.DataFrame.
        df_to_compare (pd.DataFrame): A second pd.DataFrame.
        **kwargs (dict): Keyword arguments passed through to pandas' ``assert_frame_equal``.

    Returns:
        Original `df`.

    """
    try:
        tm.assert_frame_equal(df, df_to_compare, **kwargs)
    except AssertionError as exc:
        raise AssertionError("DataFrames are not equal") from exc
    return df 

def test_within_set():
    df = pd.DataFrame({'A': [1, 2, 3], 'B': ['a', 'b', 'c']})
    items = {'A': [1, 2, 3], 'B': ['a', 'b', 'c']}

    with pytest.deprecated_call():
        tm.assert_frame_equal(df, ck.within_set(df, items))
        tm.assert_frame_equal(df, dc.WithinSet(items=items)(_noop)(df))

    items.pop('A')
    with pytest.deprecated_call():
        tm.assert_frame_equal(df, ck.within_set(df, items))
        tm.assert_frame_equal(df, dc.WithinSet(items=items)(_noop)(df))

    items['A'] = [1, 2]
    with pytest.raises(AssertionError), pytest.deprecated_call():
        ck.within_set(df, items)
    with pytest.raises(AssertionError), pytest.deprecated_call():
        dc.WithinSet(items=items)(_noop)(df) 

def test_is_same_as_with_kwargs():
    df = pd.DataFrame({'A': [1, 2, 3], 'B': [1, 2, 3]})
    df_equal = pd.DataFrame({'A': [1, 2, 3], 'B': [1, 2, 3]})
    df_equal_float = pd.DataFrame({'A': [1.0, 2, 3], 'B': [1, 2, 3.0]})

    result = ck.is_same_as(df, df_equal, check_dtype=True)
    tm.assert_frame_equal(df, result)

    result = dc.IsSameAs(df_equal, check_dtype=True)(_noop)(df)
    tm.assert_frame_equal(df, result)

    with pytest.raises(AssertionError):
        ck.is_same_as(df, df_equal_float, check_dtype=True)
        dc.IsSameAs(df_equal_float, check_dtype=True)(_noop)(df)

    result = ck.is_same_as(df, df_equal_float, check_dtype=False)
    tm.assert_frame_equal(df, result)

    result = dc.IsSameAs(df_equal_float, check_dtype=False)(_noop)(df)  # todo see why this fails
    tm.assert_frame_equal(df, result) 

def test_get_without_transform(mock):
    api_call_result = {'query_0': [{'balance': 212664.33000000002,
                                    'datetime': '2019-05-23T00:00:00Z'},
                                   {'balance': 212664.33000000002,
                                    'datetime': '2019-05-24T00:00:00Z'},
                                   {'balance': 212664.33000000002,
                                    'datetime': '2019-05-25T00:00:00Z'}]}

    # The value is passed by refference, that's why deepcopy is used for
    # expected
    mock.return_value = TestResponse(
        status_code=200, data=deepcopy(api_call_result))

    res = san.get(
        "historical_balance/santiment",
        address="0x1f3df0b8390bb8e9e322972c5e75583e87608ec2",
        from_date="2019-05-23",
        to_date="2019-05-26",
        interval="1d"
    )
    expected_df = convert_to_datetime_idx_df(api_call_result['query_0'])
    pdt.assert_frame_equal(res, expected_df, check_dtype=False) 

def test_get(mock):
    api_call_result = {'query_0': [{'balance': 212664.33000000002,
                                    'datetime': '2019-05-23T00:00:00Z'},
                                   {'balance': 212664.33000000002,
                                    'datetime': '2019-05-24T00:00:00Z'},
                                   {'balance': 212664.33000000002,
                                    'datetime': '2019-05-25T00:00:00Z'}]}
    mock.return_value = TestResponse(
        status_code=200, data=deepcopy(api_call_result))

    res = san.get(
        "historical_balance/santiment",
        address="0x1f3df0b8390bb8e9e322972c5e75583e87608ec2",
        from_date="2019-05-23",
        to_date="2019-05-26",
        interval="1d"
    )
    expected_df = convert_to_datetime_idx_df(api_call_result['query_0'])
    pdt.assert_frame_equal(res, expected_df, check_dtype=False) 

def test_comparison_plot_inputs(input_results, expected_source_dfs, expected_plot_info):
    x_padding = 0.0
    num_bins = 10

    res_source_dfs, res_plot_info = test_module.comparison_plot_inputs(
        results=input_results[0],
        x_padding=x_padding,
        num_bins=num_bins,
        color_dict=None,
        fig_height=None,
    )

    assert res_plot_info == expected_plot_info

    assert res_source_dfs.keys() == expected_source_dfs.keys()
    for group, res_dict in res_source_dfs.items():
        for param, res_df in res_dict.items():
            exp_df = expected_source_dfs[group][param]
            pdt.assert_frame_equal(res_df, exp_df, check_like=True) 

def test_monotonic_decreasing_strict():
    df = pd.DataFrame([3, 2, 1])
    tm.assert_frame_equal(df, ck.is_monotonic(df, increasing=False,
                                              strict=True))
    result = dc.IsMonotonic(increasing=False, strict=True)(_add_n)(df)
    tm.assert_frame_equal(result, df + 1)

    df = pd.DataFrame([2, 2, 1])
    with pytest.raises(AssertionError):
        ck.is_monotonic(df, increasing=False, strict=True)
    with pytest.raises(AssertionError):
        dc.IsMonotonic(increasing=False, strict=True)(_add_n)(df)

    df = pd.DataFrame([1, 2, 3])
    with pytest.raises(AssertionError):
        ck.is_monotonic(df, increasing=False, strict=True)
    with pytest.raises(AssertionError):
        dc.IsMonotonic(increasing=False, strict=True)(_add_n)(df) 

def test_pandas_from_arrow():
    arr = pa.array(["a", "b", "c"], pa.string())

    expected_series_woutname = pd.Series(fr.FletcherChunkedArray(arr))
    pdt.assert_series_equal(expected_series_woutname, fr.pandas_from_arrow(arr))

    expected_series_woutname = pd.Series(fr.FletcherContinuousArray(arr))
    pdt.assert_series_equal(
        expected_series_woutname, fr.pandas_from_arrow(arr, continuous=True)
    )

    rb = pa.RecordBatch.from_arrays([arr], ["column"])
    expected_df = pd.DataFrame({"column": fr.FletcherChunkedArray(arr)})
    table = pa.Table.from_arrays([arr], ["column"])
    pdt.assert_frame_equal(expected_df, fr.pandas_from_arrow(rb))
    pdt.assert_frame_equal(expected_df, fr.pandas_from_arrow(table))

    expected_df = pd.DataFrame({"column": fr.FletcherContinuousArray(arr)})
    table = pa.Table.from_arrays([arr], ["column"])
    pdt.assert_frame_equal(expected_df, fr.pandas_from_arrow(rb, continuous=True))
    pdt.assert_frame_equal(expected_df, fr.pandas_from_arrow(table, continuous=True)) 

def test_monotonic_decreasing():
    df = pd.DataFrame([2, 2, 1])
    tm.assert_frame_equal(df, ck.is_monotonic(df, increasing=False))
    result = dc.IsMonotonic(increasing=False)(_add_n)(df)
    tm.assert_frame_equal(result, df + 1)

    df = pd.DataFrame([1, 2, 1])
    with pytest.raises(AssertionError):
        ck.is_monotonic(df, increasing=False)
    with pytest.raises(AssertionError):
        dc.IsMonotonic(increasing=False)(_add_n)(df)

    df = pd.DataFrame([1, 2, 3])
    with pytest.raises(AssertionError):
        ck.is_monotonic(df, increasing=False)
    with pytest.raises(AssertionError):
        dc.IsMonotonic(increasing=False)(_add_n)(df) 

def test_monotonic_increasing_strict():
    df = pd.DataFrame([1, 2, 3])
    tm.assert_frame_equal(df, ck.is_monotonic(df, increasing=True, strict=True))
    result = dc.IsMonotonic(increasing=True, strict=True)(_add_n)(df)
    tm.assert_frame_equal(result, df + 1)

    df = pd.DataFrame([1, 2, 2])
    with pytest.raises(AssertionError):
        ck.is_monotonic(df, increasing=True, strict=True)
    with pytest.raises(AssertionError):
        dc.IsMonotonic(increasing=True, strict=True)(_add_n)(df)

    df = pd.DataFrame([3, 2, 1])
    with pytest.raises(AssertionError):
        ck.is_monotonic(df, increasing=True, strict=True)
    with pytest.raises(AssertionError):
        dc.IsMonotonic(increasing=True, strict=True)(_add_n)(df) 

def test_monotonic_increasing_lax():
    df = pd.DataFrame([1, 2, 2])
    tm.assert_frame_equal(df, ck.is_monotonic(df, increasing=True))
    result = dc.IsMonotonic(increasing=True)(_add_n)(df)
    tm.assert_frame_equal(result, df + 1)

    df = pd.DataFrame([1, 2, 1])
    with pytest.raises(AssertionError):
        ck.is_monotonic(df, increasing=True)
    with pytest.raises(AssertionError):
        dc.IsMonotonic(increasing=True)(_add_n)(df)

    df = pd.DataFrame([3, 2, 1])
    with pytest.raises(AssertionError):
        ck.is_monotonic(df, increasing=True)
    with pytest.raises(AssertionError):
        dc.IsMonotonic(increasing=True)(_add_n)(df) 

def test_iterative(self):
        """Test the iterative behaviour."""

        # SINGLE STEP
        index_class = Full()
        pairs = index_class.index((self.a, self.b))
        pairs = pd.DataFrame(index=pairs).sort_index()

        # MULTI STEP
        index_class = Full()

        pairs1 = index_class.index((self.a[0:50], self.b))
        pairs2 = index_class.index((self.a[50:100], self.b))

        pairs_split = pairs1.append(pairs2)
        pairs_split = pd.DataFrame(index=pairs_split).sort_index()

        pdt.assert_frame_equal(pairs, pairs_split)
        # note possible to sort MultiIndex, so made a frame out of it. 

def test_freq_nan(self, missing_value):

        # data
        array_repeated = np.repeat(np.arange(10, dtype=np.float64), 10)
        array_repeated[90:] = np.nan
        array_tiled = np.tile(np.arange(20, dtype=np.float64), 5)

        # convert to pandas data
        A = DataFrame({'col': array_repeated})
        B = DataFrame({'col': array_tiled})
        ix = MultiIndex.from_arrays([A.index.values, B.index.values])

        # the part to test
        from recordlinkage.compare import Frequency

        comp = recordlinkage.Compare()
        comp.add(Frequency(left_on='col', missing_value=missing_value))
        result = comp.compute(ix, A, B)

        expected_np = np.ones((100, )) / 10
        expected_np[90:] = missing_value
        expected = DataFrame(expected_np, index=ix)
        pdt.assert_frame_equal(result, expected) 

def test_indexing_types(self):
        # test the two types of indexing

        # this test needs improvement

        A = DataFrame({'col': ['abc', 'abc', 'abc', 'abc', 'abc']})
        B = DataFrame({'col': ['abc', 'abc', 'abc', 'abc', 'abc']})
        B_reversed = B[::-1].copy()
        ix = MultiIndex.from_arrays([np.arange(5), np.arange(5)])

        # test with label indexing type
        comp_label = recordlinkage.Compare(indexing_type='label')
        comp_label.exact('col', 'col')
        result_label = comp_label.compute(ix, A, B_reversed)

        # test with position indexing type
        comp_position = recordlinkage.Compare(indexing_type='position')
        comp_position.exact('col', 'col')
        result_position = comp_position.compute(ix, A, B_reversed)

        assert (result_position.values == 1).all(axis=0)

        pdt.assert_frame_equal(result_label, result_position) 

def test_parallel_comparing(self):

        # use single job
        comp = recordlinkage.Compare(n_jobs=1)
        comp.exact('given_name', 'given_name', label='my_feature_label')
        result_single = comp.compute(self.index_AB, self.A, self.B)
        result_single.sort_index(inplace=True)

        # use two jobs
        comp = recordlinkage.Compare(n_jobs=2)
        comp.exact('given_name', 'given_name', label='my_feature_label')
        result_2processes = comp.compute(self.index_AB, self.A, self.B)
        result_2processes.sort_index(inplace=True)

        # use two jobs
        comp = recordlinkage.Compare(n_jobs=4)
        comp.exact('given_name', 'given_name', label='my_feature_label')
        result_4processes = comp.compute(self.index_AB, self.A, self.B)
        result_4processes.sort_index(inplace=True)

        # compare results
        pdt.assert_frame_equal(result_single, result_2processes)
        pdt.assert_frame_equal(result_single, result_4processes) 

def test_parallel_comparing_api(self):

        # use single job
        comp = recordlinkage.Compare(n_jobs=1)
        comp.exact('given_name', 'given_name', label='my_feature_label')
        result_single = comp.compute(self.index_AB, self.A, self.B)
        result_single.sort_index(inplace=True)

        # use two jobs
        comp = recordlinkage.Compare(n_jobs=2)
        comp.exact('given_name', 'given_name', label='my_feature_label')
        result_2processes = comp.compute(self.index_AB, self.A, self.B)
        result_2processes.sort_index(inplace=True)

        # compare results
        pdt.assert_frame_equal(result_single, result_2processes) 

def test_compare_custom_vectorized_dedup(self):

        A = DataFrame({'col': ['abc', 'abc', 'abc', 'abc', 'abc']})
        ix = MultiIndex.from_arrays([[0, 1, 2, 3, 4], [1, 2, 3, 4, 0]])

        # test without label
        comp = recordlinkage.Compare()
        comp.compare_vectorized(lambda s1, s2: np.ones(len(s1), dtype=np.int),
                                'col', 'col')
        result = comp.compute(ix, A)
        expected = DataFrame([1, 1, 1, 1, 1], index=ix)
        pdt.assert_frame_equal(result, expected)

        # test with label
        comp = recordlinkage.Compare()
        comp.compare_vectorized(
            lambda s1, s2: np.ones(len(s1), dtype=np.int),
            'col',
            'col',
            label='test')
        result = comp.compute(ix, A)
        expected = DataFrame([1, 1, 1, 1, 1], index=ix, columns=['test'])
        pdt.assert_frame_equal(result, expected) 

def test_has_set_within_vals():
    df = pd.DataFrame({'A': [1, 2, 3], 'B': ['a', 'b', 'c']})

    items = {'A': [1, 2, 3], 'B': ['a', 'b', 'c']}
    tm.assert_frame_equal(df, ck.has_set_within_vals(df, items))
    tm.assert_frame_equal(df, dc.HasSetWithinVals(items=items)(_noop)(df))

    items = {'A': [1, 2], 'B': ['a', 'b']}
    tm.assert_frame_equal(df, ck.has_set_within_vals(df, items))
    tm.assert_frame_equal(df, dc.HasSetWithinVals(items=items)(_noop)(df))

    items = {'A': [1, 2, 4], 'B': ['a', 'b', 'd']}
    with pytest.raises(AssertionError):
        ck.has_set_within_vals(df, items)
    with pytest.raises(AssertionError):
        dc.HasSetWithinVals(items=items)(_noop)(df) 

def test_compare_custom_instance_type(self):

        A = DataFrame({'col': ['abc', 'abc', 'abc', 'abc', 'abc']})
        B = DataFrame({'col': ['abc', 'abd', 'abc', 'abc', '123']})
        ix = MultiIndex.from_arrays([A.index.values, B.index.values])

        def call(s1, s2):

            # this should raise on incorrect types
            assert isinstance(s1, np.ndarray)
            assert isinstance(s2, np.ndarray)

            return np.ones(len(s1), dtype=np.int)

        comp = recordlinkage.Compare()
        comp.compare_vectorized(lambda s1, s2: np.ones(len(s1), dtype=np.int),
                                'col', 'col')
        result = comp.compute(ix, A, B)
        expected = DataFrame([1, 1, 1, 1, 1], index=ix)
        pdt.assert_frame_equal(result, expected) 

def test_merge_datasets(
    dataset,
    evaluation_dataset,
    store_factory,
    store_session_factory,
    frozen_time,
    bound_merge_datasets,
):
    # In the __pipeline case, we also need to check that the write path is
    # correct, the tests for it are much larger.
    df_list = bound_merge_datasets(
        left_dataset_uuid=dataset.uuid,
        right_dataset_uuid=evaluation_dataset.uuid,
        store=store_session_factory,
        merge_tasks=MERGE_TASKS,
        match_how="prefix",
    )
    df_list = [mp.data for mp in df_list]

    # Two partitions
    assert len(df_list) == 2
    assert len(df_list[1]) == 1
    assert len(df_list[0]) == 1
    # By using values() this test is agnostic to the used key, which is
    # currently not of any importance
    pdt.assert_frame_equal(
        list(df_list[0].values())[0],
        MERGE_EXP_CL1,
        check_like=True,
        check_dtype=False,
        check_categorical=False,
    )
    pdt.assert_frame_equal(
        list(df_list[1].values())[0],
        MERGE_EXP_CL2,
        check_like=True,
        check_dtype=False,
        check_categorical=False,
    ) 

def test_within_range():
    df = pd.DataFrame({'A': [-1, 0, 1]})
    items = {'A': (-1, 1)}

    with pytest.deprecated_call():
        tm.assert_frame_equal(df, ck.within_range(df, items))
        tm.assert_frame_equal(df, dc.WithinRange(items)(_noop)(df))

    items['A'] = (0, 1)
    with pytest.raises(AssertionError), pytest.deprecated_call():
        ck.within_range(df, items)
    with pytest.raises(AssertionError), pytest.deprecated_call():
        dc.WithinRange(items)(_noop)(df) 

def test_read_dataset_as_dataframes_columns_primary_index_only(
    store_factory, bound_load_dataframes, metadata_version
):
    table_name = SINGLE_TABLE

    def _f(b_c):
        b, c = b_c
        df = pd.DataFrame({"a": [1, 1], "b": [b, b], "c": c, "d": [b, b + 1]})
        return {"label": str(c), "data": [(table_name, df)]}

    in_partitions = [_f([1, 100])]
    dataset_uuid = "partitioned_uuid"

    store_dataframes_as_dataset(
        dfs=in_partitions,
        store=store_factory,
        dataset_uuid=dataset_uuid,
        metadata_version=metadata_version,
        partition_on=["a", "b"],
    )
    result = bound_load_dataframes(
        dataset_uuid=dataset_uuid, store=store_factory, columns={table_name: ["a", "b"]}
    )
    probe = result[0]

    if isinstance(probe, MetaPartition):
        result_dfs = [mp.data[table_name] for mp in result]
    elif isinstance(probe, dict):
        result_dfs = [mp[table_name] for mp in result]
    else:
        result_dfs = result
    result_df = pd.concat(result_dfs).reset_index(drop=True)

    expected_df = pd.DataFrame({"a": [1, 1], "b": [1, 1]})
    pdt.assert_frame_equal(expected_df, result_df, check_like=True) 

def test_has_vals_within_range():
    df = pd.DataFrame({'A': [-1, 0, 1]})
    items = {'A': (-1, 1)}
    tm.assert_frame_equal(df, ck.has_vals_within_range(df, items))
    tm.assert_frame_equal(df, dc.HasValsWithinRange(items)(_noop)(df))

    items['A'] = (0, 1)
    with pytest.raises(AssertionError):
        ck.has_vals_within_range(df, items)
    with pytest.raises(AssertionError):
        dc.HasValsWithinRange(items)(_noop)(df) 

def test_has_vals_within_set():
    df = pd.DataFrame({'A': [1, 2, 3], 'B': ['a', 'b', 'c']})
    items = {'A': [1, 2, 3], 'B': ['a', 'b', 'c']}
    tm.assert_frame_equal(df, ck.has_vals_within_set(df, items))
    tm.assert_frame_equal(df, dc.HasValsWithinSet(items=items)(_noop)(df))

    items.pop('A')
    tm.assert_frame_equal(df, ck.has_vals_within_set(df, items))
    tm.assert_frame_equal(df, dc.HasValsWithinSet(items=items)(_noop)(df))

    items['A'] = [1, 2]
    with pytest.raises(AssertionError):
        ck.has_vals_within_set(df, items)
    with pytest.raises(AssertionError):
        dc.HasValsWithinSet(items=items)(_noop)(df) 

def test_has_no_neg_infs():
    df = pd.DataFrame(np.random.randn(5, 3))
    result = ck.has_no_neg_infs(df)
    tm.assert_frame_equal(df, result)

    result = dc.HasNoNegInfs()(_add_n)(df, 2)
    tm.assert_frame_equal(result, df + 2)
    result = dc.HasNoNegInfs()(_add_n)(df, n=2)
    tm.assert_frame_equal(result, df + 2) 

def test_monotonic_items():
    df = pd.DataFrame({'A': [1, 2, 3], 'B': [3, 2, 3]})
    tm.assert_frame_equal(df, ck.is_monotonic(df, items={'A': (True, True)}))
    tm.assert_frame_equal(dc.IsMonotonic(items={'A': (True, True)}, strict=True)(_add_n)(
        df), df + 1) 

def test_monotonic_either():
    df = pd.DataFrame({'A': [1, 2, 2], 'B': [3, 2, 2]})
    tm.assert_frame_equal(df, ck.is_monotonic(df))
    result = dc.IsMonotonic()(_add_n)(df)
    tm.assert_frame_equal(result, df + 1)

    df = pd.DataFrame({'A': [1, 2, 3], 'B': [1, 2, 1]})
    with pytest.raises(AssertionError):
        ck.is_monotonic(df)
    with pytest.raises(AssertionError):
        dc.IsMonotonic()(_add_n)(df) 

def test_has_unique_index():
    df = pd.DataFrame([1, 2, 3], index=['a', 'b', 'c'])
    tm.assert_frame_equal(df, ck.has_unique_index(df))
    result = dc.HasUniqueIndex()(_add_n)(df)
    tm.assert_frame_equal(result, df + 1)

    with pytest.raises(AssertionError):
        ck.has_unique_index(df.reindex(['a', 'a', 'b']))
    with pytest.raises(AssertionError):
        dc.HasUniqueIndex()(_add_n)(df.reindex(['a', 'a', 'b'])) 

def test_unique():
    df = pd.DataFrame([[1, 2, 3], ['a', 'b', 'c']])
    tm.assert_frame_equal(df, ck.unique(df))
    result = dc.Unique()(_noop)(df)
    tm.assert_frame_equal(result, df)

    df = pd.DataFrame([[1, 2, 3], [1, 'b', 'c']])
    with pytest.raises(AssertionError):
        ck.unique(df)
    with pytest.raises(AssertionError):
        dc.Unique()(_noop)(df) 

def test_is_shape():
    shape = 10, 2
    ig_0 = -1, 2
    ig_1 = 10, -1
    ig_2 = None, 2
    ig_3 = 10, None
    shapes = [shape, ig_0, ig_1, ig_2, ig_3]
    df = pd.DataFrame(np.random.randn(*shape))
    for shp in shapes:
        tm.assert_frame_equal(df, ck.is_shape(df, shp))
    for shp in shapes:
        result = dc.IsShape(shape=shp)(_add_n)(df, 2)
        tm.assert_frame_equal(result, df + 2)

        result = dc.IsShape(shape=shp)(_noop)(df)
        tm.assert_frame_equal(result, df)

        result = dc.IsShape(shp)(_add_n)(df, 3)  # *arg test
        tm.assert_frame_equal(result, df + 3)

        result = dc.IsShape(shp, enabled=False)(_add_n)(df, 4)  # enabled test
        tm.assert_frame_equal(result, df + 4)

    with pytest.raises(AssertionError):
        ck.is_shape(df, (9, 2))
        dc.IsShape((9, 2))(_add_n)(df)
    with pytest.raises(TypeError):
        dc.IsShape(shape=(9, 2), cheese=True)(_add_n)(df)  # bad dc param check