diff --git a/docs/user_guide/encoding/RareLabelEncoder.rst b/docs/user_guide/encoding/RareLabelEncoder.rst index c19719c66..f9173395a 100644 --- a/docs/user_guide/encoding/RareLabelEncoder.rst +++ b/docs/user_guide/encoding/RareLabelEncoder.rst @@ -179,11 +179,12 @@ In the following output, we see the number of observations per category: .. code:: python + var_A A 10 B 10 C 2 D 1 - Name: var_A, dtype: int64 + Name: count, dtype: int64 Now, we group categories only for variables with more than 3 unique categories: @@ -216,10 +217,43 @@ a new category called `Rare`: .. code:: python + var_A A 10 B 10 Rare 3 - Name: var_A, dtype: int64 + Name: count, dtype: int64 + +With polars +----------- + +:class:`RareLabelEncoder()` works the same way with polars dataframes: + +.. code:: python + + import polars as pl + from feature_engine.encoding import RareLabelEncoder + + data = {'var_A': ['A'] * 10 + ['B'] * 10 + ['C'] * 2 + ['D'] * 1} + data = pl.DataFrame(data) + + rare_encoder = RareLabelEncoder(tol=0.05, n_categories=3, max_n_categories=2) + Xt = rare_encoder.fit_transform(data) + Xt['var_A'].value_counts().sort('var_A') + +We see the same grouping as with the pandas dataframe: + +.. code:: text + + shape: (3, 2) + ┌───────┬───────┐ + │ var_A ┆ count │ + │ --- ┆ --- │ + │ str ┆ u32 │ + ╞═══════╪═══════╡ + │ A ┆ 10 │ + │ B ┆ 10 │ + │ Rare ┆ 3 │ + └───────┴───────┘ Considerations -------------- diff --git a/feature_engine/encoding/rare_label.py b/feature_engine/encoding/rare_label.py index 2bbd2bf73..720bf3dc9 100644 --- a/feature_engine/encoding/rare_label.py +++ b/feature_engine/encoding/rare_label.py @@ -4,8 +4,8 @@ import warnings from typing import List, Optional, Union -import numpy as np -import pandas as pd +import narwhals as nw +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._check_init_parameters.check_init_input_params import ( _check_return_empty_is_bool, @@ -137,6 +137,28 @@ class RareLabelEncoder(CategoricalMethodsMixin, CategoricalInitMixinNA): 3 4 b 4 5 b 5 6 Rare + + With polars + + >>> import polars as pl + >>> from feature_engine.encoding import RareLabelEncoder + >>> X = pl.DataFrame(dict(x1 = [1,2,3,4,5,6], x2 = ["b", "b", "b", "b", "b", "a"])) + >>> rle = RareLabelEncoder(n_categories = 1, tol=0.2) + >>> rle.fit(X) + >>> rle.transform(X) + shape: (6, 2) + ┌─────┬──────┐ + │ x1 ┆ x2 │ + │ --- ┆ --- │ + │ i64 ┆ str │ + ╞═════╪══════╡ + │ 1 ┆ b │ + │ 2 ┆ b │ + │ 3 ┆ b │ + │ 4 ┆ b │ + │ 5 ┆ b │ + │ 6 ┆ Rare │ + └─────┴──────┘ """ def __init__( @@ -173,7 +195,7 @@ def __init__( if not isinstance(replace_with, (str, int, float)): raise ValueError( - "replace_with can should be a string, integer or float. " + "replace_with should be a string, integer or float. " f"Got {replace_with} instead." ) @@ -186,13 +208,13 @@ def __init__( self.replace_with = replace_with self.return_empty = return_empty - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ Learn the frequent categories for each variable. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The training input samples. Can be the entire dataframe, not just selected variables @@ -200,26 +222,34 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): y is not required. You can pass y or None. """ - X = check_X(X) + nw_X = check_X(X) variables_ = self._check_or_select_variables(X) self._check_na(X, variables_) self.encoder_dict_ = {} + # n_unique() counts a null as its own category, matching pandas' + # plain unique() (used for the cardinality check below), unlike + # pandas' nunique() which drops nulls by default. for var in variables_: - if len(X[var].unique()) > self.n_categories: + col = nw_X.get_column(var) + + if col.n_unique() > self.n_categories: - # if the variable has more than the indicated number of categories - # the encoder will learn the most frequent categories - t = X[var].value_counts(normalize=True) + # learn the most frequent categories, dropping nulls and sorting + # by frequency like pandas' value_counts() + counts = col.drop_nulls().value_counts(sort=True, normalize=True) + cat_col, freq_col = counts.columns # non-rare labels: - freq_idx = t[t >= self.tol].index + freq_idx = counts.filter( + counts.get_column(freq_col) >= self.tol + ).get_column(cat_col).to_list() if self.max_n_categories: - self.encoder_dict_[var] = list(freq_idx[: self.max_n_categories]) + self.encoder_dict_[var] = freq_idx[: self.max_n_categories] else: - self.encoder_dict_[var] = list(freq_idx) + self.encoder_dict_[var] = freq_idx else: # if the total number of categories is smaller than the indicated @@ -229,56 +259,64 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): "indicated in n_categories. Thus, all categories will be " "considered frequent".format(var) ) - self.encoder_dict_[var] = list(X[var].unique()) + self.encoder_dict_[var] = col.unique(maintain_order=True).to_list() self.variables_ = variables_ self._get_feature_names_in(X) return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Group infrequent categories. Replace infrequent categories by the string 'Rare' or any other name provided by the user. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The input samples. Returns ------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The dataframe where rare categories have been grouped. """ - X = self._check_transform_input_and_state(X) + nw_X = self._check_transform_input_and_state(X) # check if dataset contains na if self.missing_values == "raise": _check_contains_na(X, self.variables_, error_msg="optional") - with_nan = [] - else: - with_nan = [np.nan] + # pandas categorical columns need replace_with added to their categories; + # work on a copy so the user's dataframe is not changed + if nw_X.implementation.is_pandas(): + native_X = nw_X.to_native().copy() + for feature in self.variables_: + if native_X[feature].dtype == "category": + native_X[feature] = native_X[feature].cat.add_categories( + self.replace_with + ) + nw_X = nw.from_native(native_X, eager_only=True) + + # narwhals resolves the dtype when mixing each column with replace_with; + # series from get_column() also work with pandas integer column names + new_columns = [] for feature in self.variables_: - # Setting an item of incompatible dtype is deprecated - # and will raise an error in a future version of pandas - if self.ignore_format is True and isinstance(self.replace_with, str): - num_vars = list( - X[self.variables_].select_dtypes(include="number").columns + col = nw_X.get_column(feature) + keep = col.is_in(self.encoder_dict_[feature]) + if self.missing_values == "ignore": + keep = keep | col.is_null() + new_columns.append( + nw.when(keep).then(col).otherwise(nw.lit(self.replace_with)).alias( + feature ) - X[num_vars] = X[num_vars].astype("O") - - if X[feature].dtype == "category": - X[feature] = X[feature].cat.add_categories(self.replace_with) - - X.loc[~X[feature].isin(self.encoder_dict_[feature] + with_nan), feature] = ( - self.replace_with ) + X = nw_X.with_columns(*new_columns).to_native() + return X - def inverse_transform(self, X: pd.DataFrame): + def inverse_transform(self, X: IntoDataFrame): """inverse_transform is not implemented for this transformer.""" raise NotImplementedError( "inverse_transform is not implemented for this transformer." diff --git a/tests/test_encoding/test_rare_label_encoder.py b/tests/test_encoding/test_rare_label_encoder.py index 9594e1cc3..7b2858d15 100644 --- a/tests/test_encoding/test_rare_label_encoder.py +++ b/tests/test_encoding/test_rare_label_encoder.py @@ -1,63 +1,115 @@ +import re from collections import Counter -import numpy as np import pandas as pd +import polars as pl import pytest from feature_engine.encoding import RareLabelEncoder +from tests.backend_helpers import frame_to_dict + +MSG_NA = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer or set the parameter " + "`missing_values='ignore'` when initialising this transformer." +) + +FREQUENT_CATEGORIES = { + "var_A": ["B", "D", "A", "G", "C"], + "var_B": ["A", "D", "B", "G", "C"], + "var_C": ["C", "D", "B", "G", "A"], +} + +# data_enc_big after grouping categories E and F as "Rare" +ENC_BIG_RARE = { + "var_A": ["A"] * 6 + ["B"] * 10 + ["C"] * 4 + ["D"] * 10 + ["Rare"] * 4 + ["G"] * 6, + "var_B": ["A"] * 10 + ["B"] * 6 + ["C"] * 4 + ["D"] * 10 + ["Rare"] * 4 + ["G"] * 6, + "var_C": ["A"] * 4 + ["B"] * 6 + ["C"] * 10 + ["D"] * 10 + ["Rare"] * 4 + ["G"] * 6, +} + + +# init parameters +@pytest.mark.parametrize("tol", ["hello", [0.5], -1, 1.5, None]) +def test_error_if_tol_not_between_0_and_1(tol): + msg = f"tol takes values between 0 and 1. Got {tol} instead." + with pytest.raises(ValueError, match=re.escape(msg)): + RareLabelEncoder(tol=tol) + + +@pytest.mark.parametrize("n_cat", ["hello", [0.5], -0.1, 1.5, -1, None]) +def test_error_if_n_categories_not_int(n_cat): + msg = f"n_categories takes only positive integer numbers. Got {n_cat} instead." + with pytest.raises(ValueError, match=re.escape(msg)): + RareLabelEncoder(n_categories=n_cat) -def test_defo_params_plus_automatically_find_variables(df_enc_big): - # test case 1: defo params, automatically select variables +@pytest.mark.parametrize("max_n_categories", ["hello", ["auto"], -1, 0.5]) +def test_raises_error_when_max_n_categories_not_allowed(max_n_categories): + msg = ( + "max_n_categories takes only positive integer numbers. " + f"Got {max_n_categories} instead." + ) + with pytest.raises(ValueError, match=re.escape(msg)): + RareLabelEncoder(max_n_categories=max_n_categories) + + +@pytest.mark.parametrize("replace_with", [set("hello"), ["auto"], None]) +def test_error_if_replace_with_not_string(replace_with): + msg = ( + "replace_with should be a string, integer or float. " + f"Got {replace_with} instead." + ) + with pytest.raises(ValueError, match=re.escape(msg)): + RareLabelEncoder(replace_with=replace_with) + + +@pytest.mark.parametrize( + "tol, n_categories, max_n_categories, replace_with, missing_values, ignore_format", + [ + (0.05, 10, None, "Rare", "raise", False), + (0, 0, 3, "Other", "ignore", True), + (1, 5, 0, 1, "raise", True), + (0.5, 2, 10, 0.5, "ignore", False), + ], +) +def test_init_param_assignment( + tol, n_categories, max_n_categories, replace_with, missing_values, ignore_format +): encoder = RareLabelEncoder( - tol=0.06, n_categories=5, variables=None, replace_with="Rare" + tol=tol, + n_categories=n_categories, + max_n_categories=max_n_categories, + replace_with=replace_with, + missing_values=missing_values, + ignore_format=ignore_format, ) - X = encoder.fit_transform(df_enc_big) + assert encoder.tol == tol + assert encoder.n_categories == n_categories + assert encoder.max_n_categories == max_n_categories + assert encoder.replace_with == replace_with + assert encoder.missing_values == missing_values + assert encoder.ignore_format is ignore_format - # expected output - df = { - "var_A": ["A"] * 6 - + ["B"] * 10 - + ["C"] * 4 - + ["D"] * 10 - + ["Rare"] * 4 - + ["G"] * 6, - "var_B": ["A"] * 10 - + ["B"] * 6 - + ["C"] * 4 - + ["D"] * 10 - + ["Rare"] * 4 - + ["G"] * 6, - "var_C": ["A"] * 4 - + ["B"] * 6 - + ["C"] * 10 - + ["D"] * 10 - + ["Rare"] * 4 - + ["G"] * 6, - } - df = pd.DataFrame(df) - frequenc_cat = { - "var_A": ["B", "D", "A", "G", "C"], - "var_B": ["A", "D", "B", "G", "C"], - "var_C": ["C", "D", "B", "G", "A"], - } +# fit and transform +def test_defo_params_plus_automatically_find_variables(make_df, data_enc_big): + encoder = RareLabelEncoder( + tol=0.06, n_categories=5, variables=None, replace_with="Rare" + ) + X = encoder.fit_transform(make_df(data_enc_big)) - # test init params - assert encoder.tol == 0.06 - assert encoder.n_categories == 5 - assert encoder.replace_with == "Rare" - assert encoder.variables is None # test fit attr assert encoder.variables_ == ["var_A", "var_B", "var_C"] assert encoder.n_features_in_ == 3 - assert encoder.encoder_dict_ == frequenc_cat + assert encoder.encoder_dict_ == FREQUENT_CATEGORIES # test transform output - pd.testing.assert_frame_equal(X, df) + assert isinstance(X, make_df) + assert frame_to_dict(X) == ENC_BIG_RARE -def test_when_varnames_are_numbers(df_enc_big): - input_df = df_enc_big.copy() +def test_when_varnames_are_numbers(data_enc_big): + # integer column names are pandas-only + input_df = pd.DataFrame(data_enc_big) input_df.columns = [1, 2, 3] encoder = RareLabelEncoder( @@ -66,73 +118,60 @@ def test_when_varnames_are_numbers(df_enc_big): X = encoder.fit_transform(input_df) # expected output - df = { - 1: ["A"] * 6 + ["B"] * 10 + ["C"] * 4 + ["D"] * 10 + ["Rare"] * 4 + ["G"] * 6, - 2: ["A"] * 10 + ["B"] * 6 + ["C"] * 4 + ["D"] * 10 + ["Rare"] * 4 + ["G"] * 6, - 3: ["A"] * 4 + ["B"] * 6 + ["C"] * 10 + ["D"] * 10 + ["Rare"] * 4 + ["G"] * 6, - } - df = pd.DataFrame(df) - - frequenc_cat = { - 1: ["B", "D", "A", "G", "C"], - 2: ["A", "D", "B", "G", "C"], - 3: ["C", "D", "B", "G", "A"], - } + df = pd.DataFrame( + { + 1: ENC_BIG_RARE["var_A"], + 2: ENC_BIG_RARE["var_B"], + 3: ENC_BIG_RARE["var_C"], + } + ) assert encoder.variables_ == [1, 2, 3] - assert encoder.encoder_dict_ == frequenc_cat + assert encoder.encoder_dict_ == { + 1: FREQUENT_CATEGORIES["var_A"], + 2: FREQUENT_CATEGORIES["var_B"], + 3: FREQUENT_CATEGORIES["var_C"], + } pd.testing.assert_frame_equal(X, df) -def test_correctly_ignores_nan_in_transform(df_enc_big): +def test_correctly_ignores_nan_in_transform(make_df, data_enc_big): encoder = RareLabelEncoder( tol=0.06, n_categories=5, missing_values="ignore", ) - X = encoder.fit_transform(df_enc_big) - - # expected: - frequenc_cat = { - "var_A": ["B", "D", "A", "G", "C"], - "var_B": ["A", "D", "B", "G", "C"], - "var_C": ["C", "D", "B", "G", "A"], - } - assert encoder.encoder_dict_ == frequenc_cat - - # input - t = pd.DataFrame( - { - "var_A": ["A", np.nan, "J"], - "var_B": ["A", np.nan, "J"], - "var_C": ["C", np.nan, "J"], - } - ) - - # expected - tt = pd.DataFrame( - { - "var_A": ["A", np.nan, "Rare"], - "var_B": ["A", np.nan, "Rare"], - "var_C": ["C", np.nan, "Rare"], - } + encoder.fit(make_df(data_enc_big)) + assert encoder.encoder_dict_ == FREQUENT_CATEGORIES + + X = encoder.transform( + make_df( + { + "var_A": ["A", None, "J"], + "var_B": ["A", None, "J"], + "var_C": ["C", None, "J"], + } + ) ) - X = encoder.transform(t) - pd.testing.assert_frame_equal(X, tt) - + assert isinstance(X, make_df) + assert frame_to_dict(X) == { + "var_A": ["A", None, "Rare"], + "var_B": ["A", None, "Rare"], + "var_C": ["C", None, "Rare"], + } -def test_correctly_ignores_nan_in_fit(df_enc_big): - df = df_enc_big.copy() - df.loc[df["var_C"] == "G", "var_C"] = np.nan +def test_correctly_ignores_nan_in_fit(make_df, data_enc_big): + data = data_enc_big + data["var_C"] = [None if v == "G" else v for v in data["var_C"]] encoder = RareLabelEncoder( tol=0.06, n_categories=3, missing_values="ignore", ) - encoder.fit(df) + encoder.fit(make_df(data)) # expected: frequent_cat = { @@ -143,72 +182,31 @@ def test_correctly_ignores_nan_in_fit(df_enc_big): for key in frequent_cat.keys(): assert Counter(encoder.encoder_dict_[key]) == Counter(frequent_cat[key]) - # input - t = pd.DataFrame( - { - "var_A": ["A", np.nan, "J", "G"], - "var_B": ["A", np.nan, "J", "G"], - "var_C": ["C", np.nan, "J", "G"], - } - ) - - # expected - tt = pd.DataFrame( - { - "var_A": ["A", np.nan, "Rare", "G"], - "var_B": ["A", np.nan, "Rare", "G"], - "var_C": ["C", np.nan, "Rare", "Rare"], - } + X = encoder.transform( + make_df( + { + "var_A": ["A", None, "J", "G"], + "var_B": ["A", None, "J", "G"], + "var_C": ["C", None, "J", "G"], + } + ) ) - X = encoder.transform(t) - pd.testing.assert_frame_equal(X, tt) - + assert isinstance(X, make_df) + assert frame_to_dict(X) == { + "var_A": ["A", None, "Rare", "G"], + "var_B": ["A", None, "Rare", "G"], + "var_C": ["C", None, "Rare", "Rare"], + } -def test_correctly_ignores_nan_in_fit_when_var_is_numerical(df_enc_big): - df = df_enc_big.copy() +def test_correctly_ignores_nan_in_fit_when_var_is_numerical(data_enc_big): + # pandas only + df = pd.DataFrame(data_enc_big) df["var_C"] = [ - 1, - 1, - 1, - 1, - 2, - 2, - 2, - 2, - 2, - 2, - 3, - 3, - 3, - 3, - 3, - 3, - 3, - 3, - 3, - 3, - 4, - 4, - 4, - 4, - 4, - 4, - 4, - 4, - 4, - 4, - 5, - 5, - 6, - 6, - np.nan, - np.nan, - np.nan, - np.nan, - np.nan, - np.nan, + 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 6, 6, + None, None, None, None, None, None, ] encoder = RareLabelEncoder( @@ -231,18 +229,19 @@ def test_correctly_ignores_nan_in_fit_when_var_is_numerical(df_enc_big): # input t = pd.DataFrame( { - "var_A": ["A", np.nan, "J", "G"], - "var_B": ["A", np.nan, "J", "G"], - "var_C": [3, np.nan, 9, 10], + "var_A": ["A", None, "J", "G"], + "var_B": ["A", None, "J", "G"], + "var_C": [3, None, 9, 10], } ) - # expected + # var_C mixes floats and strings after transform, so its + # missing value must be an actual nan tt = pd.DataFrame( { - "var_A": ["A", np.nan, "Rare", "G"], - "var_B": ["A", np.nan, "Rare", "G"], - "var_C": [3.0, np.nan, "Rare", "Rare"], + "var_A": ["A", None, "Rare", "G"], + "var_B": ["A", None, "Rare", "G"], + "var_C": [3.0, float("nan"), "Rare", "Rare"], } ) @@ -250,15 +249,18 @@ def test_correctly_ignores_nan_in_fit_when_var_is_numerical(df_enc_big): pd.testing.assert_frame_equal(X, tt, check_dtype=False) -def test_user_provides_grouping_label_name_and_variable_list(df_enc_big): - # test case 2: user provides alternative grouping value and variable list +def test_user_provides_grouping_label_name_and_variable_list(make_df, data_enc_big): encoder = RareLabelEncoder( tol=0.15, n_categories=5, variables=["var_A", "var_B"], replace_with="Other" ) - X = encoder.fit_transform(df_enc_big) + X = encoder.fit_transform(make_df(data_enc_big)) - # expected output - df = { + # test fit attr + assert encoder.variables_ == ["var_A", "var_B"] + assert encoder.n_features_in_ == 3 + # test transform output + assert isinstance(X, make_df) + assert frame_to_dict(X) == { "var_A": ["A"] * 6 + ["B"] * 10 + ["Other"] * 4 @@ -271,92 +273,44 @@ def test_user_provides_grouping_label_name_and_variable_list(df_enc_big): + ["D"] * 10 + ["Other"] * 4 + ["G"] * 6, - "var_C": ["A"] * 4 - + ["B"] * 6 - + ["C"] * 10 - + ["D"] * 10 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 6, + "var_C": data_enc_big["var_C"], } - df = pd.DataFrame(df) - - # test init params - assert encoder.tol == 0.15 - assert encoder.n_categories == 5 - assert encoder.replace_with == "Other" - assert encoder.variables == ["var_A", "var_B"] - # test fit attr - assert encoder.variables_ == ["var_A", "var_B"] - assert encoder.n_features_in_ == 3 - # test transform output - pd.testing.assert_frame_equal(X, df) - -# init params -@pytest.mark.parametrize("tol", ["hello", [0.5], -1, 1.5]) -def test_error_if_tol_not_between_0_and_1(tol): - with pytest.raises(ValueError): - RareLabelEncoder(tol=tol) - - -@pytest.mark.parametrize("n_cat", ["hello", [0.5], -0.1, 1.5]) -def test_error_if_n_categories_not_int(n_cat): - with pytest.raises(ValueError): - RareLabelEncoder(n_categories=n_cat) - - -@pytest.mark.parametrize("max_n_categories", ["hello", ["auto"], -1, 0.5]) -def test_raises_error_when_max_n_categories_not_allowed(max_n_categories): - with pytest.raises(ValueError): - RareLabelEncoder(max_n_categories=max_n_categories) - -@pytest.mark.parametrize("replace_with", [set("hello"), ["auto"]]) -def test_error_if_replace_with_not_string(replace_with): - with pytest.raises(ValueError): - RareLabelEncoder(replace_with=replace_with) - - -def test_warning_if_variable_cardinality_less_than_n_categories(df_enc_big): - # test case 3: when the variable has low cardinality - with pytest.warns(UserWarning): - encoder = RareLabelEncoder(n_categories=10) - encoder.fit(df_enc_big) +def test_warning_if_variable_cardinality_less_than_n_categories( + make_df, data_enc_big +): + msg = ( + "The number of unique categories for variable var_A is less than that " + "indicated in n_categories. Thus, all categories will be " + "considered frequent" + ) + encoder = RareLabelEncoder(n_categories=10) + with pytest.warns(UserWarning, match=re.escape(msg)): + encoder.fit(make_df(data_enc_big)) -def test_fit_raises_error_if_df_contains_na(df_enc_big_na): - # test case 4: when dataset contains na, fit method +def test_fit_raises_error_if_df_contains_na(make_df, data_enc_big_na): encoder = RareLabelEncoder(n_categories=4) - with pytest.raises(ValueError) as record: - msg = ( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer or set the parameter " - "`missing_values='ignore'` when initialising this transformer." - ) - encoder.fit(df_enc_big_na) - assert str(record.value) == msg + with pytest.raises(ValueError, match=re.escape(MSG_NA)): + encoder.fit(make_df(data_enc_big_na)) -def test_transform_raises_error_if_df_contains_na(df_enc_big, df_enc_big_na): - # test case 5: when dataset contains na, transform method +def test_transform_raises_error_if_df_contains_na( + make_df, data_enc_big, data_enc_big_na +): encoder = RareLabelEncoder(n_categories=4) - encoder.fit(df_enc_big) - with pytest.raises(ValueError) as record: - msg = ( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer or set the parameter " - "`missing_values='ignore'` when initialising this transformer." - ) - encoder.transform(df_enc_big_na) - assert str(record.value) == msg + encoder.fit(make_df(data_enc_big)) + with pytest.raises(ValueError, match=re.escape(MSG_NA)): + encoder.transform(make_df(data_enc_big_na)) -def test_max_n_categories(df_enc_big): - # test case 6: user provides the maximum number of categories they want +def test_max_n_categories(make_df, data_enc_big): rare_encoder = RareLabelEncoder(tol=0.10, max_n_categories=4, n_categories=5) - X = rare_encoder.fit_transform(df_enc_big) - df = { + X = rare_encoder.fit_transform(make_df(data_enc_big)) + + assert isinstance(X, make_df) + assert frame_to_dict(X) == { "var_A": ["A"] * 6 + ["B"] * 10 + ["Rare"] * 4 @@ -376,12 +330,11 @@ def test_max_n_categories(df_enc_big): + ["Rare"] * 4 + ["G"] * 6, } - df = pd.DataFrame(df) - pd.testing.assert_frame_equal(X, df) -def test_max_n_categories_with_numeric_var(df_enc_numeric): - # ignore_format=True +def test_max_n_categories_with_numeric_var(data_enc_numeric): + # pandas only + df_enc_numeric = pd.DataFrame(data_enc_numeric) rare_encoder = RareLabelEncoder( tol=0.10, max_n_categories=2, n_categories=1, ignore_format=True ) @@ -399,39 +352,44 @@ def test_max_n_categories_with_numeric_var(df_enc_numeric): assert str(list(X["var_B"])[i]) == str(list(df["var_B"])[i]) -def test_variables_cast_as_category(df_enc_big): - # test case 1: defo params, automatically select variables - encoder = RareLabelEncoder( - tol=0.06, n_categories=5, variables=None, replace_with="Rare" +def test_max_n_categories_with_numeric_var_polars(data_enc_numeric): + # polars can't mix int and str in one column, so a numeric variable with a + # string replace_with is cast to string + df_enc_numeric = pl.DataFrame(data_enc_numeric) + rare_encoder = RareLabelEncoder( + tol=0.10, max_n_categories=2, n_categories=1, ignore_format=True ) - df_enc_big = df_enc_big.copy() + X = rare_encoder.fit_transform(df_enc_numeric.select(["var_A", "var_B"])) + + assert isinstance(X, pl.DataFrame) + assert frame_to_dict(X) == { + "var_A": ["1"] * 6 + ["2"] * 10 + ["Rare"] * 4, + "var_B": ["1"] * 10 + ["2"] * 6 + ["Rare"] * 4, + } + + +def test_inverse_transform_raises_not_implemented_error(make_df, data_enc_big): + df_enc_big = make_df(data_enc_big) + enc = RareLabelEncoder().fit(df_enc_big) + msg = "inverse_transform is not implemented for this transformer." + with pytest.raises(NotImplementedError, match=re.escape(msg)): + enc.inverse_transform(df_enc_big) + + +def test_variables_cast_as_category(data_enc_big): + # pandas category dtype is backend-specific: polars has no equivalent + # concept in the same sense. + df_enc_big = pd.DataFrame(data_enc_big) df_enc_big["var_B"] = df_enc_big["var_B"].astype("category") + encoder = RareLabelEncoder( + tol=0.06, n_categories=5, variables=None, replace_with="Rare" + ) X = encoder.fit_transform(df_enc_big) # expected output - df = { - "var_A": ["A"] * 6 - + ["B"] * 10 - + ["C"] * 4 - + ["D"] * 10 - + ["Rare"] * 4 - + ["G"] * 6, - "var_B": ["A"] * 10 - + ["B"] * 6 - + ["C"] * 4 - + ["D"] * 10 - + ["Rare"] * 4 - + ["G"] * 6, - "var_C": ["A"] * 4 - + ["B"] * 6 - + ["C"] * 10 - + ["D"] * 10 - + ["Rare"] * 4 - + ["G"] * 6, - } - df = pd.DataFrame(df) + df = pd.DataFrame(ENC_BIG_RARE) df["var_B"] = pd.Categorical(df["var_B"]) # test fit attr @@ -441,7 +399,11 @@ def test_variables_cast_as_category(df_enc_big): pd.testing.assert_frame_equal(X, df, check_categorical=False) -def test_variables_cast_as_category_with_na_in_transform(df_enc_big): +def test_variables_cast_as_category_with_na_in_transform(data_enc_big): + # pandas category dtype is backend-specific. + df_enc_big = pd.DataFrame(data_enc_big) + df_enc_big["var_B"] = df_enc_big["var_B"].astype("category") + encoder = RareLabelEncoder( tol=0.06, n_categories=5, @@ -449,17 +411,14 @@ def test_variables_cast_as_category_with_na_in_transform(df_enc_big): replace_with="Rare", missing_values="ignore", ) - - df_enc_big = df_enc_big.copy() - df_enc_big["var_B"] = df_enc_big["var_B"].astype("category") encoder.fit(df_enc_big) # input t = pd.DataFrame( { - "var_A": ["A", np.nan, "J", "G"], - "var_B": ["A", np.nan, "J", "G"], - "var_C": ["A", np.nan, "J", "G"], + "var_A": ["A", None, "J", "G"], + "var_B": ["A", None, "J", "G"], + "var_C": ["A", None, "J", "G"], } ) t["var_B"] = pd.Categorical(t["var_B"]) @@ -467,19 +426,19 @@ def test_variables_cast_as_category_with_na_in_transform(df_enc_big): # expected tt = pd.DataFrame( { - "var_A": ["A", np.nan, "Rare", "G"], - "var_B": ["A", np.nan, "Rare", "G"], - "var_C": ["A", np.nan, "Rare", "G"], + "var_A": ["A", None, "Rare", "G"], + "var_B": ["A", None, "Rare", "G"], + "var_C": ["A", None, "Rare", "G"], } ) tt["var_B"] = pd.Categorical(tt["var_B"]) pd.testing.assert_frame_equal(encoder.transform(t), tt, check_categorical=False) -def test_variables_cast_as_category_with_na_in_fit(df_enc_big): - - df = df_enc_big.copy() - df.loc[df["var_C"] == "G", "var_C"] = np.nan +def test_variables_cast_as_category_with_na_in_fit(data_enc_big): + # pandas category dtype is backend-specific. + df = pd.DataFrame(data_enc_big) + df.loc[df["var_C"] == "G", "var_C"] = None df["var_C"] = df["var_C"].astype("category") encoder = RareLabelEncoder( @@ -492,9 +451,9 @@ def test_variables_cast_as_category_with_na_in_fit(df_enc_big): # input t = pd.DataFrame( { - "var_A": ["A", np.nan, "J", "G"], - "var_B": ["A", np.nan, "J", "G"], - "var_C": ["C", np.nan, "J", "G"], + "var_A": ["A", None, "J", "G"], + "var_B": ["A", None, "J", "G"], + "var_C": ["C", None, "J", "G"], } ) t["var_C"] = pd.Categorical(t["var_C"]) @@ -502,17 +461,11 @@ def test_variables_cast_as_category_with_na_in_fit(df_enc_big): # expected tt = pd.DataFrame( { - "var_A": ["A", np.nan, "Rare", "G"], - "var_B": ["A", np.nan, "Rare", "G"], - "var_C": ["C", np.nan, "Rare", "Rare"], + "var_A": ["A", None, "Rare", "G"], + "var_B": ["A", None, "Rare", "G"], + "var_C": ["C", None, "Rare", "Rare"], } ) tt["var_C"] = pd.Categorical(tt["var_C"]) pd.testing.assert_frame_equal(encoder.transform(t), tt, check_categorical=False) - - -def test_inverse_transform_raises_not_implemented_error(df_enc_big): - enc = RareLabelEncoder().fit(df_enc_big) - with pytest.raises(NotImplementedError): - enc.inverse_transform(df_enc_big)