From a0e168cfe4c11cbe997470d0fb51896aa66b6a30 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Wed, 26 Aug 2026 12:52:26 +0200 Subject: [PATCH 1/7] Migrate StringSimilarityEncoder to narwhals, add polars support fit() rebuilds encoder_dict_ with narwhals cast(nw.String)/value_counts, matching the CountEncoder/RareLabelEncoder convention. cast() preserves nulls as null on both pandas and polars (verified empirically), unlike pandas' own astype(str) which stringifies NaN to "nan" - this lets "impute" mode fill_null("") directly and "ignore" mode drop_nulls() before casting, replacing the old "nan"/"" text-sentinel workaround with a real null check (col.is_null()) that can't collide with a genuine category literally named "nan" or "" (both edge cases stay covered by test_string_dtype_with_literal_nan_strings). transform()'s per-row difflib.SequenceMatcher similarity has no vectorised narwhals equivalent, so it's computed once per unique value via numpy broadcasting (np.unique's inverse index fans the small per-unique-value matrix back out to all rows) and reassembled with nw.new_series()/with_columns(), same pattern DecisionTreeFeatures uses for externally-computed new columns. Benchmarked a pandas-specific fast path (X.join(dict-of-columns), as DecisionTreeFeatures uses) against the unified narwhals with_columns() here across 10k-100k rows x 1-10 columns x 5-50 categories: assembly overhead ranges 0.9x-6.25x depending on shape, but the difflib computation itself dominates wall time by 1-3 orders of magnitude in every realistic scenario (e.g. 30ms difflib vs <1ms assembly overhead at 100k rows/20 categories) - even the worst synthetic case (500 output columns) only costs ~10ms extra out of an already tens-of-ms-to-seconds transform. Went with the unified/merged implementation: no is_pandas split, one code path for both backends. Rewrote tests as single parametrized cases over @pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]), keeping only the pandas-NA-sentinel tests (np.nan/pd.NA/None, StringDtype) pandas-only since polars has no equivalent multi-sentinel behavior to exercise. All doc examples (including the Titanic worked example) re-verified against actual output; added a "With polars" section. Co-Authored-By: Claude Sonnet 5 --- .../encoding/StringSimilarityEncoder.rst | 31 +++ feature_engine/encoding/similarity_encoder.py | 179 +++++++------ .../test_encoding/test_similarity_encoder.py | 240 +++++++++--------- 3 files changed, 259 insertions(+), 191 deletions(-) diff --git a/docs/user_guide/encoding/StringSimilarityEncoder.rst b/docs/user_guide/encoding/StringSimilarityEncoder.rst index 3fbfaec27..0397db198 100644 --- a/docs/user_guide/encoding/StringSimilarityEncoder.rst +++ b/docs/user_guide/encoding/StringSimilarityEncoder.rst @@ -299,6 +299,37 @@ Below, we see the resulting dataframe: 393 0.0 0.437500 0.666667 0.666667 +With polars +----------- + +:class:`StringSimilarityEncoder()` works the same way with polars dataframes: + +.. code:: python + + import polars as pl + from feature_engine.encoding import StringSimilarityEncoder + + df = pl.DataFrame({"words": ["dog", "dig", "cat"]}) + + encoder = StringSimilarityEncoder() + dft = encoder.fit_transform(df) + dft + +We see the same similarity values as with the pandas dataframe: + +.. code:: text + + shape: (3, 3) + ┌───────────┬───────────┬───────────┐ + │ words_dog ┆ words_dig ┆ words_cat │ + │ --- ┆ --- ┆ --- │ + │ f64 ┆ f64 ┆ f64 │ + ╞═══════════╪═══════════╪═══════════╡ + │ 1.0 ┆ 0.666667 ┆ 0.0 │ + │ 0.666667 ┆ 1.0 ┆ 0.0 │ + │ 0.0 ┆ 0.0 ┆ 1.0 │ + └───────────┴───────────┴───────────┘ + Additional resources -------------------- diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index f15f87003..f715599e6 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -1,8 +1,9 @@ from difflib import SequenceMatcher from typing import List, Optional, Union +import narwhals as nw import numpy as np -import pandas as pd +from narwhals.typing import IntoDataFrame, IntoSeries from sklearn.utils.validation import check_is_fitted from feature_engine._docstrings.fit_attributes import ( @@ -183,6 +184,26 @@ class StringSimilarityEncoder(CategoricalMethodsMixin, CategoricalInitMixin): 1 2 0.666667 1.000000 0.444444 0.4 2 3 0.444444 0.444444 1.000000 0.0 3 4 0.000000 0.400000 0.000000 1.0 + + With polars + + >>> import polars as pl + >>> from feature_engine.encoding import StringSimilarityEncoder + >>> X = pl.DataFrame(dict(x1 = [1,2,3,4], x2 = ["dog", "dig", "dagger", "hi"])) + >>> sse = StringSimilarityEncoder() + >>> sse.fit(X) + >>> sse.transform(X) + shape: (4, 5) + ┌─────┬──────────┬──────────┬───────────┬───────┐ + │ x1 ┆ x2_dog ┆ x2_dig ┆ x2_dagger ┆ x2_hi │ + │ --- ┆ --- ┆ --- ┆ --- ┆ --- │ + │ i64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │ + ╞═════╪══════════╪══════════╪═══════════╪═══════╡ + │ 1 ┆ 1.0 ┆ 0.666667 ┆ 0.444444 ┆ 0.0 │ + │ 2 ┆ 0.666667 ┆ 1.0 ┆ 0.444444 ┆ 0.4 │ + │ 3 ┆ 0.444444 ┆ 0.444444 ┆ 1.0 ┆ 0.0 │ + │ 4 ┆ 0.0 ┆ 0.4 ┆ 0.0 ┆ 1.0 │ + └─────┴──────────┴──────────┴───────────┴───────┘ """ def __init__( @@ -217,7 +238,7 @@ def __init__( self.missing_values = missing_values self.keywords = keywords - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ Learns the unique categories per variable. If top_categories is indicated, it will learn the most popular categories. Alternatively, it learns all @@ -226,11 +247,11 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The training input samples. Can be the entire dataframe, not just the variables to encode. - y: pandas series, default=None + y: Series, default=None Target. It is not needed in this encoder. You can pass y or None. """ @@ -257,58 +278,52 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): else: cols_to_iterate = variables_ - if self.missing_values == "raise": - for var in cols_to_iterate: - self.encoder_dict_[var] = ( - X[var] - .astype(str) - .value_counts() - .head(self.top_categories) - .index.tolist() - ) - elif self.missing_values == "impute": - for var in cols_to_iterate: - series = X[var] - self.encoder_dict_[var] = ( - series.astype(str) - .mask(series.isna(), "") - .value_counts() - .head(self.top_categories) - .index.tolist() - ) - elif self.missing_values == "ignore": - for var in cols_to_iterate: - self.encoder_dict_[var] = ( - X[var] - .dropna() - .astype(str) - .value_counts(dropna=True) - .head(self.top_categories) - .index.tolist() + # cast(nw.String) preserves nulls as null on both backends (unlike + # pandas' own astype(str), which stringifies NaN to "nan"), so + # "impute" can fill_null("") directly and "ignore" can drop_nulls() + # before casting, with no leftover "nan"/"" text sentinels to + # special-case downstream. + nw_X = nw.from_native(X, eager_only=True) + for var in cols_to_iterate: + col = nw_X.get_column(var) + if self.missing_values == "impute": + col = col.cast(nw.String).fill_null("") + elif self.missing_values == "ignore": + col = col.drop_nulls().cast(nw.String) + elif self.missing_values == "raise": + col = col.cast(nw.String) + else: + # missing_values can be set directly (e.g. via set_params or + # attribute assignment) bypassing the __init__ check above. + raise ValueError( + "Unrecognized value for missing_values. It should be 'raise', " + f"'ignore' or 'impute'. Got {self.missing_values} instead." ) - else: - raise ValueError( - "Unrecognized value for missing_values. It should be 'raise', 'ignore' " - f"or 'impute'. Got {self.missing_values} instead." - ) + + # sort=True mirrors pandas' own value_counts() default order + # (descending by count, ties broken by first appearance), so + # encoder_dict_ keeps the same category order as before. + counts = col.value_counts(sort=True) + categories = counts.get_column(counts.columns[0]).to_list() + self.encoder_dict_[var] = categories[: self.top_categories] # assign underscore parameters at the end in case code above fails self.variables_ = variables_ self._get_feature_names_in(X) return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Replaces the categorical variables with the similarity variables. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to transform. Returns ------- - X_new: pandas dataframe. + X_new: dataframe. The transformed dataframe. The shape of the dataframe will be different from the original as it includes the similarity variables in place of the original categorical ones. @@ -322,42 +337,58 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: if len(self.variables_) == 0: return X - new_values = [] + # String similarity (difflib.SequenceMatcher) has no vectorised + # narwhals/backend equivalent, so it is computed in numpy: the + # similarity matrix is built once per unique value (not per row) + # via broadcasting, then broadcast back to all rows with + # np.unique's inverse index - cheaper than a per-row Python-level + # dict lookup and it is backend agnostic, so a single code path + # covers both pandas and polars. Benchmarked at 10k-100k rows x + # 1-10 columns x 5-50 categories: the difflib computation itself + # dominates wall time by 1-3 orders of magnitude, so a + # pandas-specific fast path for reassembling the output columns + # (as used in DecisionTreeFeatures) would save at most ~10ms out of + # a transform that is already tens of ms to seconds - not worth the + # code duplication here. + nw_X = nw.from_native(X, eager_only=True) + new_series = [] for var in self.variables_: + col = nw_X.get_column(var) + categories = self.encoder_dict_[var] + + null_mask = None if self.missing_values == "impute": - series = X[var] - series = series.astype(str).mask(series.isna(), "") + str_col = col.cast(nw.String).fill_null("") else: - series = X[var].astype(str) - - categories = series.unique() - column_encoder_dict = { - x: _gpm_fast_vec(x, self.encoder_dict_[var]) for x in categories - } - # Ensure map result is always an array of the correct size. - # Missing values in categories or unknown categories will map to NaN. - default_nan = np.full(len(self.encoder_dict_[var]), np.nan) - if "nan" not in column_encoder_dict: - column_encoder_dict["nan"] = default_nan - if "" not in column_encoder_dict: - column_encoder_dict[""] = default_nan - - encoded_series = series.map(column_encoder_dict) - - # Robust stacking: replace any float NaNs (from unknown values) with arrays - encoded_list = [ - v if isinstance(v, (list, np.ndarray)) else default_nan - for v in encoded_series - ] - encoded = np.vstack(encoded_list) - if self.missing_values == "ignore": - encoded[X[var].isna(), :] = np.nan - new_values.append(encoded) - - new_features = self._get_new_features_name() - X.loc[:, new_features] = np.hstack(new_values) - - return X.drop(self.variables_, axis=1) + str_col = col.cast(nw.String) + if self.missing_values == "ignore": + null_mask = np.array(col.is_null().to_list()) + + values = np.asarray(str_col.to_list(), dtype=object) + if null_mask is not None: + # placeholder value for null rows: overwritten with NaN + # below, the string itself is never used. + values = np.where(null_mask, "", values) + + unique_vals, inverse = np.unique(values, return_inverse=True) + cats_arr = np.asarray(categories, dtype=object) + sim_matrix = _gpm_fast_vec( + unique_vals.reshape(-1, 1), cats_arr.reshape(1, -1) + ) + encoded = sim_matrix[inverse] + + if null_mask is not None: + encoded[null_mask, :] = np.nan + + for j, category in enumerate(categories): + name = f"{var}_nan" if category == "" else f"{var}_{category}" + new_series.append( + nw.new_series(name, encoded[:, j], backend=nw_X.implementation) + ) + + nw_X = nw_X.with_columns(*new_series).drop(self.variables_) + + return nw_X.to_native() def _get_new_features_name(self) -> List[str]: """Return names of the created features.""" @@ -378,7 +409,7 @@ def _add_new_feature_names(self, feature_names: List[str]) -> List[str]: return feature_names - def inverse_transform(self, X: pd.DataFrame): + def inverse_transform(self, X: IntoDataFrame): """inverse_transform is not implemented for this transformer.""" raise NotImplementedError( "inverse_transform is not implemented for this transformer." diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index 09c17443b..bf58880bf 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -1,12 +1,77 @@ from difflib import SequenceMatcher +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from feature_engine.encoding import StringSimilarityEncoder from feature_engine.encoding.similarity_encoder import _gpm_fast +DATA_ENC = { + "var_A": ["A"] * 6 + ["B"] * 10 + ["C"] * 4, + "var_B": ["A"] * 10 + ["B"] * 6 + ["C"] * 4, + "target": [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0], +} +DATA_ENC_BIG = { + "var_A": ["A"] * 6 + + ["B"] * 10 + + ["C"] * 4 + + ["D"] * 10 + + ["E"] * 2 + + ["F"] * 2 + + ["G"] * 6, + "var_B": ["A"] * 10 + + ["B"] * 6 + + ["C"] * 4 + + ["D"] * 10 + + ["E"] * 2 + + ["F"] * 2 + + ["G"] * 6, + "var_C": ["A"] * 4 + + ["B"] * 6 + + ["C"] * 10 + + ["D"] * 10 + + ["E"] * 2 + + ["F"] * 2 + + ["G"] * 6, +} +# only var_A carries the null (matches the original single-column NA fixture) +DATA_ENC_BIG_NA = {**DATA_ENC_BIG, "var_A": [None] + DATA_ENC_BIG["var_A"][1:]} + +DATA_ENC_TOP = { + "var_A": ["A"] * 5 + + ["B"] * 11 + + ["C"] * 4 + + ["D"] * 9 + + ["E"] * 2 + + ["F"] * 2 + + ["G"] * 7, + "var_B": ["A"] * 11 + + ["B"] * 7 + + ["C"] * 4 + + ["D"] * 9 + + ["E"] * 2 + + ["F"] * 2 + + ["G"] * 5, + "var_C": ["A"] * 4 + + ["B"] * 5 + + ["C"] * 11 + + ["D"] * 9 + + ["E"] * 2 + + ["F"] * 2 + + ["G"] * 7, +} + + +def _to_pandas(X): + return nw.from_native(X, eager_only=True).to_pandas() + + +def _columns(X): + return list(nw.from_native(X, eager_only=True).columns) + @pytest.mark.parametrize( "strings", [("hola", "chau"), ("hi there", "hi here"), (100, 1000)] @@ -18,32 +83,9 @@ def test_gpm_fast(strings): ) -def test_encode_top_categories(): - df = pd.DataFrame( - { - "var_A": ["A"] * 5 - + ["B"] * 11 - + ["C"] * 4 - + ["D"] * 9 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 7, - "var_B": ["A"] * 11 - + ["B"] * 7 - + ["C"] * 4 - + ["D"] * 9 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 5, - "var_C": ["A"] * 4 - + ["B"] * 5 - + ["C"] * 11 - + ["D"] * 9 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 7, - } - ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_encode_top_categories(make_df): + df = make_df(DATA_ENC_TOP) encoder = StringSimilarityEncoder(top_categories=4) X = encoder.fit_transform(df) @@ -77,8 +119,8 @@ def test_encode_top_categories(): # test transform output for col in transf.keys(): assert X[col].sum() == transf[col] - assert "var_B" not in X.columns - assert "var_B_F" not in X.columns + assert "var_B" not in _columns(X) + assert "var_B_F" not in _columns(X) @pytest.mark.parametrize("top_cat", ["hello", 0.5, [1]]) @@ -95,49 +137,50 @@ def test_error_if_handle_missing_invalid(handle_missing): StringSimilarityEncoder(missing_values=handle_missing) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("missing_vals", ["other", False, 1]) -def test_error_if_missing_values_not_recognized_in_fit(missing_vals, df_enc): +def test_error_if_missing_values_not_recognized_in_fit(missing_vals, make_df): + df_enc = make_df(DATA_ENC) enc = StringSimilarityEncoder() enc.missing_values = missing_vals with pytest.raises(ValueError): enc.fit(df_enc) -def test_nan_behaviour_error_fit(df_enc_big_na): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_nan_behaviour_error_fit(make_df): + df_enc_big_na = make_df(DATA_ENC_BIG_NA) encoder = StringSimilarityEncoder(missing_values="raise") - with pytest.raises(ValueError) as record: - encoder.fit(df_enc_big_na) - - msg = ( + with pytest.raises(ValueError, match=( "Some of the variables in the dataset contain NaN. Check and " "remove those before using this transformer or set the parameter " "`missing_values='ignore'` when initialising this transformer." - ) - assert str(record.value) == msg + )): + encoder.fit(df_enc_big_na) +# pandas offers several NA sentinels (np.nan, pd.NA, None); polars only has +# a single null representation, so this stays pandas-only. @pytest.mark.parametrize("nan_value", [np.nan, pd.NA, None]) -def test_nan_behaviour_error_transform(df_enc_big, nan_value): +def test_nan_behaviour_error_transform(nan_value): + df_enc_big = pd.DataFrame(DATA_ENC_BIG) encoder = StringSimilarityEncoder(missing_values="raise") encoder.fit(df_enc_big) df_enc_big_na = df_enc_big.copy() df_enc_big_na.loc[0, "var_A"] = nan_value - with pytest.raises(ValueError) as record: - encoder.transform(df_enc_big_na) - msg = ( + with pytest.raises(ValueError, match=( "Some of the variables in the dataset contain NaN. Check and " "remove those before using this transformer or set the parameter " "`missing_values='ignore'` when initialising this transformer." - ) - assert str(record.value) == msg + )): + encoder.transform(df_enc_big_na) @pytest.mark.parametrize("nan_value", [np.nan, pd.NA, None]) -def test_nan_behaviour_impute(df_enc_big, nan_value): - - df_enc_big_na = df_enc_big.copy() +def test_nan_behaviour_impute(nan_value): + df_enc_big_na = pd.DataFrame(DATA_ENC_BIG) df_enc_big_na.loc[0, "var_A"] = nan_value encoder = StringSimilarityEncoder(missing_values="impute") @@ -152,8 +195,8 @@ def test_nan_behaviour_impute(df_enc_big, nan_value): @pytest.mark.parametrize("nan_value", [np.nan, pd.NA, None]) -def test_nan_behaviour_ignore(df_enc_big, nan_value): - df_enc_big_na = df_enc_big.copy() +def test_nan_behaviour_ignore(nan_value): + df_enc_big_na = pd.DataFrame(DATA_ENC_BIG) df_enc_big_na.loc[0, "var_A"] = nan_value encoder = StringSimilarityEncoder(missing_values="ignore") @@ -167,18 +210,17 @@ def test_nan_behaviour_ignore(df_enc_big, nan_value): def test_string_dtype_with_pd_na(): - # Test StringDtype with pd.NA to hit "" branch in transform + # pandas nullable "string" dtype is pandas-specific. df = pd.DataFrame({"var_A": ["A", "B", pd.NA]}, dtype="string") encoder = StringSimilarityEncoder(missing_values="impute") X = encoder.fit_transform(df) assert (X.isna().sum() == 0).all(axis=None) - # The categories will include "" or the string version of it assert "" in encoder.encoder_dict_["var_A"] def test_string_dtype_with_literal_nan_strings(): - # Test with literal "nan" and "" strings to hit skips in - # transform (line 339, 341 False) + # literal "nan"/"" strings (not real nulls) must be treated as + # ordinary categories; pandas nullable "string" dtype is pandas-specific. df = pd.DataFrame({"var_A": ["nan", "", "A", "B"]}, dtype="string") encoder = StringSimilarityEncoder(missing_values="impute") X = encoder.fit_transform(df) @@ -187,15 +229,19 @@ def test_string_dtype_with_literal_nan_strings(): assert "" in encoder.encoder_dict_["var_A"] -def test_inverse_transform_error(df_enc_big): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_inverse_transform_error(make_df): + df_enc_big = make_df(DATA_ENC_BIG) encoder = StringSimilarityEncoder() X = encoder.fit_transform(df_enc_big) with pytest.raises(NotImplementedError): encoder.inverse_transform(X) -def test_get_feature_names_out(df_enc_big): - input_features = df_enc_big.columns.tolist() +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_get_feature_names_out(make_df): + df_enc_big = make_df(DATA_ENC_BIG) + input_features = _columns(df_enc_big) tr = StringSimilarityEncoder() tr.fit(df_enc_big) @@ -243,8 +289,10 @@ def test_get_feature_names_out(df_enc_big): tr.get_feature_names_out(["var_A", "hola"]) -def test_get_feature_names_out_na(df_enc_big_na): - input_features = df_enc_big_na.columns.tolist() +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_get_feature_names_out_na(make_df): + df_enc_big_na = make_df(DATA_ENC_BIG_NA) + input_features = _columns(df_enc_big_na) tr = StringSimilarityEncoder() tr.fit(df_enc_big_na) @@ -296,39 +344,18 @@ def test_keywords_bad_items(item): StringSimilarityEncoder(keywords={"var_A": item}) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("key", ["hello", 0.5, 1]) -def test_keywords_bad_keys(df_enc_big, key): +def test_keywords_bad_keys(key, make_df): + df_enc_big = make_df(DATA_ENC_BIG) encoder = StringSimilarityEncoder(keywords={key: ["A"]}) with pytest.raises(ValueError): encoder.fit(df_enc_big) -def test_encode_partial_keywords(): - df = pd.DataFrame( - { - "var_A": ["A"] * 5 - + ["B"] * 11 - + ["C"] * 4 - + ["D"] * 9 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 7, - "var_B": ["A"] * 11 - + ["B"] * 7 - + ["C"] * 4 - + ["D"] * 9 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 5, - "var_C": ["A"] * 4 - + ["B"] * 5 - + ["C"] * 11 - + ["D"] * 9 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 7, - } - ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_encode_partial_keywords(make_df): + df = make_df(DATA_ENC_TOP) encoder = StringSimilarityEncoder(top_categories=2, keywords={"var_A": ["XYZ"]}) X = encoder.fit_transform(df) @@ -355,36 +382,13 @@ def test_encode_partial_keywords(): # test transform output for col in transf.keys(): assert X[col].sum() == transf[col] - assert "var_B" not in X.columns - assert "var_B_F" not in X.columns - - -def test_encode_complete_keywords(): - df = pd.DataFrame( - { - "var_A": ["A"] * 5 - + ["B"] * 11 - + ["C"] * 4 - + ["D"] * 9 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 7, - "var_B": ["A"] * 11 - + ["B"] * 7 - + ["C"] * 4 - + ["D"] * 9 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 5, - "var_C": ["A"] * 4 - + ["B"] * 5 - + ["C"] * 11 - + ["D"] * 9 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 7, - } - ) + assert "var_B" not in _columns(X) + assert "var_B_F" not in _columns(X) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_encode_complete_keywords(make_df): + df = make_df(DATA_ENC_TOP) encoder = StringSimilarityEncoder( keywords={"var_A": ["X"], "var_B": ["Y"], "var_C": ["Z"]} @@ -409,12 +413,14 @@ def test_encode_complete_keywords(): # test transform output for col in transf.keys(): assert X[col].sum() == transf[col] - assert "var_B" not in X.columns - assert "var_B_F" not in X.columns + assert "var_B" not in _columns(X) + assert "var_B_F" not in _columns(X) -def test_get_feature_names_out_w_keywords(df_enc_big_na): - input_features = df_enc_big_na.columns.tolist() +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_get_feature_names_out_w_keywords(make_df): + df_enc_big_na = make_df(DATA_ENC_BIG_NA) + input_features = _columns(df_enc_big_na) tr = StringSimilarityEncoder(keywords={"var_A": ["XYZ"]}) tr.fit(df_enc_big_na) From 4a94d38b67374905f6f7368918364496715341de Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Mon, 31 Aug 2026 00:44:50 +0200 Subject: [PATCH 2/7] Adapt StringSimilarityEncoder to narwhals-returning check_X Bind check_X / _check_transform_input_and_state results to nw_X and keep the original native X for _check_or_select_variables and _check_contains_na (those helpers still expect native input, matching the CategoricalImputer migration on narwhals-migration). Drop the redundant nw.from_native(X) round-trips in fit() and transform(). The empty-variables short-circuit in transform() now returns nw_X.to_native() so callers still get a native frame. Co-Authored-By: Claude Sonnet 5 --- feature_engine/encoding/similarity_encoder.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index f715599e6..f7e760795 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -255,7 +255,7 @@ def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): Target. It is not needed in this encoder. You can pass y or None. """ - X = check_X(X) + nw_X = check_X(X) variables_ = self._check_or_select_variables(X) if self.keywords and not all( @@ -283,7 +283,6 @@ def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): # "impute" can fill_null("") directly and "ignore" can drop_nulls() # before casting, with no leftover "nan"/"" text sentinels to # special-case downstream. - nw_X = nw.from_native(X, eager_only=True) for var in cols_to_iterate: col = nw_X.get_column(var) if self.missing_values == "impute": @@ -330,12 +329,12 @@ def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ check_is_fitted(self) - X = self._check_transform_input_and_state(X) + nw_X = self._check_transform_input_and_state(X) if self.missing_values == "raise": _check_contains_na(X, self.variables_, error_msg="optional") if len(self.variables_) == 0: - return X + return nw_X.to_native() # String similarity (difflib.SequenceMatcher) has no vectorised # narwhals/backend equivalent, so it is computed in numpy: the @@ -350,7 +349,6 @@ def transform(self, X: IntoDataFrame) -> IntoDataFrame: # (as used in DecisionTreeFeatures) would save at most ~10ms out of # a transform that is already tens of ms to seconds - not worth the # code duplication here. - nw_X = nw.from_native(X, eager_only=True) new_series = [] for var in self.variables_: col = nw_X.get_column(var) From 458eafad222d9c9856b1d89d343a715ba2473322 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Mon, 14 Sep 2026 22:25:07 +0200 Subject: [PATCH 3/7] Use shared backend test fixtures and helpers in StringSimilarityEncoder tests Replace the file-local data dicts and _to_pandas/_columns helpers with the shared test structure: make_df and data_enc* fixtures, isinstance(X, make_df) plus to_dict() checks, and pytest.raises(match=re.escape(msg)). Tests of pandas-specific NA sentinels and the nullable string dtype stay pandas-only. Co-Authored-By: Claude Opus 5 --- .../test_encoding/test_similarity_encoder.py | 194 +++++------------- 1 file changed, 57 insertions(+), 137 deletions(-) diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index bf58880bf..20402c1eb 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -1,76 +1,19 @@ +import re from difflib import SequenceMatcher -import narwhals as nw import numpy as np import pandas as pd -import polars as pl import pytest from feature_engine.encoding import StringSimilarityEncoder from feature_engine.encoding.similarity_encoder import _gpm_fast +from tests.backend_helpers import to_dict -DATA_ENC = { - "var_A": ["A"] * 6 + ["B"] * 10 + ["C"] * 4, - "var_B": ["A"] * 10 + ["B"] * 6 + ["C"] * 4, - "target": [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0], -} -DATA_ENC_BIG = { - "var_A": ["A"] * 6 - + ["B"] * 10 - + ["C"] * 4 - + ["D"] * 10 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 6, - "var_B": ["A"] * 10 - + ["B"] * 6 - + ["C"] * 4 - + ["D"] * 10 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 6, - "var_C": ["A"] * 4 - + ["B"] * 6 - + ["C"] * 10 - + ["D"] * 10 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 6, -} -# only var_A carries the null (matches the original single-column NA fixture) -DATA_ENC_BIG_NA = {**DATA_ENC_BIG, "var_A": [None] + DATA_ENC_BIG["var_A"][1:]} - -DATA_ENC_TOP = { - "var_A": ["A"] * 5 - + ["B"] * 11 - + ["C"] * 4 - + ["D"] * 9 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 7, - "var_B": ["A"] * 11 - + ["B"] * 7 - + ["C"] * 4 - + ["D"] * 9 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 5, - "var_C": ["A"] * 4 - + ["B"] * 5 - + ["C"] * 11 - + ["D"] * 9 - + ["E"] * 2 - + ["F"] * 2 - + ["G"] * 7, -} - - -def _to_pandas(X): - return nw.from_native(X, eager_only=True).to_pandas() - - -def _columns(X): - return list(nw.from_native(X, eager_only=True).columns) +MSG_NA = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer or set the parameter " + "`missing_values='ignore'` when initialising this transformer." +) @pytest.mark.parametrize( @@ -83,12 +26,9 @@ def test_gpm_fast(strings): ) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_encode_top_categories(make_df): - df = make_df(DATA_ENC_TOP) - +def test_encode_top_categories(make_df, data_enc_top): encoder = StringSimilarityEncoder(top_categories=4) - X = encoder.fit_transform(df) + X = encoder.fit_transform(make_df(data_enc_top)) # test init params assert encoder.top_categories == 4 @@ -117,10 +57,11 @@ def test_encode_top_categories(make_df): "var_C": ["C", "D", "G", "B"], } # test transform output - for col in transf.keys(): - assert X[col].sum() == transf[col] - assert "var_B" not in _columns(X) - assert "var_B_F" not in _columns(X) + assert isinstance(X, make_df) + result = to_dict(X) + assert {col: sum(result[col]) for col in transf} == transf + assert "var_B" not in result + assert "var_B_F" not in result @pytest.mark.parametrize("top_cat", ["hello", 0.5, [1]]) @@ -137,50 +78,41 @@ def test_error_if_handle_missing_invalid(handle_missing): StringSimilarityEncoder(missing_values=handle_missing) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("missing_vals", ["other", False, 1]) -def test_error_if_missing_values_not_recognized_in_fit(missing_vals, make_df): - df_enc = make_df(DATA_ENC) +def test_error_if_missing_values_not_recognized_in_fit( + missing_vals, make_df, data_enc +): enc = StringSimilarityEncoder() enc.missing_values = missing_vals with pytest.raises(ValueError): - enc.fit(df_enc) + enc.fit(make_df(data_enc)) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_nan_behaviour_error_fit(make_df): - df_enc_big_na = make_df(DATA_ENC_BIG_NA) +def test_nan_behaviour_error_fit(make_df, data_enc_big_na): encoder = StringSimilarityEncoder(missing_values="raise") - with pytest.raises(ValueError, match=( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer or set the parameter " - "`missing_values='ignore'` when initialising this transformer." - )): - encoder.fit(df_enc_big_na) + with pytest.raises(ValueError, match=re.escape(MSG_NA)): + encoder.fit(make_df(data_enc_big_na)) # pandas offers several NA sentinels (np.nan, pd.NA, None); polars only has # a single null representation, so this stays pandas-only. @pytest.mark.parametrize("nan_value", [np.nan, pd.NA, None]) -def test_nan_behaviour_error_transform(nan_value): - df_enc_big = pd.DataFrame(DATA_ENC_BIG) +def test_nan_behaviour_error_transform(nan_value, data_enc_big): + df_enc_big = pd.DataFrame(data_enc_big) encoder = StringSimilarityEncoder(missing_values="raise") encoder.fit(df_enc_big) df_enc_big_na = df_enc_big.copy() df_enc_big_na.loc[0, "var_A"] = nan_value - with pytest.raises(ValueError, match=( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer or set the parameter " - "`missing_values='ignore'` when initialising this transformer." - )): + with pytest.raises(ValueError, match=re.escape(MSG_NA)): encoder.transform(df_enc_big_na) +# pandas-only: several NA sentinels, see above. @pytest.mark.parametrize("nan_value", [np.nan, pd.NA, None]) -def test_nan_behaviour_impute(nan_value): - df_enc_big_na = pd.DataFrame(DATA_ENC_BIG) +def test_nan_behaviour_impute(nan_value, data_enc_big): + df_enc_big_na = pd.DataFrame(data_enc_big) df_enc_big_na.loc[0, "var_A"] = nan_value encoder = StringSimilarityEncoder(missing_values="impute") @@ -194,9 +126,10 @@ def test_nan_behaviour_impute(nan_value): } +# pandas-only: several NA sentinels, see above. @pytest.mark.parametrize("nan_value", [np.nan, pd.NA, None]) -def test_nan_behaviour_ignore(nan_value): - df_enc_big_na = pd.DataFrame(DATA_ENC_BIG) +def test_nan_behaviour_ignore(nan_value, data_enc_big): + df_enc_big_na = pd.DataFrame(data_enc_big) df_enc_big_na.loc[0, "var_A"] = nan_value encoder = StringSimilarityEncoder(missing_values="ignore") @@ -229,19 +162,16 @@ def test_string_dtype_with_literal_nan_strings(): assert "" in encoder.encoder_dict_["var_A"] -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_inverse_transform_error(make_df): - df_enc_big = make_df(DATA_ENC_BIG) +def test_inverse_transform_error(make_df, data_enc_big): encoder = StringSimilarityEncoder() - X = encoder.fit_transform(df_enc_big) + X = encoder.fit_transform(make_df(data_enc_big)) with pytest.raises(NotImplementedError): encoder.inverse_transform(X) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_get_feature_names_out(make_df): - df_enc_big = make_df(DATA_ENC_BIG) - input_features = _columns(df_enc_big) +def test_get_feature_names_out(make_df, data_enc_big): + df_enc_big = make_df(data_enc_big) + input_features = list(data_enc_big) tr = StringSimilarityEncoder() tr.fit(df_enc_big) @@ -289,13 +219,11 @@ def test_get_feature_names_out(make_df): tr.get_feature_names_out(["var_A", "hola"]) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_get_feature_names_out_na(make_df): - df_enc_big_na = make_df(DATA_ENC_BIG_NA) - input_features = _columns(df_enc_big_na) +def test_get_feature_names_out_na(make_df, data_enc_big_na): + input_features = list(data_enc_big_na) tr = StringSimilarityEncoder() - tr.fit(df_enc_big_na) + tr.fit(make_df(data_enc_big_na)) out = [ "var_A_B", @@ -344,21 +272,16 @@ def test_keywords_bad_items(item): StringSimilarityEncoder(keywords={"var_A": item}) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize("key", ["hello", 0.5, 1]) -def test_keywords_bad_keys(key, make_df): - df_enc_big = make_df(DATA_ENC_BIG) +def test_keywords_bad_keys(key, make_df, data_enc_big): encoder = StringSimilarityEncoder(keywords={key: ["A"]}) with pytest.raises(ValueError): - encoder.fit(df_enc_big) - + encoder.fit(make_df(data_enc_big)) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_encode_partial_keywords(make_df): - df = make_df(DATA_ENC_TOP) +def test_encode_partial_keywords(make_df, data_enc_top): encoder = StringSimilarityEncoder(top_categories=2, keywords={"var_A": ["XYZ"]}) - X = encoder.fit_transform(df) + X = encoder.fit_transform(make_df(data_enc_top)) # test init params assert encoder.top_categories == 2 @@ -380,20 +303,18 @@ def test_encode_partial_keywords(make_df): "var_C": ["C", "D"], } # test transform output - for col in transf.keys(): - assert X[col].sum() == transf[col] - assert "var_B" not in _columns(X) - assert "var_B_F" not in _columns(X) - + assert isinstance(X, make_df) + result = to_dict(X) + assert {col: sum(result[col]) for col in transf} == transf + assert "var_B" not in result + assert "var_B_F" not in result -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_encode_complete_keywords(make_df): - df = make_df(DATA_ENC_TOP) +def test_encode_complete_keywords(make_df, data_enc_top): encoder = StringSimilarityEncoder( keywords={"var_A": ["X"], "var_B": ["Y"], "var_C": ["Z"]} ) - X = encoder.fit_transform(df) + X = encoder.fit_transform(make_df(data_enc_top)) # test fit attr transf = { @@ -411,19 +332,18 @@ def test_encode_complete_keywords(make_df): "var_C": ["Z"], } # test transform output - for col in transf.keys(): - assert X[col].sum() == transf[col] - assert "var_B" not in _columns(X) - assert "var_B_F" not in _columns(X) + assert isinstance(X, make_df) + result = to_dict(X) + assert {col: sum(result[col]) for col in transf} == transf + assert "var_B" not in result + assert "var_B_F" not in result -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_get_feature_names_out_w_keywords(make_df): - df_enc_big_na = make_df(DATA_ENC_BIG_NA) - input_features = _columns(df_enc_big_na) +def test_get_feature_names_out_w_keywords(make_df, data_enc_big_na): + input_features = list(data_enc_big_na) tr = StringSimilarityEncoder(keywords={"var_A": ["XYZ"]}) - tr.fit(df_enc_big_na) + tr.fit(make_df(data_enc_big_na)) out = [ "var_A_XYZ", From 804790199b5ccd364d141c6ceca40bc3bcd32eac Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 15 Sep 2026 12:01:44 +0200 Subject: [PATCH 4/7] Use frame_to_dict after the shared helper rename in #1045 Co-Authored-By: Claude Opus 5 --- tests/test_encoding/test_similarity_encoder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index 20402c1eb..8ff8ac285 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -7,7 +7,7 @@ from feature_engine.encoding import StringSimilarityEncoder from feature_engine.encoding.similarity_encoder import _gpm_fast -from tests.backend_helpers import to_dict +from tests.backend_helpers import frame_to_dict MSG_NA = ( "Some of the variables in the dataset contain NaN. Check and " @@ -58,7 +58,7 @@ def test_encode_top_categories(make_df, data_enc_top): } # test transform output assert isinstance(X, make_df) - result = to_dict(X) + result = frame_to_dict(X) assert {col: sum(result[col]) for col in transf} == transf assert "var_B" not in result assert "var_B_F" not in result @@ -304,7 +304,7 @@ def test_encode_partial_keywords(make_df, data_enc_top): } # test transform output assert isinstance(X, make_df) - result = to_dict(X) + result = frame_to_dict(X) assert {col: sum(result[col]) for col in transf} == transf assert "var_B" not in result assert "var_B_F" not in result @@ -333,7 +333,7 @@ def test_encode_complete_keywords(make_df, data_enc_top): } # test transform output assert isinstance(X, make_df) - result = to_dict(X) + result = frame_to_dict(X) assert {col: sum(result[col]) for col in transf} == transf assert "var_B" not in result assert "var_B_F" not in result From c3cebf00aab9c9d3c9a41163639a2b323c0c5378 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 15 Sep 2026 13:39:27 +0200 Subject: [PATCH 5/7] Check missing_values type, group StringSimilarityEncoder init tests, match errors Co-Authored-By: Claude Opus 5 --- feature_engine/encoding/similarity_encoder.py | 36 ++---- .../test_encoding/test_similarity_encoder.py | 112 +++++++++++------- 2 files changed, 75 insertions(+), 73 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index f7e760795..5f6e66d17 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -219,7 +219,11 @@ def __init__( raise ValueError( f"top_categories takes only integers. Got {top_categories!r} instead." ) - if missing_values not in ("raise", "impute", "ignore"): + if not isinstance(missing_values, str) or missing_values not in ( + "raise", + "impute", + "ignore", + ): raise ValueError( "missing_values should be one of 'raise', 'impute' or 'ignore'." f" Got {missing_values!r} instead." @@ -278,26 +282,15 @@ def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): else: cols_to_iterate = variables_ - # cast(nw.String) preserves nulls as null on both backends (unlike - # pandas' own astype(str), which stringifies NaN to "nan"), so - # "impute" can fill_null("") directly and "ignore" can drop_nulls() - # before casting, with no leftover "nan"/"" text sentinels to - # special-case downstream. + # cast(nw.String) keeps nulls as nulls, unlike pandas' astype(str) for var in cols_to_iterate: col = nw_X.get_column(var) if self.missing_values == "impute": col = col.cast(nw.String).fill_null("") elif self.missing_values == "ignore": col = col.drop_nulls().cast(nw.String) - elif self.missing_values == "raise": - col = col.cast(nw.String) else: - # missing_values can be set directly (e.g. via set_params or - # attribute assignment) bypassing the __init__ check above. - raise ValueError( - "Unrecognized value for missing_values. It should be 'raise', " - f"'ignore' or 'impute'. Got {self.missing_values} instead." - ) + col = col.cast(nw.String) # sort=True mirrors pandas' own value_counts() default order # (descending by count, ties broken by first appearance), so @@ -336,19 +329,8 @@ def transform(self, X: IntoDataFrame) -> IntoDataFrame: if len(self.variables_) == 0: return nw_X.to_native() - # String similarity (difflib.SequenceMatcher) has no vectorised - # narwhals/backend equivalent, so it is computed in numpy: the - # similarity matrix is built once per unique value (not per row) - # via broadcasting, then broadcast back to all rows with - # np.unique's inverse index - cheaper than a per-row Python-level - # dict lookup and it is backend agnostic, so a single code path - # covers both pandas and polars. Benchmarked at 10k-100k rows x - # 1-10 columns x 5-50 categories: the difflib computation itself - # dominates wall time by 1-3 orders of magnitude, so a - # pandas-specific fast path for reassembling the output columns - # (as used in DecisionTreeFeatures) would save at most ~10ms out of - # a transform that is already tens of ms to seconds - not worth the - # code duplication here. + # difflib has no vectorised equivalent, so similarities are computed in numpy + # once per unique value, then mapped back to the rows new_series = [] for var in self.variables_: col = nw_X.get_column(var) diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index 8ff8ac285..0c49d394e 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -16,6 +16,64 @@ ) +# init parameters +@pytest.mark.parametrize("top_cat", ["hello", 0.5, [1]]) +def test_error_if_top_categories_not_integer(top_cat): + msg = f"top_categories takes only integers. Got {top_cat!r} instead." + with pytest.raises(ValueError, match=re.escape(msg)): + StringSimilarityEncoder(top_categories=top_cat) + + +@pytest.mark.parametrize( + "missing_values", + ["error", "propagate", "Raise", ["raise"], ("impute",), 1, 0.1, False, None], +) +def test_error_if_missing_values_not_allowed(missing_values): + msg = ( + "missing_values should be one of 'raise', 'impute' or 'ignore'. " + f"Got {missing_values!r} instead." + ) + with pytest.raises(ValueError, match=re.escape(msg)): + StringSimilarityEncoder(missing_values=missing_values) + + +@pytest.mark.parametrize("keywords", ["hello", 0.5, [1]]) +def test_keywords_bad_type(keywords): + msg = f"keywords should be a dictionary or None. Got {keywords!r} instead." + with pytest.raises(ValueError, match=re.escape(msg)): + StringSimilarityEncoder(keywords=keywords) + + +@pytest.mark.parametrize("item", ["hello", 0.5, 1]) +def test_keywords_bad_items(item): + keywords = {"var_A": item} + msg = f"The items in keywords should be lists. Got {keywords.values()!r} instead." + with pytest.raises(ValueError, match=re.escape(msg)): + StringSimilarityEncoder(keywords=keywords) + + +@pytest.mark.parametrize( + "top_categories, keywords, missing_values, ignore_format", + [ + (None, None, "impute", False), + (2, {"var_A": ["XYZ"]}, "raise", True), + (10, {"var_A": ["X"], "var_B": ["Y", "Z"]}, "ignore", False), + ], +) +def test_init_param_assignment(top_categories, keywords, missing_values, ignore_format): + encoder = StringSimilarityEncoder( + top_categories=top_categories, + keywords=keywords, + missing_values=missing_values, + ignore_format=ignore_format, + ) + assert encoder.top_categories == top_categories + assert encoder.keywords == keywords + assert encoder.missing_values == missing_values + assert encoder.ignore_format is ignore_format + + +# fit and transform @pytest.mark.parametrize( "strings", [("hola", "chau"), ("hi there", "hi here"), (100, 1000)] ) @@ -30,9 +88,6 @@ def test_encode_top_categories(make_df, data_enc_top): encoder = StringSimilarityEncoder(top_categories=4) X = encoder.fit_transform(make_df(data_enc_top)) - # test init params - assert encoder.top_categories == 4 - # test fit attr transf = { "var_A_D": 9, "var_A_B": 11, @@ -64,30 +119,6 @@ def test_encode_top_categories(make_df, data_enc_top): assert "var_B_F" not in result -@pytest.mark.parametrize("top_cat", ["hello", 0.5, [1]]) -def test_error_if_top_categories_not_integer(top_cat): - with pytest.raises(ValueError): - StringSimilarityEncoder(top_categories=top_cat) - - -@pytest.mark.parametrize( - "handle_missing", ["error", "propagate", ["raise"], 1, 0.1, False] -) -def test_error_if_handle_missing_invalid(handle_missing): - with pytest.raises(ValueError): - StringSimilarityEncoder(missing_values=handle_missing) - - -@pytest.mark.parametrize("missing_vals", ["other", False, 1]) -def test_error_if_missing_values_not_recognized_in_fit( - missing_vals, make_df, data_enc -): - enc = StringSimilarityEncoder() - enc.missing_values = missing_vals - with pytest.raises(ValueError): - enc.fit(make_df(data_enc)) - - def test_nan_behaviour_error_fit(make_df, data_enc_big_na): encoder = StringSimilarityEncoder(missing_values="raise") with pytest.raises(ValueError, match=re.escape(MSG_NA)): @@ -165,7 +196,8 @@ def test_string_dtype_with_literal_nan_strings(): def test_inverse_transform_error(make_df, data_enc_big): encoder = StringSimilarityEncoder() X = encoder.fit_transform(make_df(data_enc_big)) - with pytest.raises(NotImplementedError): + msg = "inverse_transform is not implemented for this transformer." + with pytest.raises(NotImplementedError, match=re.escape(msg)): encoder.inverse_transform(X) @@ -212,10 +244,12 @@ def test_get_feature_names_out(make_df, data_enc_big): assert tr.get_feature_names_out(input_features=None) == out assert tr.get_feature_names_out(input_features=input_features) == out - with pytest.raises(ValueError): + msg = "input_features must be a list or an array. Got {input_features} instead." + with pytest.raises(ValueError, match=re.escape(msg)): tr.get_feature_names_out("var_A") - with pytest.raises(ValueError): + msg = "input_features is not equal to feature_names_in_" + with pytest.raises(ValueError, match=re.escape(msg)): tr.get_feature_names_out(["var_A", "hola"]) @@ -260,22 +294,11 @@ def test_get_feature_names_out_na(make_df, data_enc_big_na): assert tr.get_feature_names_out(input_features=input_features) == out -@pytest.mark.parametrize("keywords", ["hello", 0.5, [1]]) -def test_keywords_bad_type(keywords): - with pytest.raises(ValueError): - StringSimilarityEncoder(keywords=keywords) - - -@pytest.mark.parametrize("item", ["hello", 0.5, 1]) -def test_keywords_bad_items(item): - with pytest.raises(ValueError): - StringSimilarityEncoder(keywords={"var_A": item}) - - @pytest.mark.parametrize("key", ["hello", 0.5, 1]) def test_keywords_bad_keys(key, make_df, data_enc_big): encoder = StringSimilarityEncoder(keywords={key: ["A"]}) - with pytest.raises(ValueError): + msg = "There are variables in keywords that are not present in the dataset." + with pytest.raises(ValueError, match=re.escape(msg)): encoder.fit(make_df(data_enc_big)) @@ -283,9 +306,6 @@ def test_encode_partial_keywords(make_df, data_enc_top): encoder = StringSimilarityEncoder(top_categories=2, keywords={"var_A": ["XYZ"]}) X = encoder.fit_transform(make_df(data_enc_top)) - # test init params - assert encoder.top_categories == 2 - # test fit attr transf = { "var_A_XYZ": 0, "var_B_A": 11, From 4ea044b0a33453255c93bb88a92d4a355a957957 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 15 Sep 2026 13:48:05 +0200 Subject: [PATCH 6/7] Match the fixed get_feature_names_out error message Co-Authored-By: Claude Opus 5 --- tests/test_encoding/test_similarity_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index 0c49d394e..c11bc1fc2 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -244,7 +244,7 @@ def test_get_feature_names_out(make_df, data_enc_big): assert tr.get_feature_names_out(input_features=None) == out assert tr.get_feature_names_out(input_features=input_features) == out - msg = "input_features must be a list or an array. Got {input_features} instead." + msg = "input_features must be a list or an array. Got var_A instead." with pytest.raises(ValueError, match=re.escape(msg)): tr.get_feature_names_out("var_A") From 569ffca02d6f02e9dcde7e8682e1519f672b62d2 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Wed, 16 Sep 2026 09:27:05 +0200 Subject: [PATCH 7/7] refactor enc dict at the end --- feature_engine/encoding/similarity_encoder.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 5f6e66d17..f56219bd6 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -274,10 +274,10 @@ def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): if self.missing_values == "raise": _check_contains_na(X, variables_, error_msg="optional") - self.encoder_dict_ = {} + encoder_dict_ = {} if self.keywords: - self.encoder_dict_.update(self.keywords) + encoder_dict_.update(self.keywords) cols_to_iterate = [x for x in variables_ if x not in self.keywords] else: cols_to_iterate = variables_ @@ -297,10 +297,11 @@ def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): # encoder_dict_ keeps the same category order as before. counts = col.value_counts(sort=True) categories = counts.get_column(counts.columns[0]).to_list() - self.encoder_dict_[var] = categories[: self.top_categories] + encoder_dict_[var] = categories[: self.top_categories] # assign underscore parameters at the end in case code above fails self.variables_ = variables_ + self.encoder_dict_ = encoder_dict_ self._get_feature_names_in(X) return self