diff --git a/feature_engine/datetime/_datetime_constants.py b/feature_engine/datetime/_datetime_constants.py index 2302bc651..5b2cb6755 100644 --- a/feature_engine/datetime/_datetime_constants.py +++ b/feature_engine/datetime/_datetime_constants.py @@ -62,7 +62,7 @@ "quarter": lambda x: x.dt.quarter, "semester": lambda x: np.where(x.dt.month <= 6, 1, 2).astype(np.int64), "year": lambda x: x.dt.year, - "week": lambda x: x.dt.isocalendar().week.astype(np.int64), + "week": lambda x: x.dt.isocalendar().week.astype("float64"), "day_of_week": lambda x: x.dt.dayofweek, "day_of_month": lambda x: x.dt.day, "day_of_year": lambda x: x.dt.dayofyear, @@ -74,7 +74,7 @@ "year_start": lambda x: x.dt.is_year_start.astype(np.int64), "year_end": lambda x: x.dt.is_year_end.astype(np.int64), "leap_year": lambda x: x.dt.is_leap_year.astype(np.int64), - "days_in_month": lambda x: x.dt.days_in_month.astype(np.int64), + "days_in_month": lambda x: x.dt.days_in_month, "hour": lambda x: x.dt.hour, "minute": lambda x: x.dt.minute, "second": lambda x: x.dt.second, diff --git a/feature_engine/timeseries/forecasting/window_features.py b/feature_engine/timeseries/forecasting/window_features.py index 24518a3f6..f5f73c7f7 100644 --- a/feature_engine/timeseries/forecasting/window_features.py +++ b/feature_engine/timeseries/forecasting/window_features.py @@ -221,7 +221,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: for win in self.window: tmp = ( X[self.variables_] - .rolling(window=win) + .rolling(window=win, min_periods=self.min_periods) .agg(self.functions) .shift(periods=self.periods, freq=self.freq) ) @@ -231,7 +231,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: else: tmp = ( X[self.variables_] - .rolling(window=self.window) + .rolling(window=self.window, min_periods=self.min_periods) .agg(self.functions) .shift(periods=self.periods, freq=self.freq) ) diff --git a/tests/estimator_checks/sklearn_check_wrapper.py b/tests/estimator_checks/sklearn_check_wrapper.py new file mode 100644 index 000000000..5e5deb459 --- /dev/null +++ b/tests/estimator_checks/sklearn_check_wrapper.py @@ -0,0 +1,119 @@ +"""Adapter that lets sklearn's ``check_estimator`` run against Feature-engine +transformers. + +``check_estimator`` feeds transformers numpy arrays. Feature-engine transformers +only accept dataframes: ``feature_engine.dataframe_checks.check_X`` raises +``TypeError`` on anything that is not a dataframe from a narwhals-supported +library. Historically ``check_X`` itself converted numpy arrays into pandas +dataframes (naming the columns ``x0``, ``x1``, ...) precisely so that these +tests could run; that conversion is being removed as the checks migrate to +narwhals. + +``wrap_for_check_estimator`` moves that adaptation into the test layer. It wraps +a transformer in a thin meta-estimator that turns the numpy arrays coming from +``check_estimator`` into pandas dataframes -- with the same ``x{i}`` column names +``check_X`` used to assign -- before delegating to the wrapped transformer. Tags +are delegated too, so ``check_estimator`` runs exactly the same set of checks it +would run against the bare transformer, and the ``expected_failed_checks`` +dictionaries in the test files stay unchanged. +""" + +import numpy as np +import pandas as pd +from sklearn.base import BaseEstimator, TransformerMixin, clone +from sklearn.utils.validation import check_is_fitted + + +def wrap_for_check_estimator(estimator): + """Wrap ``estimator`` so sklearn's ``check_estimator`` can feed it arrays. + + Parameters + ---------- + estimator : feature-engine transformer instance. + + Returns + ------- + _SklearnCheckInputWrapper + A meta-estimator delegating to ``estimator`` that converts numpy array + input into a pandas dataframe before every call. + """ + return _SklearnCheckInputWrapper(estimator) + + +class _SklearnCheckInputWrapper(TransformerMixin, BaseEstimator): + """Test-only shim: feeds the wrapped transformer a pandas dataframe when + sklearn's ``check_estimator`` passes numpy arrays. + + Reproduces the numpy-array handling that + ``feature_engine.dataframe_checks.check_X`` used to provide. + """ + + def __init__(self, estimator=None, random_state=None): + self.estimator = estimator + self.random_state = random_state + + @staticmethod + def _to_df(X): + """Convert a numpy array into a dataframe, mirroring the old ``check_X``. + + Dataframes are returned unchanged. Scalars, 1D arrays and complex data + raise the same errors ``check_X`` used to raise for them. + """ + if hasattr(X, "iloc"): + return X + + arr = np.asarray(X) + if arr.ndim == 0: + raise ValueError( + "Expected 2D array, got scalar array instead:\narray={}.\n" + "Reshape your data either using array.reshape(-1, 1) if your " + "data has a single feature or array.reshape(1, -1) if it " + "contains a single sample.".format(arr) + ) + if arr.ndim == 1: + raise ValueError( + "Expected 2D array, got 1D array instead:\narray={}.\n" + "Reshape your data either using array.reshape(-1, 1) if your " + "data has a single feature or array.reshape(1, -1) if it " + "contains a single sample.".format(arr) + ) + if np.iscomplexobj(arr): + raise TypeError("Complex data not supported by this transformer.") + + return pd.DataFrame(arr, columns=[f"x{i}" for i in range(arr.shape[1])]) + + def fit(self, X, y=None): + self.estimator_ = clone(self.estimator) + if ( + self.random_state is not None + and "random_state" in self.estimator_.get_params() + ): + self.estimator_.set_params(random_state=self.random_state) + if y is None: + self.estimator_.fit(self._to_df(X)) + else: + self.estimator_.fit(self._to_df(X), y) + + self.n_features_in_ = self.estimator_.n_features_in_ + for attr in ("feature_names_in_", "classes_"): + if hasattr(self.estimator_, attr): + setattr(self, attr, getattr(self.estimator_, attr)) + return self + + def transform(self, X): + check_is_fitted(self) + return self.estimator_.transform(self._to_df(X)) + + def inverse_transform(self, X): + check_is_fitted(self) + return self.estimator_.inverse_transform(self._to_df(X)) + + def get_feature_names_out(self, input_features=None): + check_is_fitted(self) + return self.estimator_.get_feature_names_out(input_features) + + def __sklearn_tags__(self): + return self.estimator.__sklearn_tags__() + + def _more_tags(self): + return self.estimator._more_tags() diff --git a/tests/test_creation/test_check_estimator_creation.py b/tests/test_creation/test_check_estimator_creation.py index dc8beaa9e..f472e3ec5 100644 --- a/tests/test_creation/test_check_estimator_creation.py +++ b/tests/test_creation/test_check_estimator_creation.py @@ -14,6 +14,7 @@ from tests.estimator_checks.non_fitted_error_checks import ( check_raises_non_fitted_error_when_fit_fails, ) +from tests.estimator_checks.sklearn_check_wrapper import wrap_for_check_estimator # Estimators for sklearn's check_estimator # Note: GeoDistanceFeatures is not included here because it requires 4 specific @@ -32,7 +33,7 @@ @pytest.mark.parametrize("estimator", _estimators) def test_check_estimator_from_sklearn(estimator): return check_estimator( - estimator=estimator, + estimator=wrap_for_check_estimator(estimator), expected_failed_checks=estimator._more_tags()["_xfail_checks"], ) @@ -72,12 +73,14 @@ def test_transformers_in_pipeline_with_set_output_pandas(transformer): # Test GeoDistanceFeatures in pipeline with proper column names def test_geo_distance_transformer_in_pipeline(): """Test GeoDistanceFeatures works in a sklearn pipeline.""" - X = pd.DataFrame({ - "lat1": [40.7128, 34.0522], - "lon1": [-74.0060, -118.2437], - "lat2": [34.0522, 41.8781], - "lon2": [-118.2437, -87.6298], - }) + X = pd.DataFrame( + { + "lat1": [40.7128, 34.0522], + "lon1": [-74.0060, -118.2437], + "lat2": [34.0522, 41.8781], + "lon2": [-118.2437, -87.6298], + } + ) y = pd.Series([0, 1]) transformer = GeoDistanceFeatures( diff --git a/tests/test_datetime/test_datetime_features.py b/tests/test_datetime/test_datetime_features.py index dbd6285ef..4934741e3 100644 --- a/tests/test_datetime/test_datetime_features.py +++ b/tests/test_datetime/test_datetime_features.py @@ -236,9 +236,11 @@ def test_extract_datetime_features_from_specified_variables(make_df): # multiple datetime variables, in different order than they appear in X Xt = DatetimeFeatures(variables=["date2", "date"]).fit_transform(X) result = nw.from_native(Xt, eager_only=True) - expected_cols = vars_non_dt + [ - f"date2{FEATURES_SUFFIXES[feat]}" for feat in FEATURES_DEFAULT - ] + feat_names_default_cb + expected_cols = ( + vars_non_dt + + [f"date2{FEATURES_SUFFIXES[feat]}" for feat in FEATURES_DEFAULT] + + feat_names_default_cb + ) assert result.columns == expected_cols for col, expected in _expected_cross_backend_features(FEATURES_DEFAULT).items(): assert _to_py_values(result.get_column(col)) == expected @@ -294,9 +296,7 @@ def test_extract_all_datetime_features(make_df): @pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -@pytest.mark.parametrize( - "features", [["semester", "week"], ["hour", "day_of_week"]] -) +@pytest.mark.parametrize("features", [["semester", "week"], ["hour", "day_of_week"]]) def test_extract_specified_datetime_features(make_df, features): X = make_df(CROSS_BACKEND_DATA) Xt = DatetimeFeatures(features_to_extract=features).fit_transform(X) @@ -517,6 +517,23 @@ def test_polars_string_parsing_needs_explicit_format_for_ambiguous_dates(): ).to_list() == [2010, 1945] +def test_ignore_nan_for_week_and_days_in_month(): + # week and days_in_month must propagate NaN like the other features when + # missing_values="ignore", instead of raising on the int cast of a NaT. + X = DatetimeFeatures( + features_to_extract=["week", "days_in_month"], missing_values="ignore" + ).fit_transform(dates_nan) + pd.testing.assert_frame_equal( + X, + pd.DataFrame( + { + "dates_na_week": [5.0, np.nan, 22.0, np.nan], + "dates_na_days_in_month": [28.0, np.nan, 30.0, np.nan], + } + ), + ) + + def test_extract_features_with_different_datetime_parsing_options(df_datetime): X = DatetimeFeatures( features_to_extract=["day_of_month"], dayfirst=True diff --git a/tests/test_discretisation/test_check_estimator_discretisers.py b/tests/test_discretisation/test_check_estimator_discretisers.py index 2c7e1a332..9d47e6593 100644 --- a/tests/test_discretisation/test_check_estimator_discretisers.py +++ b/tests/test_discretisation/test_check_estimator_discretisers.py @@ -15,6 +15,7 @@ from tests.estimator_checks.non_fitted_error_checks import ( check_raises_non_fitted_error_when_fit_fails, ) +from tests.estimator_checks.sklearn_check_wrapper import wrap_for_check_estimator _estimators = [ DecisionTreeDiscretiser(regression=False), @@ -28,7 +29,7 @@ @pytest.mark.parametrize("estimator", _estimators) def test_check_estimator_from_sklearn(estimator): return check_estimator( - estimator=estimator, + estimator=wrap_for_check_estimator(estimator), expected_failed_checks=estimator._more_tags()["_xfail_checks"], ) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index 82e299588..db6e3b7be 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -22,6 +22,7 @@ check_feature_engine_estimator, test_df, ) +from tests.estimator_checks.sklearn_check_wrapper import wrap_for_check_estimator _estimators = [ CountEncoder(ignore_format=True), @@ -50,7 +51,8 @@ def test_check_estimator_from_sklearn(estimator): if estimator.__class__.__name__ != "WoEEncoder": return check_estimator( - estimator=estimator, expected_failed_checks=expected_fails + estimator=wrap_for_check_estimator(estimator), + expected_failed_checks=expected_fails, ) diff --git a/tests/test_imputation/test_check_estimator_imputers.py b/tests/test_imputation/test_check_estimator_imputers.py index fa01f096f..7b437ee65 100644 --- a/tests/test_imputation/test_check_estimator_imputers.py +++ b/tests/test_imputation/test_check_estimator_imputers.py @@ -16,6 +16,7 @@ from tests.estimator_checks.non_fitted_error_checks import ( check_raises_non_fitted_error_when_fit_fails, ) +from tests.estimator_checks.sklearn_check_wrapper import wrap_for_check_estimator _estimators = [ MeanImputer(), @@ -31,7 +32,7 @@ @pytest.mark.parametrize("estimator", _estimators) def test_check_estimator_from_sklearn(estimator): return check_estimator( - estimator=estimator, + estimator=wrap_for_check_estimator(estimator), expected_failed_checks=estimator._more_tags()["_xfail_checks"], ) diff --git a/tests/test_outliers/test_check_estimator_outliers.py b/tests/test_outliers/test_check_estimator_outliers.py index 0b5ee3491..7b85d3f99 100644 --- a/tests/test_outliers/test_check_estimator_outliers.py +++ b/tests/test_outliers/test_check_estimator_outliers.py @@ -6,6 +6,7 @@ from feature_engine.outliers import ArbitraryOutlierCapper, OutlierTrimmer, Winsoriser from feature_engine.tags import _return_tags from tests.estimator_checks.estimator_checks import check_feature_engine_estimator +from tests.estimator_checks.sklearn_check_wrapper import wrap_for_check_estimator _estimators = [ ArbitraryOutlierCapper(max_capping_dict={"x0": 10}), @@ -38,7 +39,10 @@ ], ) def test_check_estimator_from_sklearn(estimator, failed_tests): - return check_estimator(estimator=estimator, expected_failed_checks=failed_tests) + return check_estimator( + estimator=wrap_for_check_estimator(estimator), + expected_failed_checks=failed_tests, + ) @pytest.mark.parametrize("estimator", _estimators) diff --git a/tests/test_preprocessing/test_check_estimator_preprocessing.py b/tests/test_preprocessing/test_check_estimator_preprocessing.py index ca16f8863..d5293807e 100644 --- a/tests/test_preprocessing/test_check_estimator_preprocessing.py +++ b/tests/test_preprocessing/test_check_estimator_preprocessing.py @@ -12,6 +12,7 @@ check_feature_engine_estimator, test_df, ) +from tests.estimator_checks.sklearn_check_wrapper import wrap_for_check_estimator _estimators = [MatchCategories(ignore_format=True), MatchVariables()] @@ -41,7 +42,10 @@ ], ) def test_check_estimator_from_sklearn(estimator, failed_tests): - return check_estimator(estimator=estimator, expected_failed_checks=failed_tests) + return check_estimator( + estimator=wrap_for_check_estimator(estimator), + expected_failed_checks=failed_tests, + ) @pytest.mark.parametrize("estimator", [MatchCategories(), MatchVariables()]) diff --git a/tests/test_selection/test_check_estimator_selectors.py b/tests/test_selection/test_check_estimator_selectors.py index 7ce85634b..85340f5c0 100644 --- a/tests/test_selection/test_check_estimator_selectors.py +++ b/tests/test_selection/test_check_estimator_selectors.py @@ -26,6 +26,7 @@ check_confirm_variables, check_raises_error_if_only_1_variable, ) +from tests.estimator_checks.sklearn_check_wrapper import wrap_for_check_estimator _logreg = LogisticRegression(C=0.0001, max_iter=2, random_state=1) @@ -90,7 +91,10 @@ def test_check_estimator_from_sklearn(estimator): "SelectByInformationValue", ]: failed_tests = estimator._more_tags()["_xfail_checks"] - return check_estimator(estimator=estimator, expected_failed_checks=failed_tests) + return check_estimator( + estimator=wrap_for_check_estimator(estimator), + expected_failed_checks=failed_tests, + ) @pytest.mark.parametrize("estimator", _univariate_estimators) diff --git a/tests/test_time_series/test_forecasting/test_check_estimator_forecasting.py b/tests/test_time_series/test_forecasting/test_check_estimator_forecasting.py index 85a4af38c..bc887f918 100644 --- a/tests/test_time_series/test_forecasting/test_check_estimator_forecasting.py +++ b/tests/test_time_series/test_forecasting/test_check_estimator_forecasting.py @@ -11,6 +11,7 @@ WindowFeatures, ) from tests.estimator_checks.estimator_checks import check_feature_engine_estimator +from tests.estimator_checks.sklearn_check_wrapper import wrap_for_check_estimator _estimators = [ LagFeatures(missing_values="ignore"), @@ -26,7 +27,7 @@ def test_check_estimator_from_sklearn(estimator): "or infinity." } return check_estimator( - estimator=estimator, + estimator=wrap_for_check_estimator(estimator), expected_failed_checks={ **extra_failing_checks, **estimator._more_tags()["_xfail_checks"], diff --git a/tests/test_time_series/test_forecasting/test_window_features.py b/tests/test_time_series/test_forecasting/test_window_features.py index e9701a2ef..1ec83ed8e 100644 --- a/tests/test_time_series/test_forecasting/test_window_features.py +++ b/tests/test_time_series/test_forecasting/test_window_features.py @@ -452,6 +452,43 @@ def test_multiple_windows(df_time): assert df_time_tr.equals(X_tr) +def test_min_periods_is_used(df_time): + # min_periods was accepted and documented but never reached pandas + # rolling(), so the leading rows stayed NaN whatever the user asked for. + variables = ["ambient_temp", "module_temp", "irradiation"] + + transformer = WindowFeatures(window=3, min_periods=1) + df_tr = transformer.fit_transform(df_time) + + expected = ( + df_time[variables].rolling(window=3, min_periods=1).agg("mean").shift(periods=1) + ) + expected.columns = [f"{var}_window_3_mean" for var in variables] + + assert_frame_equal(df_tr[expected.columns], expected) + + # with min_periods=1 only the very first row (shifted out) remains NaN + assert df_tr["ambient_temp_window_3_mean"].isna().sum() == 1 + + # the default is unchanged: pandas requires a full window + df_default = WindowFeatures(window=3).fit_transform(df_time) + assert df_default["ambient_temp_window_3_mean"].isna().sum() == 3 + + +def test_min_periods_is_used_with_multiple_windows(df_time): + transformer = WindowFeatures(window=[2, 3], min_periods=1) + df_tr = transformer.fit_transform(df_time) + + for win in (2, 3): + expected = ( + df_time["ambient_temp"].rolling(window=win, min_periods=1).mean().shift(1) + ) + assert_frame_equal( + df_tr[[f"ambient_temp_window_{win}_mean"]], + expected.to_frame(f"ambient_temp_window_{win}_mean"), + ) + + def test_sort_index(df_time): # Shuffle dataframe Xs = df_time.copy() diff --git a/tests/test_transformation/test_check_estimator_transformers.py b/tests/test_transformation/test_check_estimator_transformers.py index a2add5c4f..738e160fa 100644 --- a/tests/test_transformation/test_check_estimator_transformers.py +++ b/tests/test_transformation/test_check_estimator_transformers.py @@ -18,6 +18,7 @@ from tests.estimator_checks.non_fitted_error_checks import ( check_raises_non_fitted_error_when_fit_fails, ) +from tests.estimator_checks.sklearn_check_wrapper import wrap_for_check_estimator _estimators = [ BoxCoxTransformer(), @@ -66,7 +67,7 @@ def test_check_estimator_from_sklearn(estimator): extra_failing_checks.get(estimator.__class__.__name__, {}) ) return check_estimator( - estimator=estimator, + estimator=wrap_for_check_estimator(estimator), expected_failed_checks=expected_failed_checks, ) diff --git a/tests/test_wrappers/test_check_estimator_wrappers.py b/tests/test_wrappers/test_check_estimator_wrappers.py index cee500d43..75ab4b9c0 100644 --- a/tests/test_wrappers/test_check_estimator_wrappers.py +++ b/tests/test_wrappers/test_check_estimator_wrappers.py @@ -10,6 +10,7 @@ ) from tests.estimator_checks.fit_functionality_checks import check_feature_names_in from tests.estimator_checks.non_fitted_error_checks import check_raises_non_fitted_error +from tests.estimator_checks.sklearn_check_wrapper import wrap_for_check_estimator from tests.estimator_checks.variable_selection_checks import ( check_all_types_variables_assignment, check_numerical_variables_assignment, @@ -18,16 +19,14 @@ def test_sklearn_transformer_wrapper(): check_estimator( - estimator=SklearnWrapper(transformer=SimpleImputer()), - expected_failed_checks=SklearnWrapper( - transformer=SimpleImputer() - )._more_tags()["_xfail_checks"], + estimator=wrap_for_check_estimator(SklearnWrapper(transformer=SimpleImputer())), + expected_failed_checks=SklearnWrapper(transformer=SimpleImputer())._more_tags()[ + "_xfail_checks" + ], ) -@pytest.mark.parametrize( - "estimator", [SklearnWrapper(transformer=OrdinalEncoder())] -) +@pytest.mark.parametrize("estimator", [SklearnWrapper(transformer=OrdinalEncoder())]) def test_check_estimator_from_feature_engine(estimator): check_raises_non_fitted_error(estimator) check_raises_error_when_input_not_a_df(estimator) @@ -35,12 +34,8 @@ def test_check_estimator_from_feature_engine(estimator): def test_check_variables_assignment(): - check_numerical_variables_assignment( - SklearnWrapper(transformer=StandardScaler()) - ) - check_all_types_variables_assignment( - SklearnWrapper(transformer=OrdinalEncoder()) - ) + check_numerical_variables_assignment(SklearnWrapper(transformer=StandardScaler())) + check_all_types_variables_assignment(SklearnWrapper(transformer=OrdinalEncoder())) def test_raises_error_when_no_transformer_passed():