diff --git a/chainladder/methods/base.py b/chainladder/methods/base.py index 543d8c815..c909ace9c 100644 --- a/chainladder/methods/base.py +++ b/chainladder/methods/base.py @@ -32,6 +32,78 @@ def _align_cdf(self, X, sample_weight=None): """Vertically align CDF to origin period latest diagonal.""" return X.cdf_.align_pattern(X, sample_weight) + @staticmethod + def _slice_weight_origins(X, sample_weight): + """ + Slice ``sample_weight`` to the origin range it shares with ``X``. + + The methods read ``sample_weight`` by position along the origin axis. + When ``X`` and ``sample_weight`` each have more than one origin, this + keeps the weight from the later of the two first origins to the + earlier of the two last origins, and gives NaN weight to origins of + ``X`` outside that range. A warning names those origins of ``X``. + + Both triangles need consecutive origins at the same origin grain. + A ValueError is raised when the grains differ, when the ranges share + no origin, or when the origins in the shared range do not line up. + + Returns the weight and the origins of ``X`` that it lacks, which get + no ultimate. + """ + none = pd.DatetimeIndex([]) + if sample_weight is None or len(X.odims) == 1 or len(sample_weight.odims) == 1: + return sample_weight, none + if X.origin_grain != sample_weight.origin_grain: + raise ValueError( + f"X has origin grain {X.origin_grain} but sample_weight has " + f"origin grain {sample_weight.origin_grain}. They must match." + ) + x_origins = pd.DatetimeIndex(X.odims) + w_origins = pd.DatetimeIndex(sample_weight.odims) + if x_origins.equals(w_origins): + return sample_weight, none + start = max(x_origins[0], w_origins[0]) + end = min(x_origins[-1], w_origins[-1]) + x_shared = (x_origins >= start) & (x_origins <= end) + w_shared = (w_origins >= start) & (w_origins <= end) + if not x_shared.any(): + raise ValueError("X and sample_weight share no origin.") + if not x_origins[x_shared].equals(w_origins[w_shared]): + raise ValueError( + "The origins of X and sample_weight do not line up in the range " + "they share. Both need consecutive origins." + ) + if not x_shared.all(): + import warnings + + warnings.warn( + "sample_weight is sliced to the origins it shares with X. " + "Origins of X with no weight get no ultimate: " + f"{X.origin[~x_shared].astype(str).tolist()}." + ) + backend = sample_weight.array_backend + obj = sample_weight.set_backend("numpy") + xp = obj.get_array_module() + values = xp.full( + obj.shape[:2] + (len(x_origins), obj.shape[3]), xp.nan, dtype=float + ) + values[:, :, x_shared, :] = obj.values[:, :, w_shared, :] + obj.values = values + obj.odims = X.odims.copy() + return obj.set_backend(backend), x_origins[~x_shared] + + @staticmethod + def _drop_ultimates(ultimate, origins): + """Set ``ultimate`` to NaN at ``origins``, which have no weight.""" + if len(origins) == 0: + return ultimate + backend = ultimate.array_backend + obj = ultimate.set_backend("numpy") + values = obj.values.copy() + values[:, :, pd.DatetimeIndex(obj.odims).isin(origins), :] = float("nan") + obj.values = values + return obj.set_backend(backend) + def _set_ult_attr(self, ultimate): """Ultimate scaffolding""" from chainladder import options diff --git a/chainladder/methods/benktander.py b/chainladder/methods/benktander.py index add140083..f94dfe39f 100644 --- a/chainladder/methods/benktander.py +++ b/chainladder/methods/benktander.py @@ -194,11 +194,13 @@ def fit(self, X, y=None, sample_weight=None): 2008 50708.755370 """ + sample_weight, missing = self._slice_weight_origins(X, sample_weight) if sample_weight is None: raise ValueError("sample_weight is required.") super().fit(X, y, sample_weight) self.expectation_ = self._get_benktander_aprioris(X, sample_weight) self.ultimate_ = self._get_ultimate(self.X_, self.expectation_) + self.ultimate_ = self._drop_ultimates(self.ultimate_, missing) self.process_variance_ = self._include_process_variance() return self @@ -250,11 +252,13 @@ def predict(self, X, sample_weight=None): 2012 15914.716737 2013 17193.715555 """ + sample_weight, missing = self._slice_weight_origins(X, sample_weight) if sample_weight is None: raise ValueError("sample_weight is required.") X_new = super().predict(X, sample_weight) X_new.expectation_ = self._get_benktander_aprioris(X, sample_weight) X_new.ultimate_ = self._get_ultimate(X_new, X_new.expectation_) + X_new.ultimate_ = self._drop_ultimates(X_new.ultimate_, missing) X_new.n_iters = self.n_iters X_new.apriori = self.apriori return X_new diff --git a/chainladder/methods/capecod.py b/chainladder/methods/capecod.py index 33c40564a..68005163f 100644 --- a/chainladder/methods/capecod.py +++ b/chainladder/methods/capecod.py @@ -237,12 +237,16 @@ def fit(self, X, y=None, sample_weight=None): raise ValueError("sample_weight is required.") self.apriori = 1.0 self.X_ = self.validate_X(X) + sample_weight, missing = self._slice_weight_origins(X, sample_weight) sample_weight = sample_weight.set_backend(self.X_.array_backend) self.apriori_, self.detrended_apriori_ = self._get_capecod_aprioris( self.X_, sample_weight ) self.expectation_ = sample_weight * self.detrended_apriori_ super().fit(X, y, self.expectation_) + if len(missing): + self.ultimate_ = self._drop_ultimates(self.ultimate_, missing) + self.process_variance_ = self._include_process_variance() return self def _get_capecod_aprioris(self, X, sample_weight): @@ -333,6 +337,7 @@ def predict(self, X, sample_weight=None): 2012 15991.144199 2013 17489.630279 """ + sample_weight, missing = self._slice_weight_origins(X, sample_weight) if sample_weight is None: raise ValueError("sample_weight is required.") X_new = X.copy() @@ -350,6 +355,7 @@ def predict(self, X, sample_weight=None): ) X_new.expectation_ = sample_weight * detrended_apriori_ X_new = super().predict(X_new, X_new.expectation_) + X_new.ultimate_ = self._drop_ultimates(X_new.ultimate_, missing) X_new.apriori_ = apriori_ X_new.detrended_apriori_ = detrended_apriori_ return X_new diff --git a/chainladder/methods/tests/test_sample_weight.py b/chainladder/methods/tests/test_sample_weight.py new file mode 100644 index 000000000..9409b3d74 --- /dev/null +++ b/chainladder/methods/tests/test_sample_weight.py @@ -0,0 +1,113 @@ +"""Tests for slicing ``sample_weight`` to the origins it shares with ``X`` (#1646).""" + +import numpy as np +import pandas as pd +import pytest + +import chainladder as cl + +raa = cl.load_sample("RAA") +years = list(range(1981, 1991)) + + +def _premium(origins, values=None): + if values is None: + values = [20000.0 + 2000.0 * (o - 1982) for o in origins] + df = pd.DataFrame({"origin": [str(o) for o in origins], "premium": values}) + return cl.Triangle(df, origin="origin", columns="premium", cumulative=True) + + +def _same_labels(origins): + """The premium of ``_premium`` on raa's origins, NaN outside ``origins``.""" + values = [20000.0 + 2000.0 * (y - 1982) if y in origins else np.nan for y in years] + return _premium(years, values) + + +@pytest.mark.parametrize( + "estimator", + [cl.BornhuetterFerguson, cl.Benktander, cl.ExpectedLoss, cl.CapeCod], +) +def test_shifted_sample_weight_is_sliced_to_the_shared_origins(estimator): + """A weight labelled 1982-1991 on raa (1981-1990) uses 1982-1990.""" + shifted = _premium(range(1982, 1992)) + matched = _same_labels(range(1982, 1991)) + expected = estimator().fit(raa, sample_weight=matched).ultimate_ + expected = expected.values[0, 0, 1:, 0] + with pytest.warns(UserWarning, match=r"no ultimate: \['1981'\]"): + est = estimator().fit(raa, sample_weight=shifted) + with pytest.warns(UserWarning, match="sliced to the origins it shares"): + predicted = est.predict(raa, sample_weight=shifted) + for ultimate in (est.ultimate_, predicted.ultimate_): + assert np.isnan(ultimate.values[0, 0, 0, 0]) + np.testing.assert_allclose(ultimate.values[0, 0, 1:, 0], expected) + + +def test_bornhuetter_ferguson_uses_the_premium_labelled_with_the_origin(): + """The 1990 ultimate uses the premium labelled 1990, from #1646.""" + shifted = _premium(range(1982, 1992)) + with pytest.warns(UserWarning): + est = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=shifted) + assert est.ultimate_.values[0, 0, -1, 0] == pytest.approx(21241.54, abs=0.01) + + +def test_longer_sample_weight_is_sliced_to_the_origins_of_x(recwarn): + """A weight labelled 1979-1992 gives the same ultimates as one on 1981-1990.""" + longer = cl.BornhuetterFerguson(apriori=0.6).fit( + raa, sample_weight=_premium(range(1979, 1993)) + ) + same = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=_premium(years)) + np.testing.assert_allclose(longer.ultimate_.values, same.ultimate_.values) + assert not [w for w in recwarn if "sliced" in str(w.message)] + + +def test_shorter_sample_weight_leaves_origins_without_weight_with_no_ultimate(): + """A weight labelled 1983-1988 gives no ultimate for 1981, 1982, 1989, 1990.""" + expected = cl.BornhuetterFerguson(apriori=0.6).fit( + raa, sample_weight=_same_labels(range(1983, 1989)) + ) + with pytest.warns(UserWarning, match=r"\['1981', '1982', '1989', '1990'\]"): + est = cl.BornhuetterFerguson(apriori=0.6).fit( + raa, sample_weight=_premium(range(1983, 1989)) + ) + ultimate = est.ultimate_.values[0, 0, :, 0] + assert np.isnan(ultimate[[0, 1, 8, 9]]).all() + np.testing.assert_allclose(ultimate[2:8], expected.ultimate_.values[0, 0, 2:8, 0]) + + +def test_sample_weight_with_another_origin_grain_raises(): + """A quarterly weight on annual raa raises instead of lining up by position.""" + origins = pd.date_range("1981-01-01", periods=40, freq="QS") + df = pd.DataFrame({"origin": origins, "premium": 5000.0}) + quarterly = cl.Triangle(df, origin="origin", columns="premium", cumulative=True) + with pytest.raises(ValueError, match="origin grain"): + cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=quarterly) + + +def test_origins_that_do_not_line_up_raise(): + """Odd-year X with an even-year weight raises, as with #1649's slices of raa.""" + odd = raa[raa.origin.year.isin([1981, 1983, 1985, 1987, 1989])] + even = _premium(years) + even = even[even.origin.year.isin([1982, 1984, 1986, 1988, 1990])] + with pytest.raises(ValueError, match="do not line up"): + cl.BornhuetterFerguson(apriori=0.6).fit(odd, sample_weight=even) + + +def test_sample_weight_slicing_works_on_the_sparse_backend(): + """Slicing gives the same ultimates on the sparse backend.""" + shifted = _premium(range(1982, 1992)) + with pytest.warns(UserWarning): + dense = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=shifted) + with pytest.warns(UserWarning): + sparse = cl.BornhuetterFerguson(apriori=0.6).fit( + raa.set_backend("sparse"), sample_weight=shifted.set_backend("sparse") + ) + np.testing.assert_allclose( + sparse.ultimate_.set_backend("numpy").values, dense.ultimate_.values + ) + + +def test_single_origin_sample_weight_broadcasts(): + """A weight with one origin still applies to every origin of X.""" + est = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=_premium([1990])) + assert est.ultimate_.shape == (1, 1, 10, 1) + assert not np.isnan(est.ultimate_.values).any()