From 7d9fd58bbc9e0e833e8456a46ca98385e9dd3414 Mon Sep 17 00:00:00 2001 From: Simon Koch-Sultan Date: Tue, 17 Nov 2026 12:09:24 -0500 Subject: [PATCH 1/2] Match sample_weight to X by origin label in the exposure methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BornhuetterFerguson, Benktander, ExpectedLoss and CapeCod read sample_weight by position along the origin axis. A weight labelled with other origins paired each origin of X with another origin's exposure, and CapeCod raised KeyError. These methods now reindex the weight to the origins of X when both have more than one origin at the same grain. An origin of X with no weight gets no ultimate, origins found only in the weight are left out, and a warning names both. Closes #1646 Developed with an AI assistant (Árbol, running Muse). --- chainladder/methods/base.py | 64 +++++++++++++++++ chainladder/methods/benktander.py | 4 ++ chainladder/methods/capecod.py | 6 ++ .../methods/tests/test_sample_weight.py | 68 +++++++++++++++++++ 4 files changed, 142 insertions(+) create mode 100644 chainladder/methods/tests/test_sample_weight.py diff --git a/chainladder/methods/base.py b/chainladder/methods/base.py index 543d8c815..685c2de85 100644 --- a/chainladder/methods/base.py +++ b/chainladder/methods/base.py @@ -32,6 +32,70 @@ def _align_cdf(self, X, sample_weight=None): """Vertically align CDF to origin period latest diagonal.""" return X.cdf_.align_pattern(X, sample_weight) + @staticmethod + def _align_weight_origins(X, sample_weight): + """ + Match ``sample_weight`` to the origins of ``X`` by label. + + The methods read ``sample_weight`` by position along the origin axis. + When ``X`` and ``sample_weight`` each have more than one origin at the + same grain and their origins differ, this reindexes the weight to the + origins of ``X``, with NaN for an origin of ``X`` that the weight lacks, + and drops origins found only in the weight. A warning names the origins + found on one side only. + + Returns the weight and the origins of ``X`` that it lacks, which get + no ultimate. + """ + none = pd.DatetimeIndex([]) + if ( + sample_weight is None + or len(X.odims) == 1 + or len(sample_weight.odims) == 1 + or X.origin_grain != sample_weight.origin_grain + ): + return sample_weight, none + x_origins = pd.DatetimeIndex(X.odims) + w_origins = pd.DatetimeIndex(sample_weight.odims) + if x_origins.equals(w_origins): + return sample_weight, none + missing = x_origins.difference(w_origins) + only_x = X.origin[x_origins.isin(missing)].astype(str).tolist() + only_w = sample_weight.origin[~w_origins.isin(x_origins)] + only_w = only_w.astype(str).tolist() + if only_x or only_w: + import warnings + + warnings.warn( + "sample_weight is matched to X by origin. " + f"Origins of X with no weight get no ultimate: {only_x}. " + f"Origins only in sample_weight are left out: {only_w}." + ) + backend = sample_weight.array_backend + obj = sample_weight.set_backend("numpy") + xp = obj.get_array_module() + positions = w_origins.get_indexer(x_origins) + values = xp.full( + obj.shape[:2] + (len(x_origins), obj.shape[3]), xp.nan, dtype=float + ) + found = positions >= 0 + values[:, :, found, :] = obj.values[:, :, positions[found], :] + obj.values = values + obj.odims = X.odims.copy() + return obj.set_backend(backend), missing + + @staticmethod + def _drop_ultimates(ultimate, origins): + """Set ``ultimate`` to NaN at ``origins``, which have no weight.""" + if len(origins) == 0: + return ultimate + backend = ultimate.array_backend + obj = ultimate.set_backend("numpy") + values = obj.values.copy() + values[:, :, pd.DatetimeIndex(obj.odims).isin(origins), :] = float("nan") + obj.values = values + return obj.set_backend(backend) + def _set_ult_attr(self, ultimate): """Ultimate scaffolding""" from chainladder import options diff --git a/chainladder/methods/benktander.py b/chainladder/methods/benktander.py index add140083..e2b5a2262 100644 --- a/chainladder/methods/benktander.py +++ b/chainladder/methods/benktander.py @@ -194,11 +194,13 @@ def fit(self, X, y=None, sample_weight=None): 2008 50708.755370 """ + sample_weight, missing = self._align_weight_origins(X, sample_weight) if sample_weight is None: raise ValueError("sample_weight is required.") super().fit(X, y, sample_weight) self.expectation_ = self._get_benktander_aprioris(X, sample_weight) self.ultimate_ = self._get_ultimate(self.X_, self.expectation_) + self.ultimate_ = self._drop_ultimates(self.ultimate_, missing) self.process_variance_ = self._include_process_variance() return self @@ -250,11 +252,13 @@ def predict(self, X, sample_weight=None): 2012 15914.716737 2013 17193.715555 """ + sample_weight, missing = self._align_weight_origins(X, sample_weight) if sample_weight is None: raise ValueError("sample_weight is required.") X_new = super().predict(X, sample_weight) X_new.expectation_ = self._get_benktander_aprioris(X, sample_weight) X_new.ultimate_ = self._get_ultimate(X_new, X_new.expectation_) + X_new.ultimate_ = self._drop_ultimates(X_new.ultimate_, missing) X_new.n_iters = self.n_iters X_new.apriori = self.apriori return X_new diff --git a/chainladder/methods/capecod.py b/chainladder/methods/capecod.py index 33c40564a..6cf3e4049 100644 --- a/chainladder/methods/capecod.py +++ b/chainladder/methods/capecod.py @@ -237,12 +237,16 @@ def fit(self, X, y=None, sample_weight=None): raise ValueError("sample_weight is required.") self.apriori = 1.0 self.X_ = self.validate_X(X) + sample_weight, missing = self._align_weight_origins(X, sample_weight) sample_weight = sample_weight.set_backend(self.X_.array_backend) self.apriori_, self.detrended_apriori_ = self._get_capecod_aprioris( self.X_, sample_weight ) self.expectation_ = sample_weight * self.detrended_apriori_ super().fit(X, y, self.expectation_) + if len(missing): + self.ultimate_ = self._drop_ultimates(self.ultimate_, missing) + self.process_variance_ = self._include_process_variance() return self def _get_capecod_aprioris(self, X, sample_weight): @@ -333,6 +337,7 @@ def predict(self, X, sample_weight=None): 2012 15991.144199 2013 17489.630279 """ + sample_weight, missing = self._align_weight_origins(X, sample_weight) if sample_weight is None: raise ValueError("sample_weight is required.") X_new = X.copy() @@ -350,6 +355,7 @@ def predict(self, X, sample_weight=None): ) X_new.expectation_ = sample_weight * detrended_apriori_ X_new = super().predict(X_new, X_new.expectation_) + X_new.ultimate_ = self._drop_ultimates(X_new.ultimate_, missing) X_new.apriori_ = apriori_ X_new.detrended_apriori_ = detrended_apriori_ return X_new diff --git a/chainladder/methods/tests/test_sample_weight.py b/chainladder/methods/tests/test_sample_weight.py new file mode 100644 index 000000000..5241701d4 --- /dev/null +++ b/chainladder/methods/tests/test_sample_weight.py @@ -0,0 +1,68 @@ +"""Tests for matching ``sample_weight`` to ``X`` by origin label (#1646).""" + +import numpy as np +import pandas as pd +import pytest + +import chainladder as cl + +raa = cl.load_sample("RAA") + + +def _premium(origins, values=None): + if values is None: + values = [20000.0 + 2000.0 * i for i in range(len(origins))] + df = pd.DataFrame({"origin": origins, "premium": values}) + return cl.Triangle(df, origin="origin", columns="premium", cumulative=True) + + +@pytest.mark.parametrize( + "estimator", + [cl.BornhuetterFerguson, cl.Benktander, cl.ExpectedLoss, cl.CapeCod], +) +def test_sample_weight_is_matched_to_x_by_origin(estimator): + """A weight labelled 1982-1991 on raa (1981-1990) is matched by label.""" + years = list(range(1981, 1991)) + shifted = _premium([y + 1 for y in years]) + # The same premium per label on raa's origins, with none for 1981. + matched = _premium(years, [np.nan] + list(shifted.values[0, 0, :-1, 0])) + expected = estimator().fit(raa, sample_weight=matched).ultimate_ + expected = expected.values[0, 0, 1:, 0] + with pytest.warns(UserWarning, match=r"\['1981'\].*\['1991'\]"): + est = estimator().fit(raa, sample_weight=shifted) + with pytest.warns(UserWarning, match="matched to X by origin"): + predicted = est.predict(raa, sample_weight=shifted) + for ultimate in (est.ultimate_, predicted.ultimate_): + assert np.isnan(ultimate.values[0, 0, 0, 0]) + np.testing.assert_allclose(ultimate.values[0, 0, 1:, 0], expected) + + +def test_bornhuetter_ferguson_uses_the_premium_labelled_with_the_origin(): + """The 1990 ultimate uses the premium labelled 1990, from #1646.""" + years = list(range(1981, 1991)) + shifted = _premium([y + 1 for y in years]) + with pytest.warns(UserWarning): + est = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=shifted) + assert est.ultimate_.values[0, 0, -1, 0] == pytest.approx(21241.54, abs=0.01) + + +def test_sample_weight_matching_works_on_the_sparse_backend(): + """Matching by origin gives the same ultimates on the sparse backend.""" + years = list(range(1981, 1991)) + shifted = _premium([y + 1 for y in years]) + with pytest.warns(UserWarning): + dense = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=shifted) + with pytest.warns(UserWarning): + sparse = cl.BornhuetterFerguson(apriori=0.6).fit( + raa.set_backend("sparse"), sample_weight=shifted.set_backend("sparse") + ) + np.testing.assert_allclose( + sparse.ultimate_.set_backend("numpy").values, dense.ultimate_.values + ) + + +def test_single_origin_sample_weight_broadcasts(): + """A weight with one origin still applies to every origin of X.""" + est = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=_premium([1990])) + assert est.ultimate_.shape == (1, 1, 10, 1) + assert not np.isnan(est.ultimate_.values).any() From 730866c55e07b3e5c690a35a9014777024e56c73 Mon Sep 17 00:00:00 2001 From: Simon Koch-Sultan Date: Wed, 18 Nov 2026 17:53:50 +0000 Subject: [PATCH 2/2] [FIX] Slice sample_weight to the origin range it shares with X MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Following the review on #1648, replace the label matching with a slice. _slice_weight_origins raises when X and sample_weight have different origin grains, keeps the weight from the later of the two first origins to the earlier of the two last origins, and gives NaN weight to origins of X outside that range, which then get no ultimate. It raises a clear error when the ranges share no origin or when the origins in the shared range do not line up, as with the gapped slices of raa in #1649. Developed with an AI assistant (Árbol, running Muse). --- chainladder/methods/base.py | 58 ++++++++------ chainladder/methods/benktander.py | 4 +- chainladder/methods/capecod.py | 4 +- .../methods/tests/test_sample_weight.py | 79 +++++++++++++++---- 4 files changed, 99 insertions(+), 46 deletions(-) diff --git a/chainladder/methods/base.py b/chainladder/methods/base.py index 685c2de85..c909ace9c 100644 --- a/chainladder/methods/base.py +++ b/chainladder/methods/base.py @@ -33,56 +33,64 @@ def _align_cdf(self, X, sample_weight=None): return X.cdf_.align_pattern(X, sample_weight) @staticmethod - def _align_weight_origins(X, sample_weight): + def _slice_weight_origins(X, sample_weight): """ - Match ``sample_weight`` to the origins of ``X`` by label. + Slice ``sample_weight`` to the origin range it shares with ``X``. The methods read ``sample_weight`` by position along the origin axis. - When ``X`` and ``sample_weight`` each have more than one origin at the - same grain and their origins differ, this reindexes the weight to the - origins of ``X``, with NaN for an origin of ``X`` that the weight lacks, - and drops origins found only in the weight. A warning names the origins - found on one side only. + When ``X`` and ``sample_weight`` each have more than one origin, this + keeps the weight from the later of the two first origins to the + earlier of the two last origins, and gives NaN weight to origins of + ``X`` outside that range. A warning names those origins of ``X``. + + Both triangles need consecutive origins at the same origin grain. + A ValueError is raised when the grains differ, when the ranges share + no origin, or when the origins in the shared range do not line up. Returns the weight and the origins of ``X`` that it lacks, which get no ultimate. """ none = pd.DatetimeIndex([]) - if ( - sample_weight is None - or len(X.odims) == 1 - or len(sample_weight.odims) == 1 - or X.origin_grain != sample_weight.origin_grain - ): + if sample_weight is None or len(X.odims) == 1 or len(sample_weight.odims) == 1: return sample_weight, none + if X.origin_grain != sample_weight.origin_grain: + raise ValueError( + f"X has origin grain {X.origin_grain} but sample_weight has " + f"origin grain {sample_weight.origin_grain}. They must match." + ) x_origins = pd.DatetimeIndex(X.odims) w_origins = pd.DatetimeIndex(sample_weight.odims) if x_origins.equals(w_origins): return sample_weight, none - missing = x_origins.difference(w_origins) - only_x = X.origin[x_origins.isin(missing)].astype(str).tolist() - only_w = sample_weight.origin[~w_origins.isin(x_origins)] - only_w = only_w.astype(str).tolist() - if only_x or only_w: + start = max(x_origins[0], w_origins[0]) + end = min(x_origins[-1], w_origins[-1]) + x_shared = (x_origins >= start) & (x_origins <= end) + w_shared = (w_origins >= start) & (w_origins <= end) + if not x_shared.any(): + raise ValueError("X and sample_weight share no origin.") + if not x_origins[x_shared].equals(w_origins[w_shared]): + raise ValueError( + "The origins of X and sample_weight do not line up in the range " + "they share. Both need consecutive origins." + ) + if not x_shared.all(): import warnings warnings.warn( - "sample_weight is matched to X by origin. " - f"Origins of X with no weight get no ultimate: {only_x}. " - f"Origins only in sample_weight are left out: {only_w}." + "sample_weight is sliced to the origins it shares with X. " + "Origins of X with no weight get no ultimate: " + f"{X.origin[~x_shared].astype(str).tolist()}." ) backend = sample_weight.array_backend obj = sample_weight.set_backend("numpy") xp = obj.get_array_module() - positions = w_origins.get_indexer(x_origins) values = xp.full( obj.shape[:2] + (len(x_origins), obj.shape[3]), xp.nan, dtype=float ) - found = positions >= 0 - values[:, :, found, :] = obj.values[:, :, positions[found], :] + values[:, :, x_shared, :] = obj.values[:, :, w_shared, :] obj.values = values obj.odims = X.odims.copy() - return obj.set_backend(backend), missing + return obj.set_backend(backend), x_origins[~x_shared] @staticmethod def _drop_ultimates(ultimate, origins): diff --git a/chainladder/methods/benktander.py b/chainladder/methods/benktander.py index e2b5a2262..f94dfe39f 100644 --- a/chainladder/methods/benktander.py +++ b/chainladder/methods/benktander.py @@ -194,7 +194,7 @@ def fit(self, X, y=None, sample_weight=None): 2008 50708.755370 """ - sample_weight, missing = self._align_weight_origins(X, sample_weight) + sample_weight, missing = self._slice_weight_origins(X, sample_weight) if sample_weight is None: raise ValueError("sample_weight is required.") super().fit(X, y, sample_weight) @@ -252,7 +252,7 @@ def predict(self, X, sample_weight=None): 2012 15914.716737 2013 17193.715555 """ - sample_weight, missing = self._align_weight_origins(X, sample_weight) + sample_weight, missing = self._slice_weight_origins(X, sample_weight) if sample_weight is None: raise ValueError("sample_weight is required.") X_new = super().predict(X, sample_weight) diff --git a/chainladder/methods/capecod.py b/chainladder/methods/capecod.py index 6cf3e4049..68005163f 100644 --- a/chainladder/methods/capecod.py +++ b/chainladder/methods/capecod.py @@ -237,7 +237,7 @@ def fit(self, X, y=None, sample_weight=None): raise ValueError("sample_weight is required.") self.apriori = 1.0 self.X_ = self.validate_X(X) - sample_weight, missing = self._align_weight_origins(X, sample_weight) + sample_weight, missing = self._slice_weight_origins(X, sample_weight) sample_weight = sample_weight.set_backend(self.X_.array_backend) self.apriori_, self.detrended_apriori_ = self._get_capecod_aprioris( self.X_, sample_weight @@ -337,7 +337,7 @@ def predict(self, X, sample_weight=None): 2012 15991.144199 2013 17489.630279 """ - sample_weight, missing = self._align_weight_origins(X, sample_weight) + sample_weight, missing = self._slice_weight_origins(X, sample_weight) if sample_weight is None: raise ValueError("sample_weight is required.") X_new = X.copy() diff --git a/chainladder/methods/tests/test_sample_weight.py b/chainladder/methods/tests/test_sample_weight.py index 5241701d4..9409b3d74 100644 --- a/chainladder/methods/tests/test_sample_weight.py +++ b/chainladder/methods/tests/test_sample_weight.py @@ -1,4 +1,4 @@ -"""Tests for matching ``sample_weight`` to ``X`` by origin label (#1646).""" +"""Tests for slicing ``sample_weight`` to the origins it shares with ``X`` (#1646).""" import numpy as np import pandas as pd @@ -7,30 +7,35 @@ import chainladder as cl raa = cl.load_sample("RAA") +years = list(range(1981, 1991)) def _premium(origins, values=None): if values is None: - values = [20000.0 + 2000.0 * i for i in range(len(origins))] - df = pd.DataFrame({"origin": origins, "premium": values}) + values = [20000.0 + 2000.0 * (o - 1982) for o in origins] + df = pd.DataFrame({"origin": [str(o) for o in origins], "premium": values}) return cl.Triangle(df, origin="origin", columns="premium", cumulative=True) +def _same_labels(origins): + """The premium of ``_premium`` on raa's origins, NaN outside ``origins``.""" + values = [20000.0 + 2000.0 * (y - 1982) if y in origins else np.nan for y in years] + return _premium(years, values) + + @pytest.mark.parametrize( "estimator", [cl.BornhuetterFerguson, cl.Benktander, cl.ExpectedLoss, cl.CapeCod], ) -def test_sample_weight_is_matched_to_x_by_origin(estimator): - """A weight labelled 1982-1991 on raa (1981-1990) is matched by label.""" - years = list(range(1981, 1991)) - shifted = _premium([y + 1 for y in years]) - # The same premium per label on raa's origins, with none for 1981. - matched = _premium(years, [np.nan] + list(shifted.values[0, 0, :-1, 0])) +def test_shifted_sample_weight_is_sliced_to_the_shared_origins(estimator): + """A weight labelled 1982-1991 on raa (1981-1990) uses 1982-1990.""" + shifted = _premium(range(1982, 1992)) + matched = _same_labels(range(1982, 1991)) expected = estimator().fit(raa, sample_weight=matched).ultimate_ expected = expected.values[0, 0, 1:, 0] - with pytest.warns(UserWarning, match=r"\['1981'\].*\['1991'\]"): + with pytest.warns(UserWarning, match=r"no ultimate: \['1981'\]"): est = estimator().fit(raa, sample_weight=shifted) - with pytest.warns(UserWarning, match="matched to X by origin"): + with pytest.warns(UserWarning, match="sliced to the origins it shares"): predicted = est.predict(raa, sample_weight=shifted) for ultimate in (est.ultimate_, predicted.ultimate_): assert np.isnan(ultimate.values[0, 0, 0, 0]) @@ -39,17 +44,57 @@ def test_sample_weight_is_matched_to_x_by_origin(estimator): def test_bornhuetter_ferguson_uses_the_premium_labelled_with_the_origin(): """The 1990 ultimate uses the premium labelled 1990, from #1646.""" - years = list(range(1981, 1991)) - shifted = _premium([y + 1 for y in years]) + shifted = _premium(range(1982, 1992)) with pytest.warns(UserWarning): est = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=shifted) assert est.ultimate_.values[0, 0, -1, 0] == pytest.approx(21241.54, abs=0.01) -def test_sample_weight_matching_works_on_the_sparse_backend(): - """Matching by origin gives the same ultimates on the sparse backend.""" - years = list(range(1981, 1991)) - shifted = _premium([y + 1 for y in years]) +def test_longer_sample_weight_is_sliced_to_the_origins_of_x(recwarn): + """A weight labelled 1979-1992 gives the same ultimates as one on 1981-1990.""" + longer = cl.BornhuetterFerguson(apriori=0.6).fit( + raa, sample_weight=_premium(range(1979, 1993)) + ) + same = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=_premium(years)) + np.testing.assert_allclose(longer.ultimate_.values, same.ultimate_.values) + assert not [w for w in recwarn if "sliced" in str(w.message)] + + +def test_shorter_sample_weight_leaves_origins_without_weight_with_no_ultimate(): + """A weight labelled 1983-1988 gives no ultimate for 1981, 1982, 1989, 1990.""" + expected = cl.BornhuetterFerguson(apriori=0.6).fit( + raa, sample_weight=_same_labels(range(1983, 1989)) + ) + with pytest.warns(UserWarning, match=r"\['1981', '1982', '1989', '1990'\]"): + est = cl.BornhuetterFerguson(apriori=0.6).fit( + raa, sample_weight=_premium(range(1983, 1989)) + ) + ultimate = est.ultimate_.values[0, 0, :, 0] + assert np.isnan(ultimate[[0, 1, 8, 9]]).all() + np.testing.assert_allclose(ultimate[2:8], expected.ultimate_.values[0, 0, 2:8, 0]) + + +def test_sample_weight_with_another_origin_grain_raises(): + """A quarterly weight on annual raa raises instead of lining up by position.""" + origins = pd.date_range("1981-01-01", periods=40, freq="QS") + df = pd.DataFrame({"origin": origins, "premium": 5000.0}) + quarterly = cl.Triangle(df, origin="origin", columns="premium", cumulative=True) + with pytest.raises(ValueError, match="origin grain"): + cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=quarterly) + + +def test_origins_that_do_not_line_up_raise(): + """Odd-year X with an even-year weight raises, as with #1649's slices of raa.""" + odd = raa[raa.origin.year.isin([1981, 1983, 1985, 1987, 1989])] + even = _premium(years) + even = even[even.origin.year.isin([1982, 1984, 1986, 1988, 1990])] + with pytest.raises(ValueError, match="do not line up"): + cl.BornhuetterFerguson(apriori=0.6).fit(odd, sample_weight=even) + + +def test_sample_weight_slicing_works_on_the_sparse_backend(): + """Slicing gives the same ultimates on the sparse backend.""" + shifted = _premium(range(1982, 1992)) with pytest.warns(UserWarning): dense = cl.BornhuetterFerguson(apriori=0.6).fit(raa, sample_weight=shifted) with pytest.warns(UserWarning):