diff --git a/chainladder/core/pandas.py b/chainladder/core/pandas.py index ecab025f2..127bf846f 100644 --- a/chainladder/core/pandas.py +++ b/chainladder/core/pandas.py @@ -682,11 +682,12 @@ def dropna(self) -> Triangle: if obj.shape[-1] != 1: # Flag the development periods that have data. ddim = list((xp.nansum(obj.values[0, 0, :], -2) != 0).astype("int")) - ddim = obj.development[pd.Series(ddim).astype(bool)] + # Find the first and last development periods with data by position. Pattern labels + # such as "12-24" are strings, so comparing labels would sort "120-132" before "24-36". + min_ddim = ddim.index(1) + max_ddim = len(ddim) - 1 - ddim[::-1].index(1) # Slice the Triangle by the development periods that have data. - obj = self[ - (self.development >= ddim.min()) & (self.development <= ddim.max()) - ] + obj = self.iloc[..., min_ddim : max_ddim + 1] obj = cast("TriangleProtocol", cast(object, obj)) # Slice the triangle by the origin periods that have data. return obj[(self.origin >= min_odim) & (self.origin <= max_odim)] diff --git a/chainladder/core/slice.py b/chainladder/core/slice.py index 8cde99684..b440c19d7 100644 --- a/chainladder/core/slice.py +++ b/chainladder/core/slice.py @@ -36,7 +36,11 @@ class _LocBase: def __init__(self, obj: TriangleProtocol): self.obj = obj - def get_idx(self, idx: tuple[_AxisKey, _AxisKey, _AxisKey, _AxisKey]) -> Triangle: + def get_idx( + self, + idx: tuple[_AxisKey, _AxisKey, _AxisKey, _AxisKey], + allow_gaps: bool = False, + ) -> Triangle: """ Returns a slice of the original Triangle @@ -44,6 +48,10 @@ def get_idx(self, idx: tuple[_AxisKey, _AxisKey, _AxisKey, _AxisKey]) -> Triangl ---------- idx: tuple The index to slice on. + allow_gaps: bool + Accept stepped slices on origin and development. Library code that builds a coarser + development grain or keeps a valuation diagonal sets this. User-facing indexers leave it + False, so a stepped pick raises. Returns ------- @@ -58,6 +66,10 @@ def get_idx(self, idx: tuple[_AxisKey, _AxisKey, _AxisKey, _AxisKey]) -> Triangl d_idx: slice | np.ndarray = _LocBase._contig_slice(idx[3]) if type(o_idx) is not slice or type(d_idx) is not slice: raise ValueError("Fancy indexing on origin/development is not supported.") + if not allow_gaps and ( + not _LocBase._is_unit_slice(o_idx) or not _LocBase._is_unit_slice(d_idx) + ): + raise ValueError("Fancy indexing on origin/development is not supported.") if type(i_idx) is slice or type(c_idx) is slice: obj.values = obj.values[i_idx, c_idx, o_idx, d_idx] # Case when index and column indexing expressions are arrays. @@ -77,6 +89,25 @@ def get_idx(self, idx: tuple[_AxisKey, _AxisKey, _AxisKey, _AxisKey]) -> Triangl ) return obj + @staticmethod + def _is_unit_slice(key: object) -> bool: + """ + Check whether an origin or development key keeps a contiguous run of periods. + + Parameters + ---------- + key: object + The key after conversion by ``_contig_slice``. + + Returns + ------- + bool + True when ``key`` is a slice with a step of None, 1 or -1. A larger step would leave + gaps between periods, so it returns False. A step of -1 keeps adjacent periods in + reverse order, which ``sort_axis`` can undo. + """ + return isinstance(key, slice) and key.step in (None, 1, -1) + @staticmethod def _contig_slice(arr: _AxisKey) -> slice | np.ndarray: """ @@ -152,7 +183,9 @@ def __setitem__( for item in contig_key ]) norm_key = self._normalize_index(tuple_key) - if type(norm_key[2]) is not slice or type(norm_key[3]) is not slice: + if not _LocBase._is_unit_slice(norm_key[2]) or not _LocBase._is_unit_slice( + norm_key[3] + ): raise ValueError( "Setting while fancy indexing on origin/development is not supported." ) @@ -990,12 +1023,15 @@ def _slice_valuation(self: TriangleProtocol, key: np.ndarray) -> Triangle: "BackendArray", num_to_nan(obj.values * obj.get_array_module().array(key)) ) # Recalculate size of the origin and development axes and return the slice. - return _LocBase(obj).get_idx(( - slice(None), - slice(None), - np.arange(obj.shape[2])[np.sum(~key, 1) != obj.shape[3]], - np.arange(obj.shape[3])[np.sum(~key, 0) != obj.shape[2]], - )) + return _LocBase(obj).get_idx( + ( + slice(None), + slice(None), + np.arange(obj.shape[2])[np.sum(~key, 1) != obj.shape[3]], + np.arange(obj.shape[3])[np.sum(~key, 0) != obj.shape[2]], + ), + allow_gaps=True, + ) def _slice( self: TriangleProtocol, @@ -1013,11 +1049,15 @@ def _slice( The axis to which the slice would apply, `ddims` for development, `odims` for origin. """ + # noinspection PyProtectedMember + axis_slice = _LocBase._contig_slice(np.arange(len(key))[key]) + # noinspection PyProtectedMember + if not _LocBase._is_unit_slice(axis_slice): + raise ValueError("Fancy indexing on origin/development is not supported.") # Update the axis, then the values. obj = self.copy() setattr(obj, axis, getattr(obj, axis)[key]) - # noinspection PyProtectedMember - slicer = ..., _LocBase._contig_slice(np.arange(len(key))[key]) + slicer = ..., axis_slice if axis == "odims": slicer = tuple(list(slicer) + [slice(None)]) obj.values = obj.values[slicer] diff --git a/chainladder/core/tests/test_slicing.py b/chainladder/core/tests/test_slicing.py index a58485ccb..de604eb02 100644 --- a/chainladder/core/tests/test_slicing.py +++ b/chainladder/core/tests/test_slicing.py @@ -458,6 +458,149 @@ def test_set_fancy_development_raises(raa: Triangle) -> None: raa_copy.loc["Total", "values", :, [12, 24, 48]] = raa.iloc[0, 0, :, :3] +@pytest.mark.parametrize( + "origin_key", + [slice(None, None, 2), [0, 2, 4], [4, 2, 0]], + ids=["step_2", "evenly_spaced_list", "descending_list"], +) +def test_get_idx_stepped_origin_raises(raa: Triangle, origin_key) -> None: + """ + Attempt stepped indexing on the origin axis, raise an error. Evenly spaced positions convert to + a stepped slice, which would leave gaps between origin periods (#1649). + + Parameters + ---------- + raa: Triangle + The raa sample data set fixture. + origin_key: slice | list + A stepped origin key. + + Returns + ------- + None + + """ + with pytest.raises( + ValueError, match="Fancy indexing on origin/development is not supported" + ): + _ = raa.iloc[..., origin_key, :] + + +@pytest.mark.parametrize( + "development_key", + [slice(None, None, 2), [0, 2, 4]], + ids=["step_2", "evenly_spaced_list"], +) +def test_get_idx_stepped_development_raises(raa: Triangle, development_key) -> None: + """ + Attempt stepped indexing on the development axis, raise an error (#1649). + + Parameters + ---------- + raa: Triangle + The raa sample data set fixture. + development_key: slice | list + A stepped development key. + + Returns + ------- + None + + """ + with pytest.raises( + ValueError, match="Fancy indexing on origin/development is not supported" + ): + _ = raa.iloc[..., :, development_key] + + +def test_non_contiguous_boolean_mask_raises(raa: Triangle) -> None: + """ + Filter origin and development with boolean masks that keep non-adjacent periods, raise an error. + A contiguous mask still works (#1649). + + Parameters + ---------- + raa: Triangle + The raa sample data set fixture. + + Returns + ------- + None + + """ + with pytest.raises( + ValueError, match="Fancy indexing on origin/development is not supported" + ): + _ = raa[raa.origin.year % 2 == 0] + with pytest.raises( + ValueError, match="Fancy indexing on origin/development is not supported" + ): + _ = raa[raa.origin != "1985"] + with pytest.raises( + ValueError, match="Fancy indexing on origin/development is not supported" + ): + _ = raa[raa.development % 24 == 0] + assert raa[raa.origin >= "1985"].shape == (1, 1, 6, 10) + + +def test_set_stepped_origin_development_raises(raa: Triangle) -> None: + """ + Attempt setting with stepped indexing on the origin and development axes, raise an error (#1649). + + Parameters + ---------- + raa: Triangle + The raa sample data set fixture. + + Returns + ------- + None + + """ + raa_copy = raa.copy() + if raa.array_backend == "numpy": + # Test is specific to the numpy backend. + with pytest.raises( + ValueError, + match="Setting while fancy indexing on origin/development is not supported.", + ): + raa_copy.iloc[0, 0, [0, 2, 4], :] = raa.iloc[0, 0, :3, :] + with pytest.raises( + ValueError, + match="Setting while fancy indexing on origin/development is not supported.", + ): + raa_copy.iloc[0, 0, :, ::2] = raa.iloc[0, 0, :, :5] + + +def test_reversed_and_coarser_grain_still_work(raa: Triangle) -> None: + """ + Reverse the origin and development axes and coarsen the development grain. Each keeps working + after stepped slices started to raise (#1649). + + Parameters + ---------- + raa: Triangle + The raa sample data set fixture. + + Returns + ------- + None + + """ + reversed_tr = raa.iloc[..., ::-1, ::-1] + assert reversed_tr.shape == raa.shape + assert list(reversed_tr.origin.year) == list(raa.origin.year)[::-1] + assert raa.latest_diagonal.shape == (1, 1, 10, 1) + quarterly = cl.load_sample("quarterly") + yearly = quarterly.grain("OYDY") + assert yearly.shape == (1, 2, 12, 12) + assert list(yearly.ddims) == list(range(3, 136, 12)) + keep = np.isin(quarterly.ddims, yearly.ddims) + np.testing.assert_allclose( + np.nan_to_num(yearly.values), np.nan_to_num(quarterly.values[..., keep]) + ) + + def test_get_idx_non_contiguous_index_and_columns(clrd: Triangle) -> None: """ Pass lists of non-contiguous index expressions of the index and column axes to Triangle.iloc. Check the values diff --git a/chainladder/core/triangle.py b/chainladder/core/triangle.py index bb76492a5..75fdd8d50 100644 --- a/chainladder/core/triangle.py +++ b/chainladder/core/triangle.py @@ -2023,7 +2023,12 @@ def grain(self, grain="", trailing=False, inplace=False): ) if obj.is_cumulative: - obj = obj.iloc[..., d] + # Keeping every step-th age is how a cumulative triangle moves to a coarser + # development grain, so this pick may be a stepped slice. + # noinspection PyProtectedMember + obj = obj.iloc.get_idx( + obj.iloc._normalize_index((..., d)), allow_gaps=True + ) else: ddims = obj.ddims[d] d2 = [d[0]] * (d[0] + 1) + list(np.repeat(np.array(d[1:]), step)) diff --git a/chainladder/utils/tests/test_utilities.py b/chainladder/utils/tests/test_utilities.py index 041073152..adf7f67bd 100644 --- a/chainladder/utils/tests/test_utilities.py +++ b/chainladder/utils/tests/test_utilities.py @@ -102,6 +102,19 @@ def test_json_df(): assert abs(cl.read_json(x.to_json()).lambda_ - x.lambda_).sum() < 1e-5 +def test_json_pattern_keeps_development(): + """ + Round-trip a fitted MunichAdjustment through JSON. dropna trims the development patterns by + position, so the string labels of the patterns keep their order and no gap appears (#1649). + """ + x = cl.MunichAdjustment(paid_to_incurred=("paid", "incurred")).fit_transform( + cl.load_sample("mcl") + ) + y = cl.read_json(x.to_json()) + assert list(y.ldf_.development) == list(x.ldf_.development) + assert y.ldf_.shape == x.ldf_.shape + + def test_read_csv_single(raa): # Test the read_csv function for a single dimensional input.