Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions chainladder/core/pandas.py
Original file line number Diff line number Diff line change
Expand Up @@ -682,11 +682,12 @@ def dropna(self) -> Triangle:
if obj.shape[-1] != 1:
# Flag the development periods that have data.
ddim = list((xp.nansum(obj.values[0, 0, :], -2) != 0).astype("int"))
ddim = obj.development[pd.Series(ddim).astype(bool)]
# Find the first and last development periods with data by position. Pattern labels
# such as "12-24" are strings, so comparing labels would sort "120-132" before "24-36".
min_ddim = ddim.index(1)
max_ddim = len(ddim) - 1 - ddim[::-1].index(1)
# Slice the Triangle by the development periods that have data.
obj = self[
(self.development >= ddim.min()) & (self.development <= ddim.max())
]
obj = self.iloc[..., min_ddim : max_ddim + 1]
obj = cast("TriangleProtocol", cast(object, obj))
# Slice the triangle by the origin periods that have data.
return obj[(self.origin >= min_odim) & (self.origin <= max_odim)]
Expand Down
60 changes: 50 additions & 10 deletions chainladder/core/slice.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,22 @@ class _LocBase:
def __init__(self, obj: TriangleProtocol):
self.obj = obj

def get_idx(self, idx: tuple[_AxisKey, _AxisKey, _AxisKey, _AxisKey]) -> Triangle:
def get_idx(
self,
idx: tuple[_AxisKey, _AxisKey, _AxisKey, _AxisKey],
allow_gaps: bool = False,
) -> Triangle:
"""
Returns a slice of the original Triangle

Parameters
----------
idx: tuple
The index to slice on.
allow_gaps: bool
Accept stepped slices on origin and development. Library code that builds a coarser
development grain or keeps a valuation diagonal sets this. User-facing indexers leave it
False, so a stepped pick raises.

Returns
-------
Expand All @@ -58,6 +66,10 @@ def get_idx(self, idx: tuple[_AxisKey, _AxisKey, _AxisKey, _AxisKey]) -> Triangl
d_idx: slice | np.ndarray = _LocBase._contig_slice(idx[3])
if type(o_idx) is not slice or type(d_idx) is not slice:
raise ValueError("Fancy indexing on origin/development is not supported.")
if not allow_gaps and (
not _LocBase._is_unit_slice(o_idx) or not _LocBase._is_unit_slice(d_idx)
):
raise ValueError("Fancy indexing on origin/development is not supported.")
if type(i_idx) is slice or type(c_idx) is slice:
obj.values = obj.values[i_idx, c_idx, o_idx, d_idx]
# Case when index and column indexing expressions are arrays.
Expand All @@ -77,6 +89,25 @@ def get_idx(self, idx: tuple[_AxisKey, _AxisKey, _AxisKey, _AxisKey]) -> Triangl
)
return obj

@staticmethod
def _is_unit_slice(key: object) -> bool:
"""
Check whether an origin or development key keeps a contiguous run of periods.

Parameters
----------
key: object
The key after conversion by ``_contig_slice``.

Returns
-------
bool
True when ``key`` is a slice with a step of None, 1 or -1. A larger step would leave
gaps between periods, so it returns False. A step of -1 keeps adjacent periods in
reverse order, which ``sort_axis`` can undo.
"""
return isinstance(key, slice) and key.step in (None, 1, -1)

@staticmethod
def _contig_slice(arr: _AxisKey) -> slice | np.ndarray:
"""
Expand Down Expand Up @@ -152,7 +183,9 @@ def __setitem__(
for item in contig_key
])
norm_key = self._normalize_index(tuple_key)
if type(norm_key[2]) is not slice or type(norm_key[3]) is not slice:
if not _LocBase._is_unit_slice(norm_key[2]) or not _LocBase._is_unit_slice(
norm_key[3]
):
raise ValueError(
"Setting while fancy indexing on origin/development is not supported."
)
Expand Down Expand Up @@ -990,12 +1023,15 @@ def _slice_valuation(self: TriangleProtocol, key: np.ndarray) -> Triangle:
"BackendArray", num_to_nan(obj.values * obj.get_array_module().array(key))
)
# Recalculate size of the origin and development axes and return the slice.
return _LocBase(obj).get_idx((
slice(None),
slice(None),
np.arange(obj.shape[2])[np.sum(~key, 1) != obj.shape[3]],
np.arange(obj.shape[3])[np.sum(~key, 0) != obj.shape[2]],
))
return _LocBase(obj).get_idx(
(
slice(None),
slice(None),
np.arange(obj.shape[2])[np.sum(~key, 1) != obj.shape[3]],
np.arange(obj.shape[3])[np.sum(~key, 0) != obj.shape[2]],
),
allow_gaps=True,
)

def _slice(
self: TriangleProtocol,
Expand All @@ -1013,11 +1049,15 @@ def _slice(
The axis to which the slice would apply, `ddims` for development, `odims` for origin.
"""

# noinspection PyProtectedMember
axis_slice = _LocBase._contig_slice(np.arange(len(key))[key])
# noinspection PyProtectedMember
if not _LocBase._is_unit_slice(axis_slice):
raise ValueError("Fancy indexing on origin/development is not supported.")
# Update the axis, then the values.
obj = self.copy()
setattr(obj, axis, getattr(obj, axis)[key])
# noinspection PyProtectedMember
slicer = ..., _LocBase._contig_slice(np.arange(len(key))[key])
slicer = ..., axis_slice
if axis == "odims":
slicer = tuple(list(slicer) + [slice(None)])
obj.values = obj.values[slicer]
Expand Down
143 changes: 143 additions & 0 deletions chainladder/core/tests/test_slicing.py
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,149 @@ def test_set_fancy_development_raises(raa: Triangle) -> None:
raa_copy.loc["Total", "values", :, [12, 24, 48]] = raa.iloc[0, 0, :, :3]


@pytest.mark.parametrize(
"origin_key",
[slice(None, None, 2), [0, 2, 4], [4, 2, 0]],
ids=["step_2", "evenly_spaced_list", "descending_list"],
)
def test_get_idx_stepped_origin_raises(raa: Triangle, origin_key) -> None:
"""
Attempt stepped indexing on the origin axis, raise an error. Evenly spaced positions convert to
a stepped slice, which would leave gaps between origin periods (#1649).

Parameters
----------
raa: Triangle
The raa sample data set fixture.
origin_key: slice | list
A stepped origin key.

Returns
-------
None

"""
with pytest.raises(
ValueError, match="Fancy indexing on origin/development is not supported"
):
_ = raa.iloc[..., origin_key, :]


@pytest.mark.parametrize(
"development_key",
[slice(None, None, 2), [0, 2, 4]],
ids=["step_2", "evenly_spaced_list"],
)
def test_get_idx_stepped_development_raises(raa: Triangle, development_key) -> None:
"""
Attempt stepped indexing on the development axis, raise an error (#1649).

Parameters
----------
raa: Triangle
The raa sample data set fixture.
development_key: slice | list
A stepped development key.

Returns
-------
None

"""
with pytest.raises(
ValueError, match="Fancy indexing on origin/development is not supported"
):
_ = raa.iloc[..., :, development_key]


def test_non_contiguous_boolean_mask_raises(raa: Triangle) -> None:
"""
Filter origin and development with boolean masks that keep non-adjacent periods, raise an error.
A contiguous mask still works (#1649).

Parameters
----------
raa: Triangle
The raa sample data set fixture.

Returns
-------
None

"""
with pytest.raises(
ValueError, match="Fancy indexing on origin/development is not supported"
):
_ = raa[raa.origin.year % 2 == 0]
with pytest.raises(
ValueError, match="Fancy indexing on origin/development is not supported"
):
_ = raa[raa.origin != "1985"]
with pytest.raises(
ValueError, match="Fancy indexing on origin/development is not supported"
):
_ = raa[raa.development % 24 == 0]
assert raa[raa.origin >= "1985"].shape == (1, 1, 6, 10)


def test_set_stepped_origin_development_raises(raa: Triangle) -> None:
"""
Attempt setting with stepped indexing on the origin and development axes, raise an error (#1649).

Parameters
----------
raa: Triangle
The raa sample data set fixture.

Returns
-------
None

"""
raa_copy = raa.copy()
if raa.array_backend == "numpy":
# Test is specific to the numpy backend.
with pytest.raises(
ValueError,
match="Setting while fancy indexing on origin/development is not supported.",
):
raa_copy.iloc[0, 0, [0, 2, 4], :] = raa.iloc[0, 0, :3, :]
with pytest.raises(
ValueError,
match="Setting while fancy indexing on origin/development is not supported.",
):
raa_copy.iloc[0, 0, :, ::2] = raa.iloc[0, 0, :, :5]


def test_reversed_and_coarser_grain_still_work(raa: Triangle) -> None:
"""
Reverse the origin and development axes and coarsen the development grain. Each keeps working
after stepped slices started to raise (#1649).

Parameters
----------
raa: Triangle
The raa sample data set fixture.

Returns
-------
None

"""
reversed_tr = raa.iloc[..., ::-1, ::-1]
assert reversed_tr.shape == raa.shape
assert list(reversed_tr.origin.year) == list(raa.origin.year)[::-1]
assert raa.latest_diagonal.shape == (1, 1, 10, 1)
quarterly = cl.load_sample("quarterly")
yearly = quarterly.grain("OYDY")
assert yearly.shape == (1, 2, 12, 12)
assert list(yearly.ddims) == list(range(3, 136, 12))
keep = np.isin(quarterly.ddims, yearly.ddims)
np.testing.assert_allclose(
np.nan_to_num(yearly.values), np.nan_to_num(quarterly.values[..., keep])
)


def test_get_idx_non_contiguous_index_and_columns(clrd: Triangle) -> None:
"""
Pass lists of non-contiguous index expressions of the index and column axes to Triangle.iloc. Check the values
Expand Down
7 changes: 6 additions & 1 deletion chainladder/core/triangle.py
Original file line number Diff line number Diff line change
Expand Up @@ -2023,7 +2023,12 @@ def grain(self, grain="", trailing=False, inplace=False):
)

if obj.is_cumulative:
obj = obj.iloc[..., d]
# Keeping every step-th age is how a cumulative triangle moves to a coarser
# development grain, so this pick may be a stepped slice.
# noinspection PyProtectedMember
obj = obj.iloc.get_idx(
obj.iloc._normalize_index((..., d)), allow_gaps=True
)
else:
ddims = obj.ddims[d]
d2 = [d[0]] * (d[0] + 1) + list(np.repeat(np.array(d[1:]), step))
Expand Down
13 changes: 13 additions & 0 deletions chainladder/utils/tests/test_utilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,19 @@ def test_json_df():
assert abs(cl.read_json(x.to_json()).lambda_ - x.lambda_).sum() < 1e-5


def test_json_pattern_keeps_development():
"""
Round-trip a fitted MunichAdjustment through JSON. dropna trims the development patterns by
position, so the string labels of the patterns keep their order and no gap appears (#1649).
"""
x = cl.MunichAdjustment(paid_to_incurred=("paid", "incurred")).fit_transform(
cl.load_sample("mcl")
)
y = cl.read_json(x.to_json())
assert list(y.ldf_.development) == list(x.ldf_.development)
assert y.ldf_.shape == x.ldf_.shape


def test_read_csv_single(raa):
# Test the read_csv function for a single dimensional input.

Expand Down