diff --git a/CHANGELOG.md b/CHANGELOG.md index c4b80a5..499b03d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,41 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- **`sum_codes`: parallel listings of the same index, added to the primary week by week. + ES, NQ, YM and EMD now include their full-size contracts.** Each traded a big contract + beside the mini for years (S&P 500 `138741` 1986 to 2021-09, Nasdaq-100 `209741` 1996 to + 2015, DJIA `124601` 1997 to 2014, MidCap `338741` 1992 to 2008) and the registry read only + the mini. In notional terms the big contract held 99% of S&P 500 open interest in 1997 to + 1999, 82% in 2004 and 54% in 2008 (NQ 97% in 1999, YM 84% in 2002), so the stored + positioning before roughly 2008 was a minority, retail-skewed slice of each market. + `get_cot` now scales each listing into the primary's contract units (ES 5, and 10 before + the 1997-11-04 report when the big contract was $500 x index; NQ 5; YM 2; EMD 5), sums + open interest and every `*_Positions_*` column, takes the max of `Traders_*` across + listings (a lower bound on distinct traders), recomputes `Pct_of_OI_*` and `Change_in_*` + from the sums as floats, and keeps the concentration ratios and text metadata from the + primary. Weeks only the big contract reports extend the series: ES now starts 1986-01-15 + (was 1997-09-16), NQ 1996-04-16 (was 1999-06-22), YM 1997-10-07 (was 2002-05-21). + Verified against the CFTC's own Consolidated rows (`13874+`, `20974+`, `12460+`, which are + big + mini/5 in big units from 2010-06 to 2021-09): open interest and both commercial + columns agree to rounding on 99.8% of weeks for NQ and YM and 93% for ES, the remainder + being weeks the dying big contract fell under the CFTC's 20-trader publication threshold + (absent from the file, still inside Consolidated; up to 6% of ES OI in mid-2021). + Non-commercial long and short do NOT reconcile to Consolidated, because the CFTC computes + spreading per trader across both listings and a sum of per-listing rows cannot. Micro + contracts are not included. The producers fetch the four new per-code files on their next + run; until then the four symbols read exactly as before. Registry comments carry the + evidence; tests in `tests/test_cot_sum.py`. +- **`registry.codes_for(sym)`**: every code a producer must fetch for a symbol (primary, + `hist_codes`, `sum_codes`). All four producers use it. + +### Fixed + +- **The `hist_codes` scale no longer multiplies trader counts.** It applied to every + numeric column, so legacy `Traders_*` (numeric in that schema) were scaled by lumber's + 4.0 and, since the previous entry, the grains' 0.2: corn showed 72 reportable traders on + 1997-12-30 against 349 a week later. The scale now applies to `Open_Interest_All`, + `*_Positions_*` and numeric `Change_in_*` only. + - **Twelve more years of grain positioning: ZC, ZS, ZW, ZO and KE now stitch their pre-1998 codes.** The five CBOT/KCBT grains re-coded at the 1997/1998 year boundary (`002601` to `002602`, `005601` to `005602`, `001601` to `001602`, `004601` to `004603`, diff --git a/docs/design/reading-the-store.md b/docs/design/reading-the-store.md index a42d083..bdd142d 100644 --- a/docs/design/reading-the-store.md +++ b/docs/design/reading-the-store.md @@ -80,6 +80,18 @@ contracts standing in for ICE's 545k to 567k. The retiring venue's last weeks ar (its remaining holders are whoever has not migrated yet), so a window that straddles a hand-off is measuring two populations whichever rule picks the row. +**Cause one, other shape: a parallel listing, where a sibling code holds most of the +market.** The equity index futures each traded a full-size contract beside the mini for +years (S&P 500 `138741` to 2021-09, Nasdaq-100 `209741` to 2015, DJIA `124601` to 2014, +MidCap `338741` to 2008), and in notional terms the big contract held 99% of S&P 500 open +interest in 1997 to 1999 and still 54% in 2008. These are not complementary like the Russell +codes, they are concurrent, so `get_cot` ADDS them (`sum_codes` in the registry, scaled into +mini units) rather than picking one. Two things the sum cannot recover: non-commercial +spreading, which the CFTC computes per trader across both listings, and weeks a listing fell +under the 20-trader publication threshold, which are absent from the file but present in the +CFTC's own Consolidated row (`13874+` etc., 2010-06 on). Against that row, open interest and +the commercial columns agree to rounding on 99.8% of NQ and YM weeks and 93% of ES weeks. + **Cause two: a thin market falling out of the report.** Oats (`004603`) has a **294-day interval ending 2025-09-09** and five more over 50 days. A market below the reporting threshold disappears and returns when it recovers. No sibling fills these, because nothing is diff --git a/src/cotdata/cot.py b/src/cotdata/cot.py index a794539..b6d85cd 100644 --- a/src/cotdata/cot.py +++ b/src/cotdata/cot.py @@ -4,13 +4,113 @@ cot-analyzer (CotIndexer / metrics) — cotdata owns only the raw data. This returns the cleaned weekly COT table; the indexer builds indices on top. """ +import re + +import numpy as np import pandas as pd from . import store -from .registry import REGISTRY, hist_code_scales +from .registry import REGISTRY, hist_code_scales, sum_code_scales _CODE_COL = "CFTC_Contract_Market_Code" # Legacy schema contract-code column _OI_COL = "Open_Interest_All" # present in all four report schemas +# Columns denominated in CONTRACTS, the only ones a contract-size scale applies to. +# Trader counts, concentration ratios and percent-of-OI columns are invariant to the +# unit and must not be scaled (the hist scale once multiplied legacy Traders_* +# by the lumber 4.0 and the grain 0.2, so corn showed 72 traders in 1997). +_COUNT_RE = re.compile(r"^Open_Interest_All$|_Positions_|^Change_in_") +_PCT_RE = re.compile(r"^Pct_of_OI_(.+)_(Long|Short|Spread)_All$") +_CHANGE_RE = re.compile(r"^Change_in_(.+)_(Long|Short|Spread)_All$") + + +def _count_cols(df: pd.DataFrame) -> list: + return [c for c in df.columns + if _COUNT_RE.search(c) and pd.api.types.is_numeric_dtype(df[c])] + + +def _scale_counts(df: pd.DataFrame, scale) -> pd.DataFrame: + """Multiply contract-denominated columns by `scale` (a float or a per-row Series).""" + cols = _count_cols(df) + if cols: + df[cols] = df[cols].mul(scale, axis=0) if isinstance(scale, pd.Series) \ + else df[cols] * scale + return df + + +def _row_scales(index: pd.DatetimeIndex, entries) -> pd.Series: + """Per-row scale for one listing from its (scale, from_date) entries. The undated + entry covers everything; each dated entry takes over from its date. Rows no entry + covers come back NaN and are dropped by the caller.""" + s = pd.Series(np.nan, index=index, dtype=float) + for scale, start in sorted(entries, key=lambda e: (e[1] is not None, e[1] or "")): + if start is None: + s[:] = scale + else: + s[index >= pd.Timestamp(start)] = scale + return s + + +def _add_listings(base: pd.DataFrame, sym, read_fn) -> pd.DataFrame: + """Add the symbol's parallel listings (sum_codes) to `base`, week by week. + + Contract-denominated columns are scaled into the primary's units and summed + (weeks a listing reports alone extend the series). Trader counts take the max + across listings, a lower bound on distinct traders. Percent-of-OI and + Change_in_* columns are recomputed from the sums as floats. Concentration + ratios and the text metadata stay the primary's, NaN where the primary has no + row. The code column is the primary's throughout. + """ + by_code = {} + for code, scale, start in sum_code_scales(sym.sum_codes): + by_code.setdefault(code, []).append((scale, start)) + extras = [] + for code, entries in by_code.items(): + f = read_fn(f"{sym.internal}_{code}") + if f.empty: + continue + f = f[~f.index.duplicated(keep="first")].copy() + sc = _row_scales(f.index, entries) + f = _scale_counts(f[sc.notna()], sc.dropna()) + extras.append(f) + if not extras: + return base + frames = ([base] if not base.empty else []) + extras + idx = frames[0].index + for f in frames[1:]: + idx = idx.union(f.index) + idx = idx.sort_values() + + out = base.reindex(idx) + meta = [c for c in out.columns if not pd.api.types.is_numeric_dtype(out[c]) + and not c.startswith(("Change_in_", "Traders_"))] + for f in extras: # text metadata from a listing where primary is absent + out[meta] = out[meta].combine_first(f.reindex(idx)[meta]) if meta else out[meta] + if _CODE_COL in out.columns: + out[_CODE_COL] = sym.cftc_code + + counts = sorted({c for f in frames for c in _count_cols(f) + if not c.startswith("Change_in_")}) + total = sum(f.reindex(idx)[[c for c in counts if c in f.columns]] + .reindex(columns=counts).fillna(0.0) for f in frames) + out[counts] = total + + traders = sorted({c for f in frames for c in f.columns if c.startswith("Traders_")}) + for c in traders: + cols = [pd.to_numeric(f.reindex(idx)[c], errors="coerce") for f in frames if c in f.columns] + out[c] = pd.concat(cols, axis=1).max(axis=1) + + for c in list(out.columns): + if c == "Pct_of_Open_Interest_All": + out[c] = 100.0 + elif (m := _PCT_RE.match(c)): + pos = f"{m.group(1)}_Positions_{m.group(2)}_All" + out[c] = 100.0 * out[pos] / out[_OI_COL] if pos in out.columns else np.nan + elif c == "Change_in_Open_Interest_All": + out[c] = out[_OI_COL].diff() + elif (m := _CHANGE_RE.match(c)): + pos = f"{m.group(1)}_Positions_{m.group(2)}_All" + out[c] = out[pos].diff() if pos in out.columns else np.nan + return out def get_cot(name: str, report: str = "legacy") -> pd.DataFrame: @@ -50,17 +150,20 @@ def get_cot(name: str, report: str = "legacy") -> pd.DataFrame: if sym is None or not sym.cftc_code: return read_fn(name) primary = read_fn(f"{sym.internal}_{sym.cftc_code}") - if not sym.hist_codes: - return primary + stitched = _stitch_predecessors(primary, sym, read_fn) if sym.hist_codes else primary + return _add_listings(stitched, sym, read_fn) if sym.sum_codes else stitched + + +def _stitch_predecessors(primary: pd.DataFrame, sym, read_fn) -> pd.DataFrame: + """Splice hist_codes predecessors chronologically behind the primary code.""" frames = [primary] # primary first → wins ties on overlap weeks for hc, scale in hist_code_scales(sym.hist_codes): h = read_fn(f"{sym.internal}_{hc}") if h.empty: continue h = h.copy() - if scale != 1.0: # bridge a contract-size change (position/OI counts) - num = h.select_dtypes("number").columns - h[num] = h[num] * scale + if scale != 1.0: # bridge a contract-size change (contract counts only) + h = _scale_counts(h, scale) if _CODE_COL in h.columns: # present predecessor rows under the primary code h[_CODE_COL] = sym.cftc_code frames.append(h) diff --git a/src/cotdata/providers/cftc.py b/src/cotdata/providers/cftc.py index 9b18604..24e7ab6 100644 --- a/src/cotdata/providers/cftc.py +++ b/src/cotdata/providers/cftc.py @@ -19,7 +19,7 @@ import requests from .. import config, store -from ..registry import all_symbols, hist_code_scales +from ..registry import all_symbols, codes_for URL_PREFIX = "https://www.cftc.gov/files/dea/history/dea_fut_xls_" FIRST_YEAR = 1986 # CFTC Legacy futures history start @@ -114,10 +114,8 @@ def update(codes=None, first_year: int = FIRST_YEAR, last_year=None) -> dict: """ code_to_sym = {} for s in all_symbols(): - if s.cftc_code: - code_to_sym[s.cftc_code] = s.internal - for hc, _ in hist_code_scales(s.hist_codes): - code_to_sym[hc] = s.internal + for c in codes_for(s): # primary + hist_codes + sum_codes + code_to_sym[c] = s.internal last_year = last_year or dt.date.today().year if codes: diff --git a/src/cotdata/providers/cftc_cit.py b/src/cotdata/providers/cftc_cit.py index 49f5a4b..50c7add 100644 --- a/src/cotdata/providers/cftc_cit.py +++ b/src/cotdata/providers/cftc_cit.py @@ -33,7 +33,7 @@ import requests from .. import config, store -from ..registry import all_symbols, hist_code_scales +from ..registry import all_symbols, codes_for URL_PREFIX = "https://www.cftc.gov/files/dea/history/dea_cit_txt_" FIRST_YEAR = 2006 # Supplemental history start (January 2006) @@ -143,10 +143,8 @@ def update(codes=None, first_year: int = FIRST_YEAR, last_year=None) -> dict: """ code_to_sym = {} for s in all_symbols(): - if s.cftc_code: - code_to_sym[s.cftc_code] = s.internal - for hc, _ in hist_code_scales(s.hist_codes): - code_to_sym[hc] = s.internal + for c in codes_for(s): # primary + hist_codes + sum_codes + code_to_sym[c] = s.internal last_year = last_year or dt.date.today().year want = set(codes) if codes else set(code_to_sym.keys()) diff --git a/src/cotdata/providers/cftc_disagg.py b/src/cotdata/providers/cftc_disagg.py index 379149b..44f0198 100644 --- a/src/cotdata/providers/cftc_disagg.py +++ b/src/cotdata/providers/cftc_disagg.py @@ -13,7 +13,7 @@ import requests from .. import config, store -from ..registry import all_symbols, hist_code_scales +from ..registry import all_symbols, codes_for URL_PREFIX = "https://www.cftc.gov/files/dea/history/fut_disagg_txt_" FIRST_YEAR = 2006 # Disaggregated futures history start @@ -94,10 +94,8 @@ def update(codes=None, first_year: int = FIRST_YEAR, last_year=None) -> dict: """ code_to_sym = {} for s in all_symbols(): - if s.cftc_code: - code_to_sym[s.cftc_code] = s.internal - for hc, _ in hist_code_scales(s.hist_codes): - code_to_sym[hc] = s.internal + for c in codes_for(s): # primary + hist_codes + sum_codes + code_to_sym[c] = s.internal last_year = last_year or dt.date.today().year if codes: diff --git a/src/cotdata/providers/cftc_tff.py b/src/cotdata/providers/cftc_tff.py index 27f25fe..b73468d 100644 --- a/src/cotdata/providers/cftc_tff.py +++ b/src/cotdata/providers/cftc_tff.py @@ -13,7 +13,7 @@ import requests from .. import config, store -from ..registry import all_symbols, hist_code_scales +from ..registry import all_symbols, codes_for URL_PREFIX = "https://www.cftc.gov/files/dea/history/fut_fin_txt_" FIRST_YEAR = 2006 # TFF futures history start @@ -94,10 +94,8 @@ def update(codes=None, first_year: int = FIRST_YEAR, last_year=None) -> dict: """ code_to_sym = {} for s in all_symbols(): - if s.cftc_code: - code_to_sym[s.cftc_code] = s.internal - for hc, _ in hist_code_scales(s.hist_codes): - code_to_sym[hc] = s.internal + for c in codes_for(s): # primary + hist_codes + sum_codes + code_to_sym[c] = s.internal last_year = last_year or dt.date.today().year if codes: diff --git a/src/cotdata/registry.py b/src/cotdata/registry.py index cf315c2..5b1a244 100644 --- a/src/cotdata/registry.py +++ b/src/cotdata/registry.py @@ -50,6 +50,15 @@ class Symbol: # entry is a bare code string (scale 1.0) or a (code, scale) tuple. Kept as a # tuple of hashable entries so Symbol (frozen) stays hashable. hist_codes: tuple = () + # PARALLEL listings of the SAME index that traded alongside cftc_code (the + # full-size S&P 500 beside the E-mini, 1997 to 2021), ADDED to it week by week + # by get_cot in cftc_code's contract units. Each entry is (code, scale) or + # (code, scale, from_date): the scale converts one listed contract into + # cftc_code contracts, and a dated entry takes over from that report date (the + # big S&P halved from $500 to $250 x index on 1997-11-04, so it is 10 E-minis + # before and 5 after). Where hist_codes picks one row per week, sum_codes adds + # rows, and weeks only the listing reports extend the series on their own. + sum_codes: tuple = () def hist_code_scales(hist_codes) -> List[tuple]: @@ -60,6 +69,31 @@ def hist_code_scales(hist_codes) -> List[tuple]: return out +def sum_code_scales(sum_codes) -> List[tuple]: + """Normalize sum_codes entries → list of (code, scale, from_date-or-None). + + Entries are (code, scale) or (code, scale, "YYYY-MM-DD"). A bare code string is + accepted at scale 1.0 for symmetry with hist_codes. + """ + out = [] + for e in sum_codes: + if isinstance(e, (list, tuple)): + code, scale = e[0], float(e[1]) + start = str(e[2]) if len(e) > 2 and e[2] is not None else None + else: + code, scale, start = e, 1.0, None + out.append((code, scale, start)) + return out + + +def codes_for(sym: "Symbol") -> List[str]: + """Every CFTC code a producer must fetch for `sym`: primary, predecessors, listings.""" + codes = [sym.cftc_code] if sym.cftc_code else [] + codes += [hc for hc, _ in hist_code_scales(sym.hist_codes)] + codes += [sc for sc, _, _ in sum_code_scales(sym.sum_codes)] + return list(dict.fromkeys(codes)) # dedupe (a listing may carry several dated scales) + + def _coerce_hist_codes(raw) -> tuple: """YAML encodes scaled entries as lists ([code, scale]); store them as tuples so Symbol stays hashable (a list nested in the tuple would break hashing).""" @@ -120,6 +154,7 @@ def load_registry(yaml_path=None) -> Dict[str, Symbol]: report_type=attrs.get("report_type", "tff" if asset_class in ("Equities", "FX", "Rates") else "disagg"), cftc_code=attrs["cftc_code"], hist_codes=_coerce_hist_codes(attrs.get("hist_codes")), + sum_codes=_coerce_hist_codes(attrs.get("sum_codes")), ) return registry diff --git a/src/cotdata/registry.yaml b/src/cotdata/registry.yaml index 028bfd0..d51f69d 100644 --- a/src/cotdata/registry.yaml +++ b/src/cotdata/registry.yaml @@ -1,10 +1,30 @@ Equities: + # The four index futures below traded a full-size contract beside the mini for years, + # and the registry read only the mini. In notional terms the big contract held 99% of + # S&P 500 open interest in 1997 to 1999, 82% in 2004 and 54% in 2008 (NQ 97% in 1999, + # YM 84% in 2002), so the stored mini-only positioning before roughly 2008 was a minority + # slice of each market, and the earlier the week the smaller and more retail-skewed the + # slice. sum_codes ADDS the big contract week by week in mini units (and extends the + # series back to the big contract's own start where the mini did not exist yet). + # Scales are multiplier ratios: ES $250 / $50 = 5 (and $500 / $50 = 10 before the + # 1997-11-04 report, the first after CME halved the contract; OI doubles that week, + # 206k to 402k), NQ $100 / $20 = 5, YM $10 / $5 = 2, EMD $500 / $100 = 5. Verified + # against the CFTC's own Consolidated rows (13874+, 20974+, 12460+), which are + # big + mini/5 in big units from 2010-06 to 2021-09. Micro contracts are NOT included. + # Measured 2026-10-10 against the CFTC deacot1986_2016 bundle plus annual files. ES: cftc_code: "13874A" + sum_codes: + - ["138741", 10.0] # S&P 500 full-size, $500 x index + - ["138741", 5.0, "1997-11-04"] # $250 x index from this report on NQ: cftc_code: "209742" + sum_codes: + - ["209741", 5.0] # Nasdaq-100 full-size, $100 x index, to 2015 YM: cftc_code: "124603" + sum_codes: + - ["124601", 2.0] # DJIA $10 x index, to 2014 RTY: cftc_code: "239742" hist_codes: @@ -13,6 +33,8 @@ Equities: # so the strategy can be scored on them; cot-analyzer flags them role: heldout. EMD: # E-mini S&P MidCap 400 (CME) cftc_code: "33874A" + sum_codes: + - ["338741", 5.0] # S&P MidCap 400 full-size, $500 x index, to 2008 NKD: # Nikkei 225 ($-denominated, CME) cftc_code: "240741" # MSCI international indices — the liquid out-of-selection held-out markets. COT is diff --git a/tests/test_cot.py b/tests/test_cot.py index 96629b2..92fd137 100644 --- a/tests/test_cot.py +++ b/tests/test_cot.py @@ -25,12 +25,12 @@ def test_unknown_report_raises(): def test_simple_symbol_returns_primary_by_symbol_code_key(): - """A symbol with no hist_codes (ES) reads exactly `_` from - the requested report and returns it unchanged.""" - primary = _df(["2020-01-01"], [10], REGISTRY["ES"].cftc_code) + """A symbol with no hist_codes or sum_codes (GC) reads exactly `_` + from the requested report and returns it unchanged.""" + primary = _df(["2020-01-01"], [10], REGISTRY["GC"].cftc_code) with mock.patch("cotdata.cot.store.read_cot_legacy", return_value=primary) as m: - out = cot.get_cot("ES") - m.assert_called_once_with(f"ES_{REGISTRY['ES'].cftc_code}") + out = cot.get_cot("GC") + m.assert_called_once_with(f"GC_{REGISTRY['GC'].cftc_code}") pd.testing.assert_frame_equal(out, primary) @@ -45,10 +45,10 @@ def test_report_dispatch_uses_matching_store_reader(): def test_lookup_by_primary_cftc_code_resolves_symbol(): """Passing the CFTC code instead of the internal symbol resolves to the symbol and reads the same `_` key.""" - primary = _df(["2020-01-01"], [10], REGISTRY["ES"].cftc_code) + primary = _df(["2020-01-01"], [10], REGISTRY["GC"].cftc_code) with mock.patch("cotdata.cot.store.read_cot_legacy", return_value=primary) as m: - cot.get_cot(REGISTRY["ES"].cftc_code) # "13874A" - m.assert_called_once_with(f"ES_{REGISTRY['ES'].cftc_code}") + cot.get_cot(REGISTRY["GC"].cftc_code) # "088691" + m.assert_called_once_with(f"GC_{REGISTRY['GC'].cftc_code}") def test_hist_code_stitching_scales_and_dedups_by_larger_oi(): diff --git a/tests/test_cot_sum.py b/tests/test_cot_sum.py new file mode 100644 index 0000000..72601cd --- /dev/null +++ b/tests/test_cot_sum.py @@ -0,0 +1,112 @@ +"""get_cot parallel listings (sum_codes): the full-size contract ADDED to the mini. + +Synthetic TFF-shaped frames: a primary (mini) and one listing (big) at scale 5 before +2020-01-14 and scale 2 from it, overlapping on two weeks and each reporting alone on one. +""" +import numpy as np +import pandas as pd +import pytest + +from cotdata import cot +from cotdata.registry import Symbol + +D = pd.DatetimeIndex(["2020-01-07", "2020-01-14", "2020-01-21"]) + + +def _tff(dates, oi, dl, ds, traders, name, code): + n = len(dates) + return pd.DataFrame({ + "Market_and_Exchange_Names": [name] * n, + "CFTC_Contract_Market_Code": [code] * n, + "Open_Interest_All": oi, + "Dealer_Positions_Long_All": dl, + "Dealer_Positions_Short_All": ds, + "Change_in_Open_Interest_All": ["x"] * n, # stored as text in the TFF files + "Change_in_Dealer_Long_All": ["x"] * n, + "Pct_of_Open_Interest_All": [100.0] * n, + "Pct_of_OI_Dealer_Long_All": [round(100 * a / b, 1) for a, b in zip(dl, oi)], + "Traders_Tot_All": traders, + "Traders_Dealer_Long_All": [str(t) for t in traders], # text in the TFF files + "Conc_Gross_LE_4_TDR_Long_All": [40.0] * n, + "Contract_Units": ["(x $50)"] * n, + }, index=pd.DatetimeIndex(dates, name="Report_Date_as_MM_DD_YYYY")) + + +@pytest.fixture +def summed(monkeypatch): + sym = Symbol(internal="XX", asset_class="Equities", is_equity=True, cftc_code="MINI", + sum_codes=(("BIG", 5.0), ("BIG", 2.0, "2020-01-14"))) + monkeypatch.setitem(cot.REGISTRY, "XX", sym) + frames = { + "XX_MINI": _tff(D[1:], [100, 200], [10, 20], [5, 10], [50, 60], "MINI NAME", "MINI"), + "XX_BIG": _tff(D[:2], [30, 40], [3, 4], [1, 2], [70, 20], "BIG NAME", "BIG"), + } + monkeypatch.setattr("cotdata.store.read_cot_tff", + lambda n: frames.get(n, pd.DataFrame()).copy()) + return frames + + +def test_counts_are_scaled_and_summed_with_dated_scale(summed): + out = cot.get_cot("XX", report="tff") + assert list(out.index) == list(D) + # 01-07: big alone at scale 5 -> 30*5 + assert out.loc[D[0], "Open_Interest_All"] == 150 + assert out.loc[D[0], "Dealer_Positions_Long_All"] == 15 + # 01-14: mini 100 + big 40 at scale 2 (dated entry takes over) -> 180 + assert out.loc[D[1], "Open_Interest_All"] == 180 + assert out.loc[D[1], "Dealer_Positions_Short_All"] == 5 + 2 * 2 + # 01-21: mini alone + assert out.loc[D[2], "Open_Interest_All"] == 200 + + +def test_pct_and_change_recomputed_from_sums(summed): + out = cot.get_cot("XX", report="tff") + assert (out["Pct_of_Open_Interest_All"] == 100.0).all() + np.testing.assert_allclose(out["Pct_of_OI_Dealer_Long_All"].values, + 100 * np.array([15, 18, 20]) / np.array([150, 180, 200])) # 01-14: 10 + 4*2 + assert np.isnan(out.loc[D[0], "Change_in_Open_Interest_All"]) + assert out.loc[D[1], "Change_in_Open_Interest_All"] == 30 + assert out.loc[D[2], "Change_in_Dealer_Long_All"] == 20 - 18 + + +def test_traders_take_the_max_across_listings(summed): + out = cot.get_cot("XX", report="tff") + assert out["Traders_Tot_All"].tolist() == [70, 50, 60] # 01-14: max(50, 20) + assert out["Traders_Dealer_Long_All"].tolist() == [70, 50, 60] # coerced from text + + +def test_metadata_and_code_follow_the_primary(summed): + out = cot.get_cot("XX", report="tff") + assert (out["CFTC_Contract_Market_Code"] == "MINI").all() + assert out["Market_and_Exchange_Names"].tolist() == ["BIG NAME", "MINI NAME", "MINI NAME"] + assert np.isnan(out.loc[D[0], "Conc_Gross_LE_4_TDR_Long_All"]) # primary absent + assert out.loc[D[1], "Conc_Gross_LE_4_TDR_Long_All"] == 40.0 + + +def test_missing_listing_file_leaves_primary_untouched(monkeypatch): + sym = Symbol(internal="YY", asset_class="Equities", is_equity=True, cftc_code="MINI", + sum_codes=(("BIG", 5.0),)) + monkeypatch.setitem(cot.REGISTRY, "YY", sym) + prim = _tff(D, [1, 2, 3], [1, 1, 1], [0, 0, 0], [9, 9, 9], "P", "MINI") + monkeypatch.setattr("cotdata.store.read_cot_tff", + lambda n: prim.copy() if n == "YY_MINI" else pd.DataFrame()) + pd.testing.assert_frame_equal(cot.get_cot("YY", report="tff"), prim) + + +def test_hist_scale_leaves_trader_counts_alone(monkeypatch): + """The predecessor scale applies to contract counts only (legacy Traders_* are numeric).""" + sym = Symbol(internal="ZZ", asset_class="Grains", is_equity=False, cftc_code="NEW", + hist_codes=(("OLD", 0.2),)) + monkeypatch.setitem(cot.REGISTRY, "ZZ", sym) + old = pd.DataFrame({"CFTC_Contract_Market_Code": ["OLD"], "Open_Interest_All": [1000.0], + "Comm_Positions_Long_All": [500.0], "Traders_Tot_All": [366.0]}, + index=pd.DatetimeIndex(["1997-12-30"])) + new = pd.DataFrame({"CFTC_Contract_Market_Code": ["NEW"], "Open_Interest_All": [210.0], + "Comm_Positions_Long_All": [100.0], "Traders_Tot_All": [345.0]}, + index=pd.DatetimeIndex(["1998-01-06"])) + monkeypatch.setattr("cotdata.store.read_cot_legacy", + lambda n: {"ZZ_NEW": new, "ZZ_OLD": old}.get(n, pd.DataFrame()).copy()) + out = cot.get_cot("ZZ") + assert out.loc["1997-12-30", "Open_Interest_All"] == 200 + assert out.loc["1997-12-30", "Comm_Positions_Long_All"] == 100 + assert out.loc["1997-12-30", "Traders_Tot_All"] == 366 diff --git a/tests/test_registry.py b/tests/test_registry.py index 1d5e327..94f0216 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -43,7 +43,7 @@ def test_the_registry_carries_no_vendor_mappings(): fields = {f.name for f in dataclasses.fields(symbol("ES"))} assert fields == {"internal", "asset_class", "is_equity", "report_type", - "cftc_code", "hist_codes"} + "cftc_code", "hist_codes", "sum_codes"} for gone in ("PRICE_SOURCES", "resolve_source", "default_price_source", "_can_serve"): assert not hasattr(reg, gone), gone