diff --git a/docs/design/morning.md b/docs/design/morning.md index c9efd13..0d1fd78 100644 --- a/docs/design/morning.md +++ b/docs/design/morning.md @@ -16,6 +16,8 @@ Three things hold it together: - **The format is enforced in code, with one redraft** (`_is_structured`, same remedy as `_NAMES_THE_LINK`), and only when there is something to bullet (`need = min(2, len(required))`) — a user with nothing loaded still gets a plain greeting. A redraft only replaces the draft if it is better formed, so a fix for one rule cannot break another. - **Length is capped at a line boundary** (`_tidy_morning_text`, `MORNING_LINE_MAX` 560). Whitespace collapses within a line but newlines survive; the old single-space join would have flattened the bullets, and a mid-line cut leaves half a fact. +**Weather and markets are in every update.** WEATHER is required whenever the payload has weather (it needs a city). MARKETS is required whenever the payload has a price row, and a user who follows no ticker gets one anyway: `home._fetch_prices` falls back to `tickers.DEFAULT_MARKETS` (S&P 500 and Dow) only when *nothing* in their topics resolved to a symbol. Tickers that resolve but fail to fetch do not trigger it — those are the user's chosen list, and a rate-limit blip must not swap it for someone else's. A user who hid the Markets section gets neither the fallback nor the requirement. The same payload feeds the page, so their Markets card shows the indices too. + Two properties are load-bearing: - **The URL is last and alone.** Message apps only draw the rich link preview when the message carries exactly one URL at a boundary, and that preview is most of the value. Nothing may follow it — not a period, not a sign-off. - **`carries_link` gates the status callback.** A link message is sent with `add_status_callback=False`, because the `/sms-status` shorten-and-retry would truncate the URL into garbage. diff --git a/docs/design/opening.md b/docs/design/opening.md index deec484..6b6c881 100644 --- a/docs/design/opening.md +++ b/docs/design/opening.md @@ -37,6 +37,35 @@ Three things are load-bearing: on Haiku, once per city, cached — deliberately *not* a lookup table, which is the mistake `tickers.py` made twice with `PRIVATE_COMPANIES`. Ticketmaster sidesteps the problem entirely by taking `latlong` + radius instead of a name. +- **A smaller metro came back all concerts, and four things caused it.** St. + Louis showed Black Country New Road, D.L. Hughley and Journey, week after week. + (1) `_local_candidates` was `trusted_only`, and stltoday, stlpr, stlmag, fox2now + and ksdk are the real newsrooms and none is on the allowlist — it returned + nothing, so the whole press half was silently empty. It now runs untrusted at + `LOCAL_MIN_SCORE` 0.60 (`_search_raw` still sorts trusted first) over four + queries — openings, "festivals fairs events {Month Year}", "festival this + weekend", "family events … this weekend" — on a 21-day window, because the + 14-day one made the same query swing between zero and eight hits. (2) + Ticketmaster was Music and Arts only; Miscellaneous (fairs, festivals, family + shows) is added, Sports deliberately is not (regular-season fixtures belong to + people who follow the team, which is a per-user question). (3) One shared + prompt cut at 12k characters meant ~8.6k of concerts and a clipped sliver of + press; `_fit` now holds each family to its own budget and drops whole items + rather than cutting one mid-JSON. (4) With both families in one call the model + took the first good candidates it met, which were the concerts, whatever the + prompt said about spread — so events and press are **curated in separate + calls** (`EVENT_ROWS`/`PRESS_ROWS` 5 each, `_interleave`d), and the per-topic + cap (`MAX_PER_TOPIC` 2) is enforced in code. Rows carry a `topic` + (music, comedy, theatre, food, festival, family, film, other) and `_spread` + takes one of each topic before a second at read time. +- **A press-sourced event must carry a date or it is dropped.** The curator lifts + individual events out of weekend roundups, and a date there is the model's + reading of prose. Undated, a row can never expire, and "Oct 9-11" was served on + the 14th. The prompt asks for the LAST day of a multi-day event; Ticketmaster + rows have a real date upstream and are exempt. Cost: Tavily goes from 2 to 4 + searches per metro per week and Haiku from 1 to 2 calls per metro per day. + Preference-driven ordering (family, teams, what the user has said) is the + next layer and is not built; today every user in a metro shares one ranking. - **The local outlets had to be added to `trusted_sources.json`.** Palmer Home passes `trusted_only=True`, and Eater, Time Out, LAist, Thrillist and the city dailies were all tier 3 — so before they were added the section returned diff --git a/palmer/agent.py b/palmer/agent.py index d278dab..feb52ee 100644 --- a/palmer/agent.py +++ b/palmer/agent.py @@ -228,7 +228,14 @@ def _build_system(phone: str, include_recent: bool = False, is_new_user: bool = # once — userprofile._update_profile marks score_offer_sent under this # same condition after the turn, so this never becomes a nag. teams = profile.get("sports_teams") - teams = ", ".join(teams) if isinstance(teams, list) else str(teams) + # sports_teams is meant to be prose, but a profile can hold follow-list + # dicts there (the old `teams` alias migrated them in). Join names, or + # this block raises and takes the whole turn with it. + if isinstance(teams, list): + teams = ", ".join(t.get("name", "") if isinstance(t, dict) else str(t) + for t in teams) + else: + teams = str(teams) system += ( "\n\nLIVE SCORES OFFER\n" f"They've mentioned a team they follow ({teams}) and nothing is set up for it. " diff --git a/palmer/home.py b/palmer/home.py index 6f7e52f..0853edb 100644 --- a/palmer/home.py +++ b/palmer/home.py @@ -192,7 +192,7 @@ def _fetch_prices(profile: dict, previous: list[dict] | None = None) -> list[dic exactly like Palmer forgetting, and is far worse than a slightly stale number sitting under a visible "N min ago" stamp.""" from palmer.datafeeds import price_snapshot - from palmer.tickers import resolve_topic_asset + from palmer.tickers import resolve_topic_asset, DEFAULT_MARKETS stale = {(p.get("label") or "").lower(): p for p in (previous or [])} assets, seen = [], set() for topic in (profile.get("morning_topics") or []): @@ -200,6 +200,14 @@ def _fetch_prices(profile: dict, previous: list[dict] | None = None) -> list[dic if got and got[0].lower() not in seen: seen.add(got[0].lower()) assets.append(got) + # Nothing resolved to a symbol: show the broad market rather than an empty + # section, so every user's morning has a markets line. Only when NOTHING + # resolved — someone tracking tickers whose fetches all failed chose those + # tickers, and indices would be somebody else's list. A user who hid the + # section asked not to see it. + hidden = (profile.get("morning_prefs") or {}).get("hidden_sections") or [] + if not assets and "markets" not in hidden: + assets = list(DEFAULT_MARKETS) out = [] for symbol, label in assets[:MAX_PRICES]: snapshot = price_snapshot(symbol, label) or stale.get(label.lower()) diff --git a/palmer/main.py b/palmer/main.py index a034a21..73bb23f 100644 --- a/palmer/main.py +++ b/palmer/main.py @@ -388,7 +388,7 @@ async def artifact_page(token: str): async def icon_png(): """The page's favicon and apple-touch-icon. No token: it is the same mark for everyone and carries no one's data.""" - from cards import render_icon + from palmer.cards import render_icon return Response(content=render_icon(), media_type="image/png", headers={"Cache-Control": "public, max-age=86400"}) diff --git a/palmer/morning.py b/palmer/morning.py index b3de9da..97829d6 100644 --- a/palmer/morning.py +++ b/palmer/morning.py @@ -572,6 +572,10 @@ def generate_morning_line(phone: str, payload: dict) -> str: required.append("the commute") if score_lines(payload): required.append("their team — yesterday's result and/or today's game, with the score or the time") + hidden = (payload.get("page_prefs") or {}).get("hidden_sections") or [] + if payload.get("prices") and "markets" not in hidden: + required.append("the markets — how each price row below moved, with its percent " + "(for most people that is just the S&P 500 and the Dow)") if payload.get("opening"): required.append("1-2 things newly open or worth catching near them this week, named specifically") required_block = ( @@ -591,7 +595,7 @@ def generate_morning_line(phone: str, payload: dict) -> str: Rules: - Cover every REQUIRED item above, in your own words, with real specifics — the actual temperature, the actual commute time, the actual name of the place or event. Never gesture at a category ("some stuff opened nearby") instead of naming it. - FORMAT: one opener line, then one bullet per item. The opener is a single short sentence in Palmer's voice (the day's character, a greeting, or the one thing that stands out) carrying no data the bullets also carry. Each bullet is its own line, exactly "- LABEL: text", where LABEL is ALL CAPS and one of: WEATHER, COMMUTE, NEARBY, SCORES, MARKETS, NEWS. Use WEATHER for weather, COMMUTE for the drive, NEARBY for openings/events, SCORES for a followed game. Only include a bullet for an item the data actually has. Each bullet is one tight line of plain words, not a clipped fragment. No blank lines between bullets. -- Beyond the required items, you may add ONE more bullet (MARKETS or NEWS) about something else on the page ONLY if it is genuinely notable — a real change, not routine. Skip it rather than padding; this text is basics plus link, not the briefing. +- Beyond the required items, you may add ONE more NEWS bullet about something else on the page ONLY if it is genuinely notable — a real change, not routine. Skip it rather than padding; this text is basics plus link, not the briefing. - Under {MORNING_LINE_MAX} characters total, counting every line. Tight and scannable. - Never say the word "link", "page", "dashboard", "site", or "click" - the link speaks for itself and naming it makes this sound like a product notification. - Write ONLY the text. The link is attached automatically after it. Do not write a URL, and do not leave a placeholder like [link] or (url) where you think one goes - anything like that ships to them as literal text. diff --git a/palmer/opening.py b/palmer/opening.py index 62b1895..b2f158d 100644 --- a/palmer/opening.py +++ b/palmer/opening.py @@ -67,6 +67,9 @@ # dropping one. A user says "I want movie openings too" or "stop the concerts"; # they never say "kind=screen". ALL_KINDS = ("local", "event", "screen") +# What a row is ABOUT, as opposed to its kind. Kind decides which user control +# trims it; topic is what keeps one category from filling the section. +TOPICS = ("music", "comedy", "theatre", "food", "festival", "family", "film", "other") KIND_WORDS = { "restaurants": "local", "places": "local", "food": "local", "bars": "local", "events": "event", "concerts": "event", "festivals": "event", "shows": "event", @@ -77,8 +80,17 @@ # — a user who only wants restaurants should not come up empty because the # three cached rows happened to all be concerts. Costs nothing extra: same one # Haiku call, a few more output tokens. -CURATE_POOL = 6 -# Radius for the events pull. 25 miles is a metro, not a neighbourhood, which +CURATE_POOL = 10 +# Ticketmaster and local press are curated in SEPARATE calls, five rows each. +# In one mixed call the model took the first good candidates it met, and with +# thirty-odd structured concerts ahead of the press, "pick the best ten" came +# back as ten concerts however the prompt asked for spread. Separate calls make +# the split a fact of the pipeline instead of a request. +EVENT_ROWS = 5 +PRESS_ROWS = 5 +# At most this many rows of one topic survive curation, enforced in code — the +# prompt says the same thing and the model breaks it. +MAX_PER_TOPIC = 2# Radius for the events pull. 25 miles is a metro, not a neighbourhood, which # is deliberate: nobody picks a concert by how close it is to their street. EVENT_RADIUS_MI = 25 @@ -173,27 +185,58 @@ def _metro(city: str) -> str: return metro -def _local_candidates(metro: str) -> list[dict]: - """New-opening coverage from local press. - - There is no structured API for "restaurants that opened this month", but - it is a well-established local-media beat — Eater, Time Out, LAist and the - city dailies all run it as a standing column. Those outlets are tier 2 in - trusted_sources.json specifically so this search can pass trusted_only and - still return something; before they were added it returned nothing at all. +# The floor an untrusted local outlet has to clear. Same bar and same reason as +# home.UNTRUSTED_MIN_SCORE: a source that is not on the allowlist earns its place +# on match strength, because it is not earning it on provenance. +LOCAL_MIN_SCORE = 0.60 + + +def _local_queries(metro: str, today: date) -> list[str]: + """Four questions, one per thing people actually go out for. Each is worded + the way the answer is headlined: St. Louis returns real results for + "festivals fairs October 2026", "festival this weekend" and "family events + ... this weekend", and returned nothing useful for "things to do ... this + weekend", whose top hits were other cities' Axios newsletters.""" + return [f"new restaurant openings {metro}", + f"{metro} festivals fairs events {today.strftime('%B %Y')}", + f"{metro} festival this weekend", + f"family events {metro} this weekend"] + + +def _local_candidates(metro: str, today: date | None = None) -> list[dict]: + """Local-press coverage of what is new and what is on. + + There is no structured API for "restaurants that opened this month" or "the + fall fair this weekend", but both are standing local-media beats — the + openings column, the weekend roundup. The roundups are the only place a + smaller metro's fairs and family events are written down at all, so the + blurb is long enough (700 chars) for the curator to lift individual events + out of one. + + Not trusted_only. This used to be, and for St. Louis that returned nothing: + stltoday, stlpr, stlmag, fox2now and ksdk are the real newsrooms and none is + on the allowlist, so the whole non-Ticketmaster half of the section was + silently empty and every row was a concert. _search_raw already sorts by + tier, so trusted sources still come first; the higher floor is what stands + in for provenance on the rest. """ from palmer.datafeeds import _search_raw from palmer.sources import canonical_domain - out = [] - for query in (f"new restaurant openings {metro}", - f"festivals and events happening in {metro} this week"): + out, seen = [], set() + for query in _local_queries(metro, today or date.today()): try: - for r in _search_raw(query, days=14, max_age_hours=24 * 14, - min_score=0.4, trusted_only=True)[:4]: + # 21 days, not 14: measured on St. Louis, the narrower window made + # the same query swing between zero and eight hits, and a weekend + # roundup is published days before the weekend it covers. + for r in _search_raw(query, days=21, max_age_hours=24 * 21, + min_score=LOCAL_MIN_SCORE, trusted_only=False)[:4]: + if r.get("url") in seen: + continue + seen.add(r.get("url")) out.append({"title": (r.get("title") or "")[:140], "url": r.get("url"), "source": canonical_domain(r.get("url", "")), - "blurb": (r.get("content") or "")[:300]}) + "blurb": (r.get("content") or "")[:700]}) except Exception as e: print(f"opening: local search failed for {query!r}: {type(e).__name__}: {e}") return out @@ -223,12 +266,16 @@ def _events(lat: float, lon: float, start_days: int = 0, end_days: int = 7, f"&startDateTime={start.strftime('%Y-%m-%dT%H:%M:%SZ')}" f"&endDateTime={end.strftime('%Y-%m-%dT%H:%M:%SZ')}" f"&size={size}&sort=relevance,desc" - # Music and Arts only. Unfiltered, a metro's next seven days are + # Music, Arts and Miscellaneous only. Unfiltered, a metro's next seven days are # mostly regular-season ball games — eight Cardinals fixtures crowded # out every concert in St. Louis — and a Tuesday home game is not # something opening. Filtering here rather than in the prompt keeps # the candidate list short and the curation call cheap. - f"&segmentName=Music&segmentName=Arts%20%26%20Theatre") + # Miscellaneous is where Ticketmaster files fairs, festivals and the + # family shows (a marching-band championship, a kids' touring act). + # Sports stays out: it is regular-season fixtures, which belong to + # people who follow the team and not to the section's default. + f"&segmentName=Music&segmentName=Arts%20%26%20Theatre&segmentName=Miscellaneous") data = _http_get_json(url, timeout=10) out = [] for ev in ((data or {}).get("_embedded") or {}).get("events") or []: @@ -333,14 +380,18 @@ def _in_window(iso: str | None) -> bool: CANDIDATES (JSON): {candidates} -Return JSON: {{"rows": [{{"title": "...", "subtitle": "...", "when": "...", "url": "...", "kind": "local|event|screen", "date": "..."}}]}} +Return JSON: {{"rows": [{{"title": "...", "subtitle": "...", "when": "...", "url": "...", "kind": "local|event", "topic": "music|comedy|theatre|food|festival|family|film|other", "date": "..."}}]}} Pick up to {max_rows}, best first. If that many clear the bar, return that many — under-filling is as wrong as padding, and a metro with three good things happening should show three. +Spread matters as much as quality. This is a whole metro's list and the people reading it want different things — one wants a concert, another a fair to take the kids to, another a new place for dinner. Take at most two rows of any one topic, even when a topic is full of good candidates, so the list covers what is on rather than the single loudest category. Set `topic` honestly: a concert is music, a standup is comedy, a play or symphony is theatre, a restaurant or bar opening is food, a fair, fest or market is festival, and something built for kids or a family outing is family. + Order matters: something happening in the next few days outranks something weeks away, which outranks a dated event months out; any dated event outranks an article about a place. A named act at a named venue this week is one of the strongest rows this section can have. A dated EVENT more than about three weeks from {today} only earns a row if it is genuinely major — a well-known touring artist, a marquee festival, something someone would want advance notice of to plan around or buy tickets before it sells out. Allow at most one such row per pass; a handful of distant tour dates must never crowd out what is actually happening near {today}. +Weekend roundups ("things to do this weekend", "ready set weekend") are not rows themselves, but the specific events inside them are. Lift out each event that has a name, a place and a day — a fall festival, a pumpkin patch, a street fair, a parade — and make it its own row, with the roundup's url and the event's date. Resolve "Saturday" against {today}. Skip an event the article only gestures at. + What earns a row: - A specific place that actually opened recently, named. A chef, a neighbourhood, a thing that distinguishes it. - A festival, show or one-off with a date, that a person would tell a friend about. @@ -366,14 +417,42 @@ def _in_window(iso: str | None) -> bool: - subtitle: at most 8 words on why it is worth knowing. Concrete, not promotional. Never "a must-visit" or "don't miss". For an EVENT this is usually just the venue name, nothing else — the day already goes in `when`, so do not repeat "Friday" or a date here too. For a PLACE, say what makes it distinct (a chef, a neighborhood, a dish) rather than restating the venue name from the title. - when: short and human — a day name for something dated within the next couple weeks ("Friday"), a short date for anything further out ("Oct 12") so it reads as advance notice rather than this week's plans, or "opened this week", "in theaters", "streaming now" for undated rows. Pick ONE form, not both — "Friday" on its own is enough; do not also add the date ("Friday, August 29"). - url: copy verbatim from the candidate. Never invent one. -- date: for an EVENT with a known date, copy it verbatim in YYYY-MM-DD form from the candidate data. Empty string for a PLACE, a screen, or anything undated. +- date: for an EVENT with a known date, copy it verbatim in YYYY-MM-DD form from the candidate data; for one lifted out of an article, resolve it against {today} and give it in YYYY-MM-DD, using the LAST day of a multi-day event so it expires when it actually ends. An event whose last day is already before {today} is over — drop it. Empty string for a PLACE, a screen, or anything undated. - Plain ASCII. No emoji, no markdown, no exclamation marks. - subtitle and when must never say the same thing twice between them — if you catch yourself writing the same day, date, or venue in both fields, cut it from subtitle. If nothing clears the bar, return {{"rows": []}}. An empty section is correct and normal; a padded one is not.""" -def _curate(city: str, candidates: list[dict], today: date | None = None) -> list[dict]: +# The curation prompt takes a bounded amount of candidate text, and the two +# halves of the pool must not eat each other's share. Ticketmaster alone is +# ~8.6k characters for a busy metro; with one shared 12k cut the press half — +# the restaurants, fairs and family events — was clipped to a sliver, mid-item, +# and the curated section came out all concerts. +EVENT_BUDGET = 7000 +PRESS_BUDGET = 9000 + + +def _fit(candidates: list[dict]) -> list[dict]: + """Whole candidates only, each source family held to its own budget, in the + order given. Dropping an item beats cutting one: a half-written JSON object + is a candidate the model has to guess at.""" + import json + used = {"events": 0, "press": 0} + cap = {"events": EVENT_BUDGET, "press": PRESS_BUDGET} + out = [] + for c in candidates: + fam = "events" if c.get("source") == "ticketmaster.com" else "press" + n = len(json.dumps(c)) + 2 + if used[fam] + n > cap[fam]: + continue + used[fam] += n + out.append(c) + return out + + +def _curate(city: str, candidates: list[dict], today: date | None = None, + max_rows: int = CURATE_POOL) -> list[dict]: """The taste gate. One Haiku call, on the write path only. Everything upstream of this is a firehose: local press runs listicles and @@ -397,7 +476,7 @@ def _curate(city: str, candidates: list[dict], today: date | None = None) -> lis try: resp = client.messages.create( model=HAIKU_MODEL, - max_tokens=900, + max_tokens=1500, messages=[{"role": "user", "content": _CURATE_PROMPT.format( # The READER's date, not the dyno's. CLAUDE.md documents that # stating today is load-bearing here — without it the model @@ -406,7 +485,7 @@ def _curate(city: str, candidates: list[dict], today: date | None = None) -> lis # on tomorrow, so a UTC date tells the curator to drop tonight's # show as "already past". today=(today or date.today()).strftime("%A, %B %d, %Y"), city=city, - candidates=json.dumps(candidates)[:9000], max_rows=CURATE_POOL)}], + candidates=json.dumps(_fit(candidates)), max_rows=max_rows)}], ) parsed = _parse_json(resp.content[0].text) or {} except Exception as e: @@ -414,7 +493,7 @@ def _curate(city: str, candidates: list[dict], today: date | None = None) -> lis return [] rows = [] - for r in (parsed.get("rows") or [])[:CURATE_POOL]: + for r in (parsed.get("rows") or [])[:max_rows]: title = (r.get("title") or "").strip() if not title: continue @@ -424,16 +503,28 @@ def _curate(city: str, candidates: list[dict], today: date | None = None) -> lis # kind == "screen" is what puts TMDB's attribution on the page, so the # page would have credited TMDB for data TMDB never supplied. kind = r.get("kind") if r.get("kind") in ("local", "event") else "local" + topic = r.get("topic") if r.get("topic") in TOPICS else "other" + url = r.get("url") or None + date_s = _valid_iso_date(r.get("date")) + # An event lifted out of a press roundup has no structured date behind + # it, only the model's reading of prose. Undated, it can never expire, + # and "Oct 9-11" was served on the 14th. Ticketmaster rows carry a real + # date upstream, so only the press-sourced ones are held to this. + if kind == "event" and not date_s and "ticketmaster" not in _source_of(url or ""): + continue + if sum(1 for x in rows if x["topic"] == topic and topic != "other") >= MAX_PER_TOPIC: + continue rows.append({ "kind": kind, + "topic": topic, "title": title[:80], "subtitle": (r.get("subtitle") or "").strip()[:90], "when": (r.get("when") or "").strip()[:32], - "url": r.get("url") or None, - "source": _source_of(r.get("url") or ""), + "url": url, + "source": _source_of(url or ""), # Kept only to expire the row once its date passes — see # opening_snapshot. Not rendered; "when" is what the page shows. - "date": _valid_iso_date(r.get("date")), + "date": date_s, }) return rows @@ -453,6 +544,32 @@ def _is_far(row: dict, today: date) -> bool: return False +def _interleave(a: list[dict], b: list[dict]) -> list[dict]: + """Alternate two ranked lists, each keeping its own order, so slicing the + front of the result takes from both.""" + out = [] + for i in range(max(len(a), len(b))): + out += [x for x in (a[i:i + 1] + b[i:i + 1])] + return out + + +def _spread(rows: list[dict]) -> list[dict]: + """Best-first order, but one row of each topic before a second of any. + + The curated pool is ranked, and ranked pools clump: Ticketmaster's relevance + sort put three concerts ahead of everything else for St. Louis, so a section + with three slots was three concerts. A row with no topic (cached before the + field existed, or tagged "other") counts as its own topic so it is never + held back by its neighbours.""" + seen, first, rest = set(), [], [] + for i, r in enumerate(rows): + t = r.get("topic") + key = t if t and t != "other" else ("unique", i) + (rest if key in seen else first).append(r) + seen.add(key) + return first + rest + + def _rotate(rows: list, today: date, take: int) -> list: """`take` rows, starting at a different offset each day. @@ -565,7 +682,7 @@ def opening_snapshot(profile: dict) -> list[dict]: # "their city: Kirkwood, MO", it correctly dropped every venue in # St. Louis as somewhere else, which is all of them. metro = _metro(city) - candidates = _local_candidates(metro) + candidates = _local_candidates(metro, today) with _cache_lock: _candidate_cache[ckey] = candidates @@ -579,14 +696,19 @@ def opening_snapshot(profile: dict) -> list[dict]: near = _events(lat, lon) far = _events(lat, lon, start_days=7, end_days=LONG_LEAD_DAYS, size=LONG_LEAD_SIZE) - pool = list(candidates) + # Structured events only. Press is curated in its own call below, so + # neither family can crowd the other out of the prompt or the result. + pool = [] for ev in near + far: pool.append({"title": ev["title"], "url": ev.get("url"), "source": "ticketmaster.com", "blurb": " ".join(str(x) for x in (ev.get("genre"), ev.get("venue"), ev.get("date")) if x)}) - hit = _curate(metro or _metro(city), pool, today=today) + metro = metro or _metro(city) + hit = _interleave( + _curate(metro, pool, today=today, max_rows=EVENT_ROWS) if pool else [], + _curate(metro, list(candidates), today=today, max_rows=PRESS_ROWS)) with _cache_lock: _local_cache[dkey] = hit @@ -645,8 +767,8 @@ def opening_snapshot(profile: dict) -> list[dict]: # the next seven days outranks everything beyond them, so with a busy metro # the long-lead pull — Kacey Musgraves in twelve days, Zac Brown in eleven — # never won a slot and the section read as this weekend, forever. - soon = [r for r in mine if not _is_far(r, today)] - later = [r for r in mine if _is_far(r, today)] + soon = _spread([r for r in mine if not _is_far(r, today)]) + later = _spread([r for r in mine if _is_far(r, today)]) picked_local = soon[:max(local_allowance - 1, 0)] if later else soon[:local_allowance] if later and len(picked_local) < local_allowance: picked_local = picked_local + _rotate(later, today, local_allowance - len(picked_local)) diff --git a/palmer/tickers.py b/palmer/tickers.py index 5b71305..38fee65 100644 --- a/palmer/tickers.py +++ b/palmer/tickers.py @@ -46,10 +46,16 @@ # Symbol -> how it should read on the page. Yahoo's index symbols are correct # but unreadable; nobody wants "^GSPC" in their Markets section. +# What the Markets section shows for someone who follows no ticker at all, so +# every morning carries a market line. Hand-mapped like INDEX_TICKERS: search +# returns futures for indices, which would be the wrong number. +DEFAULT_MARKETS = (("^GSPC", "S&P 500"), ("^DJI", "Dow")) + INDEX_LABELS = { "^GSPC": "S&P 500", "^DJI": "Dow", "^IXIC": "Nasdaq", + "^NDX": "Nasdaq 100", "^RUT": "Russell 2000", "^VIX": "VIX", } @@ -62,6 +68,10 @@ INDEX_TICKERS = { "s&p 500": "^GSPC", "s&p500": "^GSPC", "sp500": "^GSPC", "s&p": "^GSPC", "dow jones": "^DJI", "the dow": "^DJI", + # Nasdaq 100 is not the Nasdaq Composite. Without these keys "nasdaq 100 + # updates" matched the bare "nasdaq" below and quietly served ^IXIC under + # the label "Nasdaq" — a different index, with nothing to show it. + "nasdaq 100": "^NDX", "nasdaq-100": "^NDX", "nasdaq100": "^NDX", "ndx": "^NDX", "nasdaq": "^IXIC", "russell 2000": "^RUT", "vix": "^VIX", "stock market": "^GSPC", "the market": "^GSPC", "markets": "^GSPC", } diff --git a/tests/test_home.py b/tests/test_home.py index f16da5d..9ffe8f4 100644 --- a/tests/test_home.py +++ b/tests/test_home.py @@ -394,6 +394,39 @@ def test_a_removed_ticker_does_not_come_back_from_the_stale_set(self): assert [p["label"] for p in out] == ["NVDA"] +class TestMarketsFallback: + """Everyone's morning carries a market line. Someone who follows no ticker + gets the S&P 500 and the Dow; someone who does keeps exactly their own.""" + + def _labels(self, profile, snap=None): + snap = snap or (lambda sym, label=None: {"label": label, "price": 1.0}) + with patch("palmer.datafeeds.price_snapshot", side_effect=snap) as ps: + out = home._fetch_prices(profile, None) + return [p["label"] for p in out], ps + + def test_no_topics_shows_the_broad_market(self): + labels, ps = self._labels({}) + assert labels == ["S&P 500", "Dow"] + assert [c.args[0] for c in ps.call_args_list] == ["^GSPC", "^DJI"] + + def test_topics_that_name_no_ticker_still_get_it(self): + labels, _ = self._labels({"morning_topics": ["Philadelphia Eagles news", "SpaceX news"]}) + assert labels == ["S&P 500", "Dow"] + + def test_a_tracked_ticker_replaces_it_entirely(self): + labels, _ = self._labels({"morning_topics": ["Nvidia stock"]}) + assert labels == ["NVDA"] + + def test_tracked_tickers_that_fail_do_not_fall_back_to_indices(self): + """They chose those tickers; a blip is not a reason to swap the list.""" + with patch("palmer.datafeeds.price_snapshot", return_value=None): + assert home._fetch_prices({"morning_topics": ["Nvidia stock"]}, None) == [] + + def test_a_user_who_hid_markets_gets_nothing(self): + labels, ps = self._labels({"morning_prefs": {"hidden_sections": ["markets"]}}) + assert labels == [] and ps.call_count == 0 + + class TestInvalidate: """"add apple stock" has to show up now, not in five minutes.""" diff --git a/tests/test_morning.py b/tests/test_morning.py index ebd42e1..ac3849e 100644 --- a/tests/test_morning.py +++ b/tests/test_morning.py @@ -296,6 +296,19 @@ def test_opening_is_not_required_without_data(self): body = self._required_block(payload) assert "newly open" not in body + def test_markets_are_required_when_there_is_a_price_row(self): + body = self._required_block(dict(PAYLOAD, prices=[{"label": "S&P 500", "pct_24h": -0.4}, + {"label": "Dow", "pct_24h": -0.2}])) + assert "the markets" in body and "S&P 500 and the Dow" in body + + def test_markets_are_not_required_without_prices(self): + body = self._required_block({k: v for k, v in PAYLOAD.items() if k != "prices"}) + assert "the markets" not in body + + def test_markets_are_not_required_when_the_user_hid_them(self): + payload = dict(PAYLOAD, page_prefs={"section_order": [], "hidden_sections": ["markets"]}) + assert "the markets" not in self._required_block(payload) + def test_no_structured_data_falls_back_to_a_plain_greeting_instruction(self): body = self._required_block({}) assert "plain greeting is correct" in body diff --git a/tests/test_news.py b/tests/test_news.py index f73ef11..3e442be 100644 --- a/tests/test_news.py +++ b/tests/test_news.py @@ -593,6 +593,13 @@ def test_a_company_in_a_news_topic_gets_no_ticker(self, topic): def test_the_same_company_with_a_price_word_does_resolve(self, topic, symbol): assert resolve(topic)[0] == symbol + def test_nasdaq_100_is_not_the_composite(self): + """"give me nasdaq 100 updates" matched the bare "nasdaq" key and served + ^IXIC labelled "Nasdaq" — the wrong index, silently.""" + for topic in ("Nasdaq 100 updates", "nasdaq-100", "NDX", "Nasdaq 100 index price"): + assert resolve(topic) == ("^NDX", "Nasdaq 100"), topic + assert resolve("Nasdaq composite")[0] == "^IXIC" + def test_indices_need_no_price_word(self): """"nasdaq" is unambiguously a market reference on its own.""" assert resolve("nasdaq")[0] == "^IXIC" diff --git a/tests/test_opening.py b/tests/test_opening.py index 6a88fe3..546ead5 100644 --- a/tests/test_opening.py +++ b/tests/test_opening.py @@ -41,6 +41,9 @@ ]} +N_QUERIES = len(opening._local_queries("X", date(2026, 10, 14))) + + def _seed_caches(local_rows, screen_rows, tz="America/Los_Angeles", coords=(34.0, -118.5)): """Pre-populate all three caches so a snapshot makes no outbound call. @@ -118,8 +121,8 @@ def test_two_users_in_one_metro_share_a_single_fetch(self): cl.messages.create.return_value = _resp(CURATED) opening.opening_snapshot(LA) opening.opening_snapshot(LA2) - assert tav.call_count == 2, ("one snapshot runs two local queries; the " - "second user must add none") + assert tav.call_count == N_QUERIES, ("one snapshot runs the local queries; " + "the second user must add none") def test_a_different_metro_does_not_reuse_the_first(self): opening._clear_caches() @@ -130,7 +133,7 @@ def test_a_different_metro_does_not_reuse_the_first(self): cl.messages.create.return_value = _resp(CURATED) opening.opening_snapshot(LA) opening.opening_snapshot(STL) - assert tav.call_count == 4, "St. Louis is not Los Angeles" + assert tav.call_count == 2 * N_QUERIES, "St. Louis is not Los Angeles" def test_culver_city_and_woodland_hills_land_in_one_bucket(self): """35 miles apart, one metro. The bucket is what makes that true without @@ -392,7 +395,7 @@ def test_curate_drops_a_malformed_date_rather_than_raising(self): patch.object(opening, "client") as cl: cl.messages.create.return_value = _resp({"rows": [ {"title": "Bad Date", "subtitle": "", "when": "someday", - "url": "https://t.com/1", "kind": "event", "date": "not-a-date"}]}) + "url": "https://www.ticketmaster.com/1", "kind": "event", "date": "not-a-date"}]}) rows = opening.opening_snapshot(LA) assert rows and rows[0]["date"] is None @@ -585,14 +588,14 @@ def test_two_users_with_different_tastes_still_share_one_fetch(self): with patch.object(opening, "_http_get_json", return_value={}), \ patch("palmer.weather._geocode", side_effect=lambda c: COORDS[c]), \ patch("palmer.datafeeds._search_raw", - return_value=[{"title": "gig", "url": "https://t.com/1"}]) as tav, \ + return_value=[{"title": "gig", "url": "https://www.ticketmaster.com/1"}]) as tav, \ patch.object(opening, "client") as cl: cl.messages.create.return_value = _resp( - {"rows": [{"title": "Muse", "url": "https://t.com/1", "kind": "event"}]}) + {"rows": [{"title": "Muse", "url": "https://www.ticketmaster.com/1", "kind": "event"}]}) a = opening.opening_snapshot({"city": "Culver City"}) b = opening.opening_snapshot({"city": "Woodland Hills, California", "morning_prefs": {"opening_kinds": ["screen"]}}) - assert tav.call_count == 2, "two searches for the metro, not four for two users" + assert tav.call_count == N_QUERIES, "one set of searches for the metro, not two for two users" assert [r["kind"] for r in a] == ["event"] assert b == [], "different taste, same cache, correctly filtered to nothing" @@ -1137,3 +1140,84 @@ def test_the_dispatch_refuses_to_pick(self): .split("elif b.name")[0] assert "ambiguous_shows" in block assert "Do NOT pick one yourself" in block + + +class TestNearbyVariety: + """St. Louis returned only concerts: the press half was trusted_only and + came back empty, Ticketmaster was Music and Arts, and its relevance sort + clumped. These pin the three fixes.""" + + def test_local_press_is_not_limited_to_the_allowlist(self): + """stltoday, stlpr, stlmag and fox2now are the real St. Louis newsrooms + and none is on the allowlist, so trusted_only returned nothing.""" + with patch("palmer.datafeeds._search_raw", return_value=[]) as tav: + opening._local_candidates("St. Louis", date(2026, 10, 14)) + for call in tav.call_args_list: + assert call.kwargs["trusted_only"] is False + assert call.kwargs["min_score"] >= 0.6, "an untrusted source earns its place on score" + + def test_the_queries_cover_food_festivals_and_family(self): + qs = opening._local_queries("St. Louis", date(2026, 10, 14)) + assert any("restaurant" in q for q in qs) + assert any("festivals" in q and "October 2026" in q for q in qs) + assert any("family" in q for q in qs) + + def test_a_url_found_by_two_queries_is_one_candidate(self): + hit = {"title": "Weekend roundup", "url": "https://fox2now.com/a", "content": "x" * 900, "score": 0.8} + with patch("palmer.datafeeds._search_raw", return_value=[hit]): + out = opening._local_candidates("St. Louis", date(2026, 10, 14)) + assert len(out) == 1 + assert len(out[0]["blurb"]) == 700, "long enough to lift events out of a roundup" + + def test_ticketmaster_includes_the_family_and_festival_segment(self): + with patch.object(opening, "TICKETMASTER_API_KEY", "k"), \ + patch.object(opening, "_http_get_json", return_value={}) as http: + opening._events(38.58, -90.40) + assert "segmentName=Miscellaneous" in http.call_args.args[0] + assert "Sports" not in http.call_args.args[0] + + def test_curate_keeps_a_valid_topic_and_defaults_the_rest(self): + tm = "https://www.ticketmaster.com/" + rows = [{"title": "A", "kind": "event", "topic": "comedy", "url": tm + "a"}, + {"title": "B", "kind": "event", "topic": "nonsense", "url": tm + "b"}, + {"title": "C", "kind": "local"}] + with patch.object(opening, "client") as cl: + cl.messages.create.return_value = _resp({"rows": rows}) + out = opening._curate("St. Louis", [{"title": "x"}]) + assert [r["topic"] for r in out] == ["comedy", "other", "other"] + + def test_spread_takes_one_of_each_topic_before_a_second(self): + rows = [{"title": t, "topic": k} for t, k in + [("m1", "music"), ("m2", "music"), ("m3", "music"), ("c1", "comedy"), ("f1", "food")]] + assert [r["title"] for r in opening._spread(rows)][:3] == ["m1", "c1", "f1"] + assert len(opening._spread(rows)) == 5, "nothing is dropped, only reordered" + + def test_untagged_rows_are_never_held_back(self): + rows = [{"title": "a"}, {"title": "b"}, {"title": "c", "topic": "other"}] + assert [r["title"] for r in opening._spread(rows)] == ["a", "b", "c"] + + def test_a_clumped_pool_comes_out_varied(self): + """Three concerts ranked ahead of everything else used to be the whole + section.""" + today = _seed_caches([ + {"kind": "event", "topic": "music", "title": "Gig 1", "date": None}, + {"kind": "event", "topic": "music", "title": "Gig 2", "date": None}, + {"kind": "event", "topic": "music", "title": "Gig 3", "date": None}, + {"kind": "event", "topic": "comedy", "title": "Standup", "date": None}, + {"kind": "local", "topic": "food", "title": "New Diner", "date": None}, + ], [], tz="America/Chicago", coords=opening._bucket(38.58, -90.40)) + with patch("palmer.weather._geocode", side_effect=lambda c: COORDS[c]): + rows = opening.opening_snapshot(dict(STL, morning_prefs={"opening_kinds": ["local", "event"]})) + assert {r["topic"] for r in rows[:3]} == {"music", "comedy", "food"}, today + + def test_the_two_halves_of_the_pool_do_not_eat_each_other(self): + """Ticketmaster is ~8.6k characters on its own; one shared cut clipped + the press half to a sliver and the section came out all concerts.""" + events = [{"title": f"Gig {i}", "url": "https://t.com/" + "x" * 90, "source": "ticketmaster.com", + "blurb": "Rock Enterprise Center 2026-10-20"} for i in range(60)] + press = [{"title": f"Roundup {i}", "url": f"https://fox2now.com/{i}", "source": "fox2now.com", + "blurb": "y" * 700} for i in range(12)] + kept = opening._fit(events + press) + assert sum(1 for c in kept if c["source"] == "fox2now.com") >= 8, "press keeps its own share" + assert sum(1 for c in kept if c["source"] == "ticketmaster.com") >= 20 + assert all(isinstance(c, dict) and c["title"] for c in kept), "whole items only" diff --git a/tests/test_page.py b/tests/test_page.py index 7849638..603e429 100644 --- a/tests/test_page.py +++ b/tests/test_page.py @@ -1037,3 +1037,11 @@ def test_the_description_leads_with_the_weather_when_named(self): def test_the_icon_is_a_png(self): assert cards.render_icon()[:8] == b"\x89PNG\r\n\x1a\n" + + def test_the_icon_route_serves_it(self): + """The route imported `cards` by its pre-package name and 500'd in + production while every unit test of render_icon passed.""" + from fastapi.testclient import TestClient + from palmer.main import app + r = TestClient(app).get("/icon.png") + assert r.status_code == 200 and r.headers["content-type"] == "image/png" diff --git a/tests/test_sports.py b/tests/test_sports.py index c6ee823..1da1b23 100644 --- a/tests/test_sports.py +++ b/tests/test_sports.py @@ -204,6 +204,18 @@ def test_the_tool_descriptions_agree(self): assert "without unfollowing" in setter["description"] assert setter["input_schema"]["required"] == ["mode"] + def test_follow_dicts_in_sports_teams_do_not_crash_the_prompt(self): + """A production profile held team dicts in sports_teams; the offer block + joined them as strings and every turn for that user raised.""" + from palmer import agent + base = {"intro_sent": True, "name": "Jeff", "city": "Kirkwood, MO", + "sports_teams": [{"league": "mlb", "abbrev": "STL", "name": "St. Louis Cardinals"}, + {"league": "nfl", "abbrev": "CHI", "name": "Chicago Bears"}]} + with patch.object(agent, "get_profile", return_value=dict(base)), \ + patch.object(agent, "get_history", return_value=[]): + system = agent._build_system("+1") + assert "St. Louis Cardinals, Chicago Bears" in system + def test_the_offer_block_fires_once_for_a_named_team(self): from palmer import agent from palmer import userprofile diff --git a/tests/test_time.py b/tests/test_time.py index 261c525..7ad5f86 100644 --- a/tests/test_time.py +++ b/tests/test_time.py @@ -849,4 +849,5 @@ def test_the_snapshot_passes_the_local_day(self): import inspect from palmer import opening src = inspect.getsource(opening.opening_snapshot) - assert "_curate(metro or _metro(city), pool, today=today)" in src + assert "_curate(metro, pool, today=today" in src, "events call" + assert "_curate(metro, list(candidates), today=today" in src, "press call"