"""catalyst_build.py — builds catalysts.json for the Bird's Eye dashboard.

Sources, in tier order (same order as Bernie's catalyst check):
  1. Bernie Wire (--wire)          -> Neo's own drops since the last session close; always listed first
  2. Alpha Vantage NEWS_SENTIMENT  -> headlines tagged to a CORE50 name, with AV's per-ticker sentiment
  2b FMP stock news (--fmp)        -> same slot, used only when AV returns no feed (premium-gated key)
  3. Finviz news (finviz.json)     -> the per-ticker headlines already pulled for earnings; no sentiment
                                      of its own, so tone is the keyword read and the tier is weighted down
  +  Alpha Vantage EARNINGS_CALENDAR -> earnings due within EARN_DAYS calendar days
AV calls are made one after the other (free tier: 25/day, 1/sec, never in parallel).
Tier weight (TIER_W) multiplies relevance when the per-name tone is averaged, so a Finviz headline
can light an icon on a clear catalyst kind but cannot swing tone against a scored source.

Output (beside quotes.json, served by the receiver as a static file — no receiver change):
  {"asof", "date", "window_from", "names": {SYM: {"cat": bool, "tone": "bull"|"bear"|"neutral",
     "kinds": [...], "earnings": {...}|null, "items": [...]}}}

`cat` is true only for a real catalyst: earnings in the window, or a headline that matches a catalyst
kind (rating change, earnings/guidance, FDA/trial, deal/contract, financing, big mover). Plain mentions
and holdings/insider filings are kept in `items` for the detail sheet but do not light the icon.

Usage
  python catalyst_build.py --apikey KEY            # fetch from Alpha Vantage and write catalysts.json
  python catalyst_build.py --earnings e.csv --news n.json --wire wire.json   # from saved responses (no calls)
  wire.json = the Bernie Wire page's <script id="bernie-feed"> JSON ({"entries":[{ts,ticker,headline,note,tone?}]});
  an entry's optional "tone" (bull/bear/neutral) overrides the keyword read.
  python catalyst_build.py --news n.json --wire wire.json   # earnings from finviz.json beside the script (2026-10-13)
  python catalyst_build.py --news n.json --fmp fmp.json --wire wire.html   # FMP steps in if n.json's feed is empty
  fmp.json = FMP search-stock-news rows ([{symbol,title,publishedDate,site,url,text}, ...] or {"items":[...]}).
  Finviz news needs no flag: it is read from the same finviz.json the earnings come from (2026-10-21).
  python catalyst_build.py --selftest
"""
import argparse, csv, io, json, os, re, sys, time, urllib.request
from datetime import datetime, timedelta, timezone

HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "catalysts.json")
CORE50 = ("AXTI MRNA AEHR AAOI TEM SMTC HUT MXL TWST DOCN CRCL RBRK SEI P TTMI FROG MSTR MP TXG INTC "
          "WING CRWV EIX AFRM DKS ASTS PBF VSXY AVAV HOOD HPE HNGE SYRE ANF DAR RKLB VSAT NOW SWKS KGS "
          "CAVA CVNA FLUT ESTC MCHP SFM FLR CART CTSH FCX").split()
EARN_DAYS = 7        # earnings within this many calendar days count as a catalyst

# 2026-10-13: the CORE50 tuple above is the 09-18 roster and is now only the selftest's
# universe and a last-resort fallback. 18 of the live 50 were missing from it (and 18 cut
# names were still lighting icons), so main() reads the live roster instead:
#   1. roster.json beside this script (the dashboard's copy of the morning scan)
#   2. finviz.json names (the Finviz feed tracks the shortlist + SPY/QQQ)
#   3. the tuple above, flagged in the output as stale
def load_universe(here=None):
    here = here or HERE
    try:
        r = json.load(open(os.path.join(here, "roster.json"), encoding="utf-8"))
        names = list((r.get("names") or {}).keys())
        if len(names) >= 10:
            return names, f"roster.json ({r.get('log_date') or '?'}, {len(names)} names)"
    except Exception:
        pass
    try:
        f = json.load(open(os.path.join(here, "finviz.json"), encoding="utf-8"))
        names = [s for s in (f.get("names") or {}) if s not in ("SPY", "QQQ")]
        if len(names) >= 10:
            return names, f"finviz.json ({len(names)} names)"
    except Exception:
        pass
    return list(CORE50), "built-in 2026-09-18 list (STALE -- roster.json and finviz.json unreadable)"


FINVIZ_MAX_AGE_DAYS = 4   # earnings dates barely move; older than this and AV is used instead


def finviz_earnings(fvz_doc, uni, today):
    """{SYM: (date, 'pre-market'|'post-market'|None)} from finviz.json's per-name 'earnings'
    ('10/15/2026 4:30:00 PM'). Finviz reports the last date until the next is set, so past
    dates simply fall outside the window."""
    out = {}
    for sym, n in ((fvz_doc or {}).get("names") or {}).items():
        if sym not in uni or not n.get("earnings"):
            continue
        m = re.match(r"(\d+)/(\d+)/(\d{4})(?: (\d+):(\d+):\d+ (AM|PM))?", str(n["earnings"]))
        if not m:
            continue
        mo, d, y, hh, mm, ap = m.groups()
        try:
            day = datetime(int(y), int(mo), int(d)).date()
        except ValueError:
            continue
        when = None
        if hh:
            h = int(hh) % 12 + (12 if ap == "PM" else 0)
            when = "pre-market" if (h, int(mm)) < (9, 30) else "post-market" if h >= 16 else "during market"
        out[sym] = (day, when)
    return out
MIN_REL = 0.5        # AV relevance below this = the article is not about the name
HARD_REL = 0.8       # a catalyst kind only counts when the article is mainly about the name;
                     # below this a sector piece ("Rocket Lab jumps..." tagged to HPE) is a mention
TONE_CUT = 0.15      # AV's own boundary between Neutral and Somewhat-Bullish/Bearish
MAX_ITEMS = 4

# 2026-10-21: tier weight, applied to relevance when the per-name tone is averaged and when
# items are ordered. Only the Wire is Neo's own read; AV is the only source with a real
# sentiment score; FMP and Finviz are headlines whose tone is the same keyword read the Wire
# uses, so Finviz -- which is also attached to a ticker by Finviz rather than by relevance --
# is held at half weight.
TIER_W = {"wire": 2.0, "av": 1.0, "fmp": 1.0, "finviz": 0.5}
FVZ_REL = 0.9        # Finviz files a headline under a ticker deliberately: treat as high relevance ...
FVZ_SECTOR_MIN = 2   # ... unless the same headline is filed under this many tracked names, which makes
                     # it a sector/sympathy piece ("Space stocks rally on SpaceX results") -> a mention
FMP_REL = 1.0        # FMP articles are fetched per symbol
KEYWORD_SCORE = {"bull": 0.5, "bear": -0.5, "neutral": 0.0}   # stand-in for a real sentiment score

# Order matters: first match wins. "soft" kinds are shown but never light the icon.
KINDS = [
    ("mention",  r"transcript|to present at|to participate|conference|investor alert|fiduciar|law firm|class action|investigat|stocks? to watch|final trades"),
    ("rating",   r"upgrad|downgrad|price target|initiat|reiterat|outperform|underperform|overweight|underweight|strong-buy|\bbuy rating|\bsell rating"),
    ("earnings", r"earnings|quarterly results|\bq[1-4]\b|\beps\b|guidance|outlook|forecast|preannounc"),
    ("fda",      r"\bfda\b|approval|clinical|trial|phase [123i]"),
    ("deal",     r"acqui|merger|buyout|takeover|to buy |contract|partnership|awarded|\bdeal\b"),
    ("finance",  r"offering|dilut|convertible|buyback|repurchase|stock split|debt"),
    ("mover",    r"soar|surg|jump|plung|tumbl|sink|rall|spike|shares (are )?(up|down)|why .* (stock|shares)|what's (going on|happening)"),
]
# holdings must be tested before "mover" so "Sell Alert: X Cashes Out $2M" is not read as a price move
KINDS.insert(6, ("holdings", r"insider|sell alert|cashes out|sells [\d,]+ shares|buys [\d,]+ shares|invests \$|stake in|position in|13f|stock purchase"))
SOFT = {"holdings", "mention"}


def kind_of(title):
    t = title.lower()
    for k, rx in KINDS:
        if re.search(rx, t):
            return k
    return "mention"


# Keyword tone for Wire headlines (AV supplies its own sentiment). First rule that fires wins:
# a price-target move outranks the rating it is attached to ("Maintains Outperform, Lowers PT" = bearish).
TONE_RULES = [
    ("bull", r"raises price target|raises (its |the )?(q\d |fy|full-year |annual |\d{4} )?(adjusted )?(ebitda |revenue |sales )?(guidance|outlook)|upgrad"),
    ("bear", r"lowers price target|cuts price target|lowers (its |the )?.{0,20}(guidance|outlook)|downgrad|cuts? guidance"),
    ("bear", r"offering|convertible|dilut|warns|miss(es|ed)?\b|slides|slid\b|falls|fell\b|drops|plung|tumbl|sinks|trading lower|sell alert|cashes out|investor alert|investigat"),
    ("bull", r"initiat\w* .{0,40}(buy|overweight|outperform)|beat|met (its )?primary endpoint|repurchase|buyback|surg|soar|jump|gain|trading higher|rall|pops|record|awarded|wins\b|approval"),
]


def wire_tone(text):
    t = text.lower()
    for tone, rx in TONE_RULES:
        if re.search(rx, t):
            return tone
    return "neutral"


def et_offset(dt_utc):
    """US Eastern offset for a UTC datetime, without zoneinfo (Windows Python has no tz database without tzdata)."""
    y = dt_utc.year
    mar1 = datetime(y, 3, 1); start = mar1 + timedelta(days=(6 - mar1.weekday()) % 7 + 7)   # 2nd Sunday of March
    nov1 = datetime(y, 11, 1); end = nov1 + timedelta(days=(6 - nov1.weekday()) % 7)         # 1st Sunday of November
    naive = dt_utc.replace(tzinfo=None)
    return timedelta(hours=-4) if start + timedelta(hours=7) <= naive < end + timedelta(hours=6) else timedelta(hours=-5)


def last_session_close(now_utc):
    """UTC time of the most recent weekday 16:00 ET at or before now — the Wire's freshness window start.
    Weekends roll back to Friday. Market holidays are not modelled: the window then starts at the holiday's
    16:00, which only drops Wire entries dropped before 16:00 on the holiday itself."""
    off = et_offset(now_utc)
    et = now_utc.replace(tzinfo=None) + off
    d = et.date()
    if et.hour < 16:
        d -= timedelta(days=1)
    while d.weekday() >= 5:
        d -= timedelta(days=1)
    close_et = datetime(d.year, d.month, d.day, 16, 0)
    return (close_et - et_offset(datetime(d.year, d.month, d.day, 20))).replace(tzinfo=timezone.utc)


def tone_of(score):
    return "bull" if score >= TONE_CUT else "bear" if score <= -TONE_CUT else "neutral"


def parse_av_time(s):
    # AV: 20260918T160031
    try:
        return datetime.strptime(s, "%Y%m%dT%H%M%S")
    except Exception:
        return None


def parse_wall_time(s, iso_ok=True):
    """A wall-clock stamp from Finviz or FMP ('2026-10-21 08:24:08') -> aware UTC.
    Both publish in ET without a zone. An ISO stamp that carries its own offset is taken at face value.
    et_offset() wants UTC and is handed ET here: the two disagree only for a stamp within five hours
    of a DST switch, which at worst moves it either side of a 16:00 window edge it was already at."""
    t = str(s or "").strip()
    if not t:
        return None
    if iso_ok and ("T" in t or t.endswith("Z")):
        try:
            d = datetime.fromisoformat(t.replace("Z", "+00:00"))
            return d if d.tzinfo else d.replace(tzinfo=timezone.utc)
        except ValueError:
            pass
    for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d"):
        try:
            et = datetime.strptime(t, fmt)
        except ValueError:
            continue
        return (et - et_offset(et)).replace(tzinfo=timezone.utc)
    return None


def finviz_news(fvz_doc, uni, since_utc):
    """Tier 3: finviz.json's per-ticker headlines dropped since the last session close, as
    [(SYM, item), ...]. The file's own 'tone' (a keyword read of the title, per its source note)
    is used when present. A headline filed under FVZ_SECTOR_MIN or more of the tracked names is a
    sector piece rather than a catalyst for any one of them, so it is demoted to a mention."""
    feed = (fvz_doc or {}).get("news") or {}
    spread = {}
    for items in feed.values():
        for a in items or []:
            k = ((a or {}).get("title") or "").lower()[:70]
            spread[k] = spread.get(k, 0) + 1
    out = []
    for sym, items in feed.items():
        if sym not in uni:
            continue
        for a in items or []:
            title = (a or {}).get("title") or ""
            ts = parse_wall_time((a or {}).get("t"), iso_ok=False)
            if not title or not ts or (since_utc and ts < since_utc):
                continue
            tone = a.get("tone") if a.get("tone") in ("bull", "bear", "neutral") else wire_tone(title)
            sector = spread.get(title.lower()[:70], 0) >= FVZ_SECTOR_MIN
            out.append((sym, {
                "kind": "mention" if sector else kind_of(title), "title": title,
                "url": a.get("url"), "source": a.get("src") or "Finviz",
                "tier": "finviz", "w": TIER_W["finviz"],
                "time": ts.strftime("%Y%m%dT%H%M%S"), "tone": tone,
                "score": KEYWORD_SCORE[tone], "rel": FVZ_REL}))
    return out


def fmp_news(fmp_doc, uni, since_utc):
    """Tier 2 fallback: FMP stock-news rows as [(SYM, item), ...]. Accepts the bare list or a
    wrapper dict. FMP carries no sentiment either, so tone is the keyword read over the title and
    the head of the body text."""
    rows = fmp_doc
    if isinstance(rows, dict):
        rows = next((rows[k] for k in ("items", "feed", "content", "data", "results", "news")
                     if isinstance(rows.get(k), list)), [])
    out = []
    for a in rows or []:
        if not isinstance(a, dict):
            continue
        sym = str(a.get("symbol") or a.get("ticker") or "").strip().upper()
        title = a.get("title") or ""
        ts = parse_wall_time(a.get("publishedDate") or a.get("date") or a.get("publishedAt"))
        if sym not in uni or not title or not ts or (since_utc and ts < since_utc):
            continue
        tone = wire_tone(title + " " + str(a.get("text") or "")[:300])
        out.append((sym, {
            "kind": kind_of(title), "title": title, "url": a.get("url"),
            "source": a.get("site") or a.get("publisher") or "FMP",
            "tier": "fmp", "w": TIER_W["fmp"],
            "time": ts.strftime("%Y%m%dT%H%M%S"), "tone": tone,
            "score": KEYWORD_SCORE[tone], "rel": FMP_REL}))
    return out


def build(earn_csv, news_doc, today, universe=CORE50, wire_doc=None, since_utc=None, fvz_doc=None,
          fmp_doc=None):
    uni = set(universe)
    names = {}

    def slot(sym):
        return names.setdefault(sym, {"cat": False, "tone": "neutral", "kinds": [], "earnings": None, "items": []})

    # --- earnings: Finviz first (2026-10-13), Alpha Vantage only for names Finviz did not cover
    def set_earn(sym, d, when, estimate=None):
        days = (d - today).days
        if 0 <= days <= EARN_DAYS:
            n = slot(sym)
            label = "today" if days == 0 else "tomorrow" if days == 1 else d.strftime("%a %b %d").replace(" 0", " ")
            n["earnings"] = {"date": d.isoformat(), "days": days, "time": when or None,
                             "estimate": estimate or None,
                             "label": f"Earnings {label}{' ' + when if when else ''}"}
            n["cat"] = True
            n["kinds"].append("earnings")
    fz = finviz_earnings(fvz_doc, uni, today)
    for sym, (d, when) in fz.items():
        set_earn(sym, d, when)
    for r in csv.DictReader(io.StringIO(earn_csv or "")):
        fsym = (r.get("symbol") or "").strip()
        if fsym in fz and fz[fsym][0] >= today:
            continue   # Finviz already has an upcoming date; a past Finviz date lets AV's next one through
        sym = (r.get("symbol") or "").strip()
        if sym not in uni:
            continue
        try:
            d = datetime.strptime(r["reportDate"], "%Y-%m-%d").date()
        except Exception:
            continue
        days = (d - today).days
        if 0 <= days <= EARN_DAYS:
            n = slot(sym)
            when = (r.get("timeOfTheDay") or "").strip()
            label = "today" if days == 0 else "tomorrow" if days == 1 else d.strftime("%a %b %d").replace(" 0", " ")
            n["earnings"] = {"date": d.isoformat(), "days": days, "time": when or None,
                             "estimate": r.get("estimate") or None,
                             "label": f"Earnings {label}{' ' + when if when else ''}"}
            n["cat"] = True
            n["kinds"].append("earnings")

    # --- news
    per = {}
    for a in (news_doc or {}).get("feed", []):
        title = a.get("title") or ""
        for ts in a.get("ticker_sentiment", []):
            sym = ts.get("ticker")
            if sym not in uni:
                continue
            try:
                rel = float(ts.get("relevance_score", 0)); sc = float(ts.get("ticker_sentiment_score", 0))
            except ValueError:
                continue
            if rel < MIN_REL:
                continue
            per.setdefault(sym, []).append({
                "kind": kind_of(title) if rel >= HARD_REL else "mention", "title": title, "url": a.get("url"), "source": a.get("source"),
                "tier": "av", "w": TIER_W["av"],
                "time": a.get("time_published"), "tone": tone_of(sc), "score": round(sc, 3), "rel": round(rel, 3)})

    # --- Bernie Wire (tier 1): his own drops since the last session close
    for e in (wire_doc or {}).get("entries", []):
        sym = (e.get("ticker") or "").strip().upper()
        if sym not in uni:
            continue
        try:
            ts = datetime.fromisoformat(e["ts"].replace("Z", "+00:00"))
        except Exception:
            continue
        if since_utc and ts < since_utc:
            continue
        head = e.get("headline") or ""
        text = head + " " + (e.get("note") or "")
        tone = e.get("tone") if e.get("tone") in ("bull", "bear", "neutral") else wire_tone(text)
        per.setdefault(sym, []).append({
            "kind": kind_of(head), "title": head, "url": None, "source": "Bernie Wire", "wire": True,
            "tier": "wire", "w": TIER_W["wire"],
            "time": ts.strftime("%Y%m%dT%H%M%S"), "tone": tone,
            "score": KEYWORD_SCORE[tone], "rel": 1.0})

    # --- FMP (AV's slot when AV came back empty) and Finviz (tier 3), both already window-filtered
    for sym, item in fmp_news(fmp_doc, uni, since_utc) + finviz_news(fvz_doc, uni, since_utc):
        per.setdefault(sym, []).append(item)

    for sym, items in per.items():
        # hard catalysts first, then most relevant, then newest
        items.sort(key=lambda i: i["time"] or "", reverse=True)          # newest first ...
        items.sort(key=lambda i: (i["kind"] in SOFT, -i.get("w", 1.0), -i["rel"]))   # hard first, best tier first, most relevant
        seen, uniq = set(), []
        for i in items:
            key = i["title"].lower()[:60]
            if key in seen:
                continue
            seen.add(key); uniq.append(i)
        n = slot(sym)
        n["items"] = uniq[:MAX_ITEMS]
        # counted over every deduped item, not just the MAX_ITEMS kept for the detail sheet, so the
        # number says what each source contributed (n["wire"] below stays hard-only for the dashboard)
        n["tiers"] = {t: c for t, c in ((t, sum(1 for i in uniq if i.get("tier") == t))
                                        for t in ("wire", "av", "fmp", "finviz")) if c}
        hard = [i for i in uniq if i["kind"] not in SOFT]
        if hard:
            n["cat"] = True
            for i in hard:
                if i["kind"] not in n["kinds"]:
                    n["kinds"].append(i["kind"])
            wt = lambda i: i["rel"] * i.get("w", 1.0)               # TIER_W: Wire x2, AV/FMP x1, Finviz x0.5
            w = sum(wt(i) for i in hard)
            n["score"] = round(sum(i["score"] * wt(i) for i in hard) / w, 3) if w else 0
            n["wire"] = sum(1 for i in hard if i.get("wire"))
            n["tone"] = tone_of(n["score"])

    return names


def av_get(params, key):
    q = "&".join(f"{k}={v}" for k, v in {**params, "apikey": key}.items())
    with urllib.request.urlopen("https://www.alphavantage.co/query?" + q, timeout=60) as r:
        return r.read().decode("utf-8")


def write(names, today, window_from, sources):
    doc = {"asof": datetime.now(timezone.utc).isoformat(timespec="seconds"), "date": today.isoformat(),
           "window_from": window_from, "source": " + ".join(sources),
           "count": sum(1 for n in names.values() if n["cat"]), "names": names}
    tmp = OUT + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        json.dump(doc, f, indent=1)
    os.replace(tmp, OUT)  # atomic: the dashboard may fetch mid-write
    return doc


def selftest():
    ok = 0
    def check(c, msg):
        nonlocal ok
        if not c:
            print("FAIL:", msg); sys.exit(1)
        ok += 1
    today = datetime(2026, 9, 18).date()
    csvt = ("symbol,name,reportDate,fiscalDateEnding,estimate,currency,timeOfTheDay\n"
            "INTC,Intel,2026-09-18,,0.2,USD,post-market\nHPE,HPE,2026-09-24,,,USD,\n"
            "AEHR,Aehr,2026-10-05,,,USD,\nZZZZ,Other,2026-09-18,,,USD,\n")
    news = {"feed": [
        {"title": "Hut 8 Upgraded by Wells Fargo to Strong-Buy", "url": "u1", "time_published": "20260918T120000",
         "ticker_sentiment": [{"ticker": "HUT", "relevance_score": "1", "ticker_sentiment_score": "0.4"}]},
        {"title": "Morgan Stanley Lowers Price Target on Edison", "url": "u2", "time_published": "20260918T110000",
         "ticker_sentiment": [{"ticker": "EIX", "relevance_score": "0.9", "ticker_sentiment_score": "-0.3"}]},
        {"title": "Bank of America Invests $129M in Darling", "url": "u3", "time_published": "20260918T100000",
         "ticker_sentiment": [{"ticker": "DAR", "relevance_score": "1", "ticker_sentiment_score": "0.5"}]},
        {"title": "Why Are Moderna Shares Soaring Today", "url": "u4", "time_published": "20260918T090000",
         "ticker_sentiment": [{"ticker": "MRNA", "relevance_score": "1", "ticker_sentiment_score": "0.5"},
                              {"ticker": "RKLB", "relevance_score": "0.2", "ticker_sentiment_score": "0.9"}]},
        {"title": "Is AeroVironment Becoming a Platform?", "url": "u5", "time_published": "20260918T080000",
         "ticker_sentiment": [{"ticker": "AVAV", "relevance_score": "1", "ticker_sentiment_score": "0.5"}]},
    ]}
    n = build(csvt, news, today)
    check(n["INTC"]["cat"] and n["INTC"]["earnings"]["days"] == 0, "earnings today flags")
    check("today" in n["INTC"]["earnings"]["label"] and "post-market" in n["INTC"]["earnings"]["label"], "earnings label")
    check(n["HPE"]["cat"] and n["HPE"]["earnings"]["days"] == 6, "earnings within window flags")
    check("AEHR" not in n, "earnings beyond window ignored")
    check("ZZZZ" not in n, "non-universe ignored")
    check(n["HUT"]["cat"] and n["HUT"]["tone"] == "bull" and n["HUT"]["kinds"] == ["rating"], "upgrade = bullish rating")
    check(n["EIX"]["cat"] and n["EIX"]["tone"] == "bear", "PT cut = bearish")
    check(not n["DAR"]["cat"] and n["DAR"]["items"][0]["kind"] == "holdings", "holdings shown, no icon")
    check(n["MRNA"]["cat"] and n["MRNA"]["kinds"] == ["mover"], "mover flags")
    check("RKLB" not in n, "low relevance ignored")
    check(not n["AVAV"]["cat"] and n["AVAV"]["items"][0]["kind"] == "mention", "plain mention, no icon")
    n2 = build("", {"feed": [{"title": "Rocket Lab Jumps 6% on Deal", "time_published": "x",
        "ticker_sentiment": [{"ticker": "HPE", "relevance_score": "0.6", "ticker_sentiment_score": "0.3"}]}]}, today)
    check(not n2["HPE"]["cat"] and n2["HPE"]["items"][0]["kind"] == "mention", "side-mention never lights the icon")
    check(kind_of("Freeport Shares Up 2.3% on Analyst Upgrade") == "rating", "rating beats mover")
    # Wire
    since = datetime(2026, 9, 17, 20, 0, tzinfo=timezone.utc)
    wire = {"entries": [
        {"ts": "2026-09-18T11:00:00.000Z", "ticker": "CRWV", "headline": "CoreWeave Prices Upsized $3.7 Billion Convertible Senior Notes Offering"},
        {"ts": "2026-09-18T11:01:00.000Z", "ticker": "CAVA", "headline": "CAVA Group Board Approves $100M Share Repurchase Program"},
        {"ts": "2026-09-18T11:59:00.000Z", "ticker": "WING", "headline": "RBC Capital Maintains Outperform on Wingstop, Lowers Price Target to $150"},
        {"ts": "2026-09-18T11:59:00.000Z", "ticker": "AFRM", "headline": "Sell Alert: Siphelele Jiyane Cashes Out $1.95M In Affirm Holdings Stock"},
        {"ts": "2026-09-18T11:59:00.000Z", "ticker": "CRCL", "headline": "Full Transcript: Circle Internet Group Q2 2026 Earnings Call"},
        {"ts": "2026-09-17T12:00:00.000Z", "ticker": "SEI", "headline": "Solaris Raises Q3 Adjusted EBITDA Guidance"},
        {"ts": "2026-09-18T11:59:00.000Z", "ticker": "EIX", "headline": "Some Wire Upgrade", "tone": "bull"},
        {"ts": "2026-09-18T11:59:00.000Z", "ticker": "AAPL", "headline": "Upgrade"},
    ]}
    w = build("", news, today, wire_doc=wire, since_utc=since)
    check(w["CRWV"]["cat"] and w["CRWV"]["tone"] == "bear" and w["CRWV"]["kinds"] == ["finance"], "convertible = bearish financing")
    check(w["CAVA"]["cat"] and w["CAVA"]["tone"] == "bull", "buyback = bullish")
    check(w["WING"]["tone"] == "bear" and w["WING"]["kinds"] == ["rating"], "PT cut outranks the Outperform")
    check(not w["AFRM"]["cat"] and w["AFRM"]["items"][0]["kind"] == "holdings", "insider sale shown, no icon")
    check(not w["CRCL"]["cat"], "call transcript is not a catalyst")
    check("SEI" not in w, "Wire entry before the last close is not fresh")
    check("AAPL" not in w, "Wire entry outside CORE50 ignored")
    check(w["EIX"]["items"][0]["source"] == "Bernie Wire", "Wire listed first, ahead of AV")
    check(w["EIX"]["tone"] == "bull", "Wire (x2, tone override) outweighs a bearish AV rating")
    check(wire_tone("Moderna shares are trading lower after Rothschild downgraded") == "bear", "downgrade bearish")
    check(wire_tone("Cantor initiated coverage with an Overweight rating") == "bull", "initiation bullish")
    # session-close window
    fri = last_session_close(datetime(2026, 9, 21, 12, 35, tzinfo=timezone.utc))   # Mon 08:35 EDT
    check(fri == datetime(2026, 9, 18, 20, 0, tzinfo=timezone.utc), "Monday morning window starts Friday 16:00 ET")
    thu = last_session_close(datetime(2026, 9, 18, 12, 35, tzinfo=timezone.utc))
    check(thu == datetime(2026, 9, 17, 20, 0, tzinfo=timezone.utc), "Friday morning window starts Thursday 16:00 ET")
    est = last_session_close(datetime(2026, 11, 3, 13, 35, tzinfo=timezone.utc))   # Tue 08:35 EST
    check(est == datetime(2026, 11, 2, 21, 0, tzinfo=timezone.utc), "EST close is 21:00 UTC")
    # Finviz earnings (2026-10-13)
    fz = {"names": {"INTC": {"earnings": "9/18/2026 4:30:00 PM"}, "AEHR": {"earnings": "9/21/2026 8:00:00 AM"},
                    "HPE": {"earnings": "8/1/2026 4:30:00 PM"}, "MRNA": {"earnings": "9/19/2026"}}}
    f = build(csvt, {"feed": []}, today, fvz_doc=fz)
    check(f["INTC"]["earnings"]["time"] == "post-market" and f["INTC"]["earnings"]["days"] == 0, "finviz AMC today")
    check(f["AEHR"]["earnings"]["time"] == "pre-market" and f["AEHR"]["earnings"]["days"] == 3, "finviz BMO, overrides AV's later date")
    check(f["HPE"]["earnings"]["days"] == 6, "finviz still shows HPE's August date -- AV's upcoming 09-24 date is used")
    check(f["MRNA"]["earnings"]["days"] == 1 and f["MRNA"]["earnings"]["time"] is None, "finviz date without a time")
    # Finviz news (tier 3, 2026-10-21)
    fsince = datetime(2026, 9, 17, 20, 0, tzinfo=timezone.utc)
    fzn = {"names": {}, "news": {
        "TWST": [{"t": "2026-09-18 08:00:00", "title": "Twist Bioscience Awarded $40M Contract", "src": "PRNewswire", "url": "f1", "tone": "bull"}],
        "SMTC": [{"t": "2026-09-18 08:05:00", "title": "Space Stocks Rally On SpaceX Results", "src": "Yahoo", "url": "f2", "tone": "bull"}],
        "AXTI": [{"t": "2026-09-18 08:05:00", "title": "Space Stocks Rally On SpaceX Results", "src": "Yahoo", "url": "f3", "tone": "bull"}],
        "MXL":  [{"t": "2026-09-16 09:00:00", "title": "MaxLinear Upgraded To Buy", "src": "Yahoo", "url": "f4", "tone": "bull"}],
        "SPY":  [{"t": "2026-09-18 08:10:00", "title": "Futures Rise", "src": "Yahoo", "url": "f5", "tone": "bull"}],
    }}
    fn = build("", {"feed": []}, today, since_utc=fsince, fvz_doc=fzn)
    check(fn["TWST"]["cat"] and fn["TWST"]["kinds"] == ["deal"] and fn["TWST"]["tone"] == "bull", "finviz headline lights a deal")
    check(fn["TWST"]["items"][0]["tier"] == "finviz" and fn["TWST"]["items"][0]["w"] == TIER_W["finviz"], "finviz item carries tier 3 weight")
    check(fn["TWST"]["tiers"] == {"finviz": 1}, "tier counts recorded")
    check(not fn["SMTC"]["cat"] and fn["SMTC"]["items"][0]["kind"] == "mention", "headline filed under 2+ names is a sector piece")
    check("MXL" not in fn, "finviz headline before the last close is not fresh")
    check("SPY" not in fn, "finviz name outside the universe ignored")
    check(parse_wall_time("2026-09-18 08:00:00") == datetime(2026, 9, 18, 12, 0, tzinfo=timezone.utc), "ET wall clock read as UTC")
    check(parse_wall_time("2026-09-18T12:00:00Z") == datetime(2026, 9, 18, 12, 0, tzinfo=timezone.utc), "ISO stamp taken at face value")
    check(parse_wall_time("") is None and parse_wall_time("not a date") is None, "unparseable stamp dropped")
    # FMP (tier 2 fallback)
    fmpdoc = [{"symbol": "HUT", "publishedDate": "2026-09-18 09:30:00", "title": "Hut 8 Wins $200M Hosting Contract", "site": "Reuters", "url": "m1"},
              {"symbol": "EIX", "publishedDate": "2026-09-16 09:30:00", "title": "Edison Downgraded", "site": "Reuters", "url": "m2"},
              {"symbol": "AAPL", "publishedDate": "2026-09-18 09:30:00", "title": "Apple Wins Contract", "site": "Reuters", "url": "m3"}]
    m = build("", {"feed": []}, today, since_utc=fsince, fmp_doc=fmpdoc)
    check(m["HUT"]["cat"] and m["HUT"]["kinds"] == ["deal"] and m["HUT"]["items"][0]["tier"] == "fmp", "FMP headline flags in AV's slot")
    check(m["HUT"]["tone"] == "bull" and m["HUT"]["items"][0]["source"] == "Reuters", "FMP tone and source kept")
    check("EIX" not in m, "FMP article before the last close is not fresh")
    check("AAPL" not in m, "FMP name outside the universe ignored")
    check(build("", {"feed": []}, today, since_utc=fsince, fmp_doc={"items": fmpdoc})["HUT"]["cat"], "FMP wrapper dict accepted")
    # tier order and weight
    mx = build("", news, today, since_utc=fsince, fvz_doc={"news": {
        "EIX": [{"t": "2026-09-18 09:00:00", "title": "Edison Shares Jump On Upgrade", "src": "Yahoo", "url": "x1", "tone": "bull"}]}})
    check(mx["EIX"]["items"][0]["tier"] == "av", "a scored AV article outranks a Finviz headline")
    check(mx["EIX"]["tiers"] == {"av": 1, "finviz": 1}, "both tiers listed for the name")
    wmix = build("", {"feed": []}, today, wire_doc={"entries": [
        {"ts": "2026-09-18T11:00:00.000Z", "ticker": "CAVA", "headline": "CAVA Group Board Approves $100M Share Repurchase Program"}]},
        since_utc=fsince, fvz_doc={"news": {"CAVA": [{"t": "2026-09-18 09:00:00", "title": "CAVA Cuts Guidance", "src": "Yahoo", "url": "x2", "tone": "bear"}]}})
    check(wmix["CAVA"]["items"][0]["tier"] == "wire", "the Wire still leads the list")
    check(wmix["CAVA"]["tone"] == "bull", "Wire at x2 outweighs a bearish Finviz headline at x0.5")
    import tempfile
    d = tempfile.mkdtemp()
    json.dump({"log_date": "2026-10-13", "names": {f"T{i}": {} for i in range(50)}}, open(os.path.join(d, "roster.json"), "w"))
    u, note = load_universe(d)
    check(len(u) == 50 and note.startswith("roster.json"), "universe from roster.json")
    os.remove(os.path.join(d, "roster.json"))
    u, note = load_universe(d)
    check(u == list(CORE50) and "STALE" in note, "fallback is labelled stale")
    print(f"selftest: {ok} assertions passed")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--apikey", default=os.environ.get("ALPHAVANTAGE_API_KEY"))
    ap.add_argument("--earnings"); ap.add_argument("--news"); ap.add_argument("--wire")
    ap.add_argument("--finviz", help="finviz.json (default: beside this script); earnings AND tier-3 news come from it")
    ap.add_argument("--fmp", help="FMP stock-news JSON; used only when Alpha Vantage returns no feed")
    ap.add_argument("--hours", type=int, default=18, help="news look-back (default 18h: prior close to now)")
    ap.add_argument("--selftest", action="store_true")
    a = ap.parse_args()
    if a.selftest:
        return selftest()
    now = datetime.now(timezone.utc)
    today = (now.replace(tzinfo=None) + et_offset(now)).date()          # ET date — what the dashboard compares
    close = last_session_close(now)
    since = close.strftime("%Y%m%dT%H%M")                                # AV news from the last close too
    wire = None
    if a.wire:
        raw = open(a.wire, encoding="utf-8").read()
        m = re.search(r'<script type="application/json" id="bernie-feed">(.*?)</script>', raw, re.S)
        wire = json.loads(m.group(1) if m else raw)   # accepts the page HTML or the bare JSON
    fvz, fvz_note = None, "finviz.json not used"
    fpath = a.finviz or os.path.join(HERE, "finviz.json")
    if os.path.exists(fpath):
        try:
            fvz = json.load(open(fpath, encoding="utf-8"))
            age = (now - datetime.fromisoformat(fvz["asof"])).total_seconds() / 86400
            if age > FINVIZ_MAX_AGE_DAYS:
                fvz_note, fvz = f"finviz.json is {age:.1f} days old -- ignored, AV earnings used", None
            else:
                fvz_note = f"Finviz earnings (finviz.json {fvz['asof']})"
        except Exception as e:
            fvz_note, fvz = f"finviz.json unreadable ({type(e).__name__}) -- AV earnings used", None
    if a.news:
        earn = ""
        if a.earnings:
            earn = open(a.earnings, encoding="utf-8").read()
            if earn.lstrip().startswith("{"):
                earn = json.loads(earn).get("result", "")
        news = json.load(open(a.news, encoding="utf-8"))
    elif a.apikey:
        earn = av_get({"function": "EARNINGS_CALENDAR", "horizon": "3month"}, a.apikey)
        time.sleep(2)  # free tier: 1 request/second, sequential only
        news = json.loads(av_get({"function": "NEWS_SENTIMENT", "time_from": since, "limit": 1000, "sort": "LATEST"}, a.apikey))
        if "feed" not in news:
            if a.fmp:
                print("Alpha Vantage refused the news call, falling back to --fmp: "
                      + json.dumps(news)[:200], file=sys.stderr)
                news = {"feed": []}
            else:
                sys.exit("Alpha Vantage refused the news call: " + json.dumps(news)[:300])
    else:
        sys.exit("Give --apikey (or ALPHAVANTAGE_API_KEY), or --news (and optionally --earnings) files.")
    uni, uni_note = load_universe()
    uniset = set(uni)
    fmp = None
    if a.fmp:
        try:
            fmp = json.load(open(a.fmp, encoding="utf-8"))
        except Exception as e:
            print(f"--fmp unreadable ({type(e).__name__}) -- ignored", file=sys.stderr)
    av_n = len((news or {}).get("feed") or [])
    fmp_n = len(fmp_news(fmp, uniset, close)) if fmp is not None else 0
    if fmp is not None and av_n:
        fmp_note = f"FMP held back (Alpha Vantage returned {av_n} article{'' if av_n == 1 else 's'})"
        fmp, fmp_n = None, 0
    elif fmp is not None:
        fmp_note = f"FMP stock news ({fmp_n} in window -- Alpha Vantage fallback)"
    else:
        fmp_note = None
    fvz_n = len(finviz_news(fvz, uniset, close))
    names = build(earn, news, today, universe=uni, wire_doc=wire, since_utc=close, fvz_doc=fvz, fmp_doc=fmp)
    srcs = ((["Bernie Wire"] if wire else []) + [fvz_note]
            + [f"Alpha Vantage NEWS_SENTIMENT ({av_n} articles)"
               + (" + EARNINGS_CALENDAR (fallback)" if earn else "")]
            + ([fmp_note] if fmp_note else [])
            + ([f"Finviz news ({fvz_n} in window)"] if fvz_n else [])
            + ["universe: " + uni_note])
    print("universe:", uni_note); print("earnings:", fvz_note)
    print(f"news: AV {av_n}, FMP {fmp_n}, Finviz {fvz_n} (window from {close.strftime('%Y-%m-%d %H:%MZ')})"
          + (f" -- {fmp_note}" if fmp_note and not fmp_n else ""))
    doc = write(names, today, close.isoformat(), srcs)
    flagged = {s: n for s, n in names.items() if n["cat"]}
    print(f"catalysts.json: {len(flagged)} flagged of {len(names)} with news/earnings")
    label = {"wire": "Wire", "av": "AV", "fmp": "FMP", "finviz": "FVZ"}
    for s, n in sorted(flagged.items()):
        tiers = " ".join(f"{label[t]} x{c}" for t, c in (n.get("tiers") or {}).items())
        print(f"  {s:5} {n['tone']:7} {','.join(n['kinds']):22} {tiers}")


if __name__ == "__main__":
    main()