"""Push BidBrain's finished daily list into Dealer OS. Dealer OS (Steven's and Mark's own multi tenant dealership platform, CarDealer-OS/Dealer_OS, live at cardealeros.co.uk) shows BidBrain's live auctions page. The agreed shape of the integration (Steven 2026-09-01, from the "BidBrain today, Dealer OS tomorrow" briefing, widened 2026-09-02 once Dealer OS was chosen as the ONLY screen people use) is the kitchen and the restaurant: BidBrain stays on the Mac and keeps every scraper and the whole buying brain, and pushes ONLY finished, decided results across, priced and gated cars with everything the cockpit itself draws for them, never raw listings and never the gate or pricing logic itself, so there is exactly one set of buying rules that can never drift apart. Standard library only, like the rest of this project: a plain HTTPS POST of one JSON document, gzipped on the wire, authenticated with a shared secret in a bearer header. The secret lives in dealer_config.py on this Mac (DEALER_OS_API_KEY) and, as one entry of BIDBRAIN_API_KEYS, in Dealer OS's own environment, never in either repo's shared code. The receiving end (the "letterbox", POST {DEALER_OS_BASE_URL}/api/bidbrain/ cars) is Dealer OS's, built against the contract in DEALER_OS_PUSH.md at the repo root. A push runs AFTER the run has written its own pages, records its own separate run_status and site_health, and never raises into run(). The document is a superset of the schema 1 contract: every field added since (2026-09-02, the cockpit rebuild inside Dealer OS) is additive, so schema_version stays 1 and an older Dealer OS simply ignores what it does not know. The one naming trap, kept on purpose so nothing already built breaks: suggested_bid_comps is the COUNT of comparable sales; the list itself lives under drawer.compmatch.comps. """ import datetime import gzip import json import urllib.error import urllib.request # The one route BidBrain posts to, appended to DEALER_OS_BASE_URL. API_PATH = "/api/bidbrain/cars" # Bumped whenever a field is renamed or removed (an added field never # needs one, Dealer OS should ignore what it does not know). Sent in the # document so the receiving side can refuse a shape it has not seen. SCHEMA_VERSION = 1 # The document travels gzipped, with Content-Encoding: gzip. Plain JSON # text compresses about ten to one, so the genuinely large day found live # on 2026-09-21 (2,078 held back cars plus a normal day's purchases and # lost bids, 5.17 MB of JSON) goes over the wire as roughly 555 KB. # # Gzip was tried that same morning and reverted, because Dealer OS's own # route read the bytes exactly as they arrived and answered "Body is not # valid JSON". Dealer OS fixed that later the same day (its issue #740): # all four of its machine doors now decompress the body and measure the # size AFTER decoding. That is what makes this switch safe. COMPRESS_LEVEL = 6 # The two ceilings a push has to fit under. Both are real and they # measure different things, so both are checked: # # * MAX_BODY_BYTES is the uncompressed document, which is the number # Dealer OS itself enforces. /api/bidbrain/cars stops decompressing at # 20 MB and answers 413, so 19 MB leaves a margin under it. # * MAX_WIRE_BYTES is what actually travels. Vercel refuses any request # body over 4.5 MB whatever is inside it, so 4 MB leaves a margin # there. # # The old cap was 3.85 MB of uncompressed JSON, which a large day blew # straight through. Compressed, that same day is nowhere near either # number, so the photo trim and the row caps below are now a genuine last # resort rather than something a busy Tuesday runs into. MAX_BODY_BYTES = 19_000_000 MAX_WIRE_BYTES = 4_000_000 TRIM_PHOTOS_TO = 12 # How many of the least essential rows survive the final, most aggressive # trim: held back cars are "read now and then, not bid on" (held_back_ # record's own docstring), already sorted cheapest first, so the cap # keeps the ones nearest the buying threshold. Lost bids keeps the most # recent. Purchases, under_offer and the settings blocks are never # capped, Dealer OS draws its own Purchases and settings pages from them # whole. These were a stopgap on 2026-09-21 and a real day hit them; the # largest day seen so far is now about a quarter of the smaller of the two # ceilings below, so the whole held back list goes across again. CAP_HELD_BACK_TO = 200 CAP_LOST_BIDS_TO = 200 # Size of the last document actually sent, for the run summary line: # the JSON itself, and what went over the wire once compressed. LAST_BODY_BYTES = 0 LAST_WIRE_BYTES = 0 class DealerOsPushError(RuntimeError): """A push that did not land, with a message written for a person.""" def _reserve_label(platform_key): """Carwow's reserve is CAP Clean standing in (2026-08-19), Dealer Auction's is Retail minus Margin (2026-08-20), DealerWay's is the current bid (2026-08-24): the cockpit labels each honestly, so does Dealer OS.""" return {"carwow": "Trade value", "dealerauction": "AT Trade", "dealerway": "Bid price"}.get(platform_key, "Reserve") def car_record(a, make_canon=None, sale_date=None): """One car's decided result as a plain dict. Only fields BidBrain has actually read or computed: an unread figure is None, never a guess, the same golden rule the cockpit's own cards follow. Private, never persisted, card annotations (the _ prefixed attributes daily_run's _annotate sets on the Car object) are read with getattr so a car that never went through _annotate still packs, just without them. Everything the cockpit card's own data attributes and drawer carry is here too, derived by the SAME render.py helpers the cockpit uses, so Dealer OS never re-derives a label, a family or a headroom figure.""" from bidbrain import render car = a.car p = a.pricing make_canon = make_canon or {} plat = render._platform_class(car.source) rows = render._insight_rows(a) headroom = ((p.max_bid - car.reserve) if (p is not None and p.max_bid is not None and car.reserve is not None) else None) cazana_headroom = None cazana_price = None if p is not None: cazana_headroom = ((p.calc_cazana - car.reserve) if (p.calc_cazana is not None and car.reserve is not None) else headroom) cazana_price = p.calc_cazana if p.calc_cazana is not None else p.governing_value gallery = list(getattr(car, "photo_urls", None) or []) if car.photo_url and car.photo_url not in gallery: gallery.insert(0, car.photo_url) seen_before = any(f.startswith("Seen before") or f.startswith("In both") for f in (a.flags or [])) reg = "".join((car.reg or "").upper().split()) or None return { "reg": reg, "key": reg or car.listing_url or None, "make": car.make or None, "make_canon": make_canon.get(car.make, car.make) or None, "model": car.model or None, "model_family": render._model_family(car.make, car.model) or None, "derivative": car.derivative or None, "name": render._title(car) or None, "search": " ".join(x for x in [reg or "", car.make or "", car.model or "", car.derivative or ""] if x).lower(), "year": car.year, "mileage": car.mileage, "owners": car.owners, "grade": car.grade, "fuel": car.fuel or None, "transmission": car.transmission or None, "gearbox_label": render._gearbox_label(car), "engine": car.engine or None, "engine_label": render._engine_label(car), "engine_litres": render._engine_litres(car) or None, "body_type": car.body_type or None, "service_history": car.service_history or None, "vat_qualifying": bool(car.vat_qualifying), "reserve": car.reserve, "reserve_label": _reserve_label(plat), "cap_clean": car.cap_clean, "distance_miles": car.distance_miles, "location": car.location or None, "platform": car.source or None, "platform_key": plat, "listing_url": car.listing_url or None, "photo_url": car.photo_url or None, "photo_urls": gallery, "auction_ends_at": car.auction_ends_at or None, "keeper_start": getattr(car, "keeper_start", None) or None, "selling_vrm": getattr(car, "selling_vrm", "") or None, "cazana_url": getattr(car, "cazana_url", "") or None, "glass_checked": bool(getattr(car, "glass_checked", False)), "status": a.status, "reasons": list(a.reasons or []), "flags": list(a.flags or []), "insights": [{"kind": k, "topic": t, "text": x} for k, t, x in rows], "pricing": None if p is None else { "glass_retail": p.glass_retail, "cazana_retail": p.cazana_retail, "calc_glass": p.calc_glass, "calc_cazana": p.calc_cazana, "governing_value": p.governing_value, "governing_source": p.governing_source, "max_bid": p.max_bid, "spread": (p.governing_value - p.max_bid if p.governing_value is not None and p.max_bid is not None else None), }, "headroom": headroom, "cazana_headroom": cazana_headroom, "cazana_price": cazana_price, "suggested_bid": getattr(car, "_suggested_bid", None), "suggested_bid_comps": getattr(car, "_suggested_bid_n", None), "stock_level": getattr(car, "_stock_level", None), "stock_count": getattr(car, "_stock_count", None), "taste_match": bool(getattr(car, "_taste_match", False)), "taste_score": getattr(car, "_taste_score", 0) or 0, "pick_rank": getattr(car, "_pick_rank", None), "pick_score": getattr(car, "_pick_score", 0) or 0, "is_new": bool(getattr(car, "_is_new", False)), "seen_before": seen_before, "starred": bool(getattr(car, "_starred", False)), "drawer": render._drawer_data(a, rows), # Every earlier sighting in the last 30 days (2026-09-10), so the # card can say how many days ago each one was and what the reserve # was that day. Empty when the sale date is not known here. "sightings": _sightings((reg or "", getattr(car, "selling_vrm", None) or ""), sale_date), # The shortlist deep read's answer (2026-09-17), hung on in # build_payload from deep_reads; null until the Mac has read the # car's own page for Dealer OS's ask. "deep": None, } def _sightings(reg, sale_date): """reg is the listing plate, or the listing plate and the original plate underneath, see db.sighting_plates.""" if not reg or not any(reg if isinstance(reg, tuple) else (reg,)) or not sale_date: return [] try: from bidbrain import db return db.sighting_history(reg, sale_date) except Exception: return [] def _deep_reads(sale_date): """Every answered deep read for the sale, keyed (platform_key, key), never fatal to the push.""" if not sale_date: return {} try: from bidbrain import db return db.deep_reads_for(sale_date) except Exception: return {} def held_back_record(a): """The slim record of a car the rules held back (2026-09-10): enough to recognise it, every reason it was held on, and the link. No pricing, no drawer, one photo: these are read now and then, not bid on.""" from bidbrain import render car = a.car reg = "".join((car.reg or "").upper().split()) or None return { "key": reg or car.listing_url or None, "reg": reg, "make": car.make or None, "model": car.model or None, "name": render._title(car) or None, "year": car.year, "mileage": car.mileage, "reserve": car.reserve, "reserve_label": _reserve_label(render._platform_class(car.source)), "platform": car.source or None, "platform_key": render._platform_class(car.source), "listing_url": car.listing_url or None, "photo_url": car.photo_url or None, "reasons": list(a.reasons or []), } def build_payload(shortlist, held, sale_date, dealer_name="", generated_at=None, page=None, state=None, partial=False, records=None, held_back=None): """The whole document one push sends: the sale date the list is for, when it was built, which dealership it belongs to, every shortlisted and held car, and (page) the page level facts the cockpit is drawn from plus (state) the Mac's own stars, hides and saved views so the two screens stay in step. Rejected cars were left out until 2026-09-10; now they go as the slim held_back list (held_back_record) so Steven can check nothing is held back unnecessarily and release one with a reason. A car with no reg is still sent, Dealer OS keys on listing_url as a fallback the same way this project's own detail cache does. partial=True marks a mid run push (one platform finished, others still to come).""" from bidbrain import render when = generated_at or datetime.datetime.now().replace(microsecond=0).isoformat() every = list(shortlist) + list(held) make_canon = render._canon_make_map(a.car.make for a in every) cars = [car_record(a, make_canon, sale_date) for a in every] deep = _deep_reads(sale_date) if deep: for c in cars: c["deep"] = deep.get((c.get("platform_key"), c.get("key"))) doc = { "schema_version": SCHEMA_VERSION, "dealer_name": dealer_name or None, "sale_date": sale_date, "generated_at": when, "partial": bool(partial), "shortlisted": len(shortlist), "held": len(held), "cars": cars, } if held_back is not None: # Additive, still schema 1: the cars the rules held back, slim. doc["held_back"] = [held_back_record(a) for a in held_back] for k, v in (page or {}).items(): doc[k] = v if state is not None: doc["state"] = state for k, v in (records or {}).items(): doc[k] = v return doc def page_record(settings, notices=None, summary=None, model_families=None, platform_enabled=None, hidden_connections=None, integrations=None, run=None, bidbrain_version=None): """The page level block: the settings that shape the cockpit page, the health notices, the summary counts, the model families for the Model filter, which filters apply to which platforms, and the platforms themselves. settings is the raw db.get_settings() dict merged over the defaults by the caller (daily_run._get_settings).""" from bidbrain import render settings = settings or {} hidden = list(hidden_connections or []) enabled = platform_enabled or {} platforms = {} for key in render._ALL_PLATFORMS: platforms[key] = {"label": render._platform_label(key), "enabled": bool(enabled.get(key, True)), "hidden": key in hidden} kinds = [] for n in (notices or []): kinds.append({"text": n, "kind": _notice_kind(n)}) updated = settings.get("_updated_at") or {} return { "summary": summary or {}, "notices": kinds, "model_families": [list(x) for x in (model_families or [])], "filter_scope": {k: list(v) for k, v in render._FILTER_SCOPE.items()}, "lenient_filters": list(render._LENIENT_FILTERS), "platforms": platforms, "settings": { "details_style": settings.get("details_style") or "drawer", "default_price_ceiling": settings.get("default_price_ceiling"), "top_picks": render.TOP_PICKS_LABEL, "theme": { "bg": settings.get("theme_bg"), "price": settings.get("theme_price"), "pill": settings.get("theme_pill"), "icon": settings.get("theme_icon"), "sidebar": settings.get("theme_sidebar"), }, "updated_at": {k: v for k, v in updated.items() if k in ("details_style", "default_price_ceiling")}, }, "integrations": integrations or {}, "run_checks": [{"key": k, "label": lab, "site": site, "gate": gate, "extra": extra} for k, lab, site, gate, extra in render._RUN_MODAL_CHECKS], "run": run, "bidbrain_version": bidbrain_version, } def _notice_kind(text): """Same buckets the cockpit's own bell and Run button use (render._categorize_notices): "ok" for the happy path confirmation, else glass, dealerkit, platform or other.""" from bidbrain import render if text.startswith("All "): return "ok" glass, dealerkit, platform, _alerts = render._categorize_notices([text]) if glass: return "glass" if dealerkit: return "dealerkit" if platform: return "platform" return "other" def _trim_photos(payload, n=TRIM_PHOTOS_TO): for c in payload.get("cars", []): if isinstance(c.get("photo_urls"), list): c["photo_urls"] = c["photo_urls"][:n] d = c.get("drawer") if isinstance(d, dict) and isinstance(d.get("photos"), list): d["photos"] = d["photos"][:n] for r in payload.get("purchases", []): if isinstance(r.get("photo_urls"), list): r["photo_urls"] = r["photo_urls"][:n] return payload def _cap_lists(payload): """The last, most aggressive resort on a genuinely large day (2026-09- 21), after the photo trim above still is not enough: cap held_back and lost_bids to their own CAP_*_TO length, dropped_counts says exactly how many of each were left off so nothing reads as the whole list when it is not. held_back is already sorted cheapest first, so the cap keeps the ones nearest the buying threshold; lost_bids keeps the most recent (its own newest first order). purchases, under_offer and every settings block are never touched here, Dealer OS draws its own Purchases and settings pages from them whole.""" dropped = {} held_back = payload.get("held_back") if isinstance(held_back, list) and len(held_back) > CAP_HELD_BACK_TO: dropped["held_back"] = len(held_back) - CAP_HELD_BACK_TO payload["held_back"] = held_back[:CAP_HELD_BACK_TO] lost_bids = payload.get("lost_bids") if isinstance(lost_bids, list) and len(lost_bids) > CAP_LOST_BIDS_TO: dropped["lost_bids"] = len(lost_bids) - CAP_LOST_BIDS_TO payload["lost_bids"] = lost_bids[:CAP_LOST_BIDS_TO] if dropped: payload["capped"] = dropped return payload # Any remaining path on the Mac Studio's own disk (a document not uploaded # yet, see dealer_os_files) is dropped from every list, never sent. LOCAL_PATH_KEYS = () def _jsonable(d): """A plain dict Dealer OS can parse: anything json cannot carry (a date, a Decimal) becomes its string.""" return json.loads(json.dumps(d, default=str)) def mac_settings_block(merged, fee_bands=None, banned_engine_names=None, run_schedule=None, updated_at=None, editable_sections=(), buying_rule_keys=()): """Everything the Mac Studio's own Settings page shows, so Dealer OS can draw a settings page from the push alone (2026-09-03, step three). buying_rules carries the current value of every rule Dealer OS may send back; the read only blocks (fee_bands, banned_engine_names, run_schedule) are shown, never accepted back. updated_at is the Mac's own stamp per key, the newest wins comparison.""" merged = merged or {} return {"mac_settings": _jsonable({ "buying_rules": {k: merged.get(k) for k in buying_rule_keys}, "integrations": merged.get("integrations") or {}, "hidden_connections": merged.get("hidden_connections") or [], "fee_bands": fee_bands or {}, "banned_engine_names": list(banned_engine_names or []), "run_schedule": list(run_schedule or []), "editable_sections": list(editable_sections), "updated_at": {k: v for k, v in (updated_at or {}).items() if k in set(buying_rule_keys) | {"purchase_dropdowns", "purchase_columns", "purchase_flow", "integrations", "hidden_connections"}}, })} def records_block(purchases=None, under_offer=None, lost_bids=None, purchase_settings=None, purchase_events=None, lost_bids_views=None, dealerkit_stock=None): """The three record lists the Mac's own Purchases, Under Offer and Didn't win pages draw (2026-09-02), sent so Dealer OS can draw the same pages. Each row is exactly what the Mac's own page gets from its API (serve._purchases_with_bias, db.list_under_offer, serve._lost_bids_with_bias), minus local file paths. Read only on the Dealer OS side for now: an edit made there does not come back yet, so the Mac's own Purchases page stays the place to edit a record. purchase_settings carries the per dealership dropdown answers, column layout, journey flags and saved views those pages are built from.""" def clean(rows): out = [] for r in rows or []: d = {k: v for k, v in dict(r).items() if k not in LOCAL_PATH_KEYS} for k, v in list(d.items()): if isinstance(v, list) and any(isinstance(x, str) and x.startswith("/") for x in v): d[k] = [x for x in v if not (isinstance(x, str) and x.startswith("/"))] elif isinstance(v, str) and v.startswith("/Users/"): d[k] = None out.append(_jsonable(d)) return out return { "purchases": clean(purchases), "under_offer": clean(under_offer), "lost_bids": clean(lost_bids), "purchase_settings": _jsonable(purchase_settings or {}), # The timeline (every change, chip, comment and status move, each # row keyed by purchase_id) and the Didn't win page's own saved # views (2026-09-02). "purchase_events": clean(purchase_events), "lost_bids_views": _jsonable(list(lost_bids_views or [])), # DealerKit's own in stock cars with its purchase cost (2026-09-10). "dealerkit_stock": _jsonable(list(dealerkit_stock or [])), } def _encode(payload): """The document as the pair that matters: the JSON bytes Dealer OS measures once it has decompressed them, and the gzipped bytes that actually travel. mtime=0 so the same document always compresses to the same bytes, which keeps a push comparable run to run.""" body = json.dumps(payload).encode("utf-8") return body, gzip.compress(body, COMPRESS_LEVEL, mtime=0) def _too_big(body, wire): """Either ceiling blown: the uncompressed size Dealer OS enforces, or the compressed size Vercel will actually carry.""" return len(body) > MAX_BODY_BYTES or len(wire) > MAX_WIRE_BYTES def push(payload, base_url, api_key, timeout=30, opener=None): """POST the document. Returns whatever JSON the server answered with ({} on an empty 2xx). Raises DealerOsPushError, in plain words, on a missing setting, a refused secret, a missing letterbox, or a network failure. opener exists only so the tests can hand in a fake; real callers never pass it. The body is gzipped: Dealer OS's own route decompresses it and measures the size afterwards. A document over either ceiling has its photo galleries trimmed and is built once more; still too big it also has held_back and lost_bids capped (_cap_lists) and is built a third time before giving up.""" base_url = (base_url or "").strip().rstrip("/") api_key = (api_key or "").strip() if not base_url or not api_key: raise DealerOsPushError( "Dealer OS is not set up on this Mac: DEALER_OS_BASE_URL and " "DEALER_OS_API_KEY both need a value in dealer_config.py.") body, wire = _encode(payload) if _too_big(body, wire): body, wire = _encode(_trim_photos(payload)) if _too_big(body, wire): body, wire = _encode(_cap_lists(_trim_photos(payload))) if _too_big(body, wire): raise DealerOsPushError( f"The list is too big to send ({len(body) // 1000} KB, " f"{len(wire) // 1000} KB compressed, even with the photo " "galleries trimmed and the held back and lost bids lists " "capped). Dealer OS accepts about 19 MB once uncompressed.") req = urllib.request.Request( base_url + API_PATH, data=wire, method="POST", headers={"Content-Type": "application/json", "Content-Encoding": "gzip", "Authorization": f"Bearer {api_key}", "User-Agent": "BidBrain"}) do_open = opener or urllib.request.urlopen try: with do_open(req, timeout=timeout) as resp: raw = resp.read() except urllib.error.HTTPError as e: if e.code in (401, 403): raise DealerOsPushError( "Dealer OS refused the shared secret. The DEALER_OS_API_KEY in " "dealer_config.py must be this dealership's own token from Dealer " "OS's BIDBRAIN_API_KEYS.") from e if e.code == 404: raise DealerOsPushError( f"Dealer OS has no letterbox at {API_PATH} yet, so there is " "nowhere to send the list. That route is Dealer OS's side to " "build (see DEALER_OS_PUSH.md).") from e detail = "" try: detail = e.read().decode("utf-8", "replace")[:200].strip() except Exception: pass if e.code in (400, 415) and ("not valid JSON" in detail or "Content-Encoding" in detail): # What a Dealer OS older than its own gzip fix answers: it # hands the compressed bytes straight to its JSON parser. raise DealerOsPushError( "Dealer OS could not read the compressed list, so its side is " "older than the version that accepts a gzipped push. It needs " f"updating first. It answered {e.code}: {detail}") from e raise DealerOsPushError( f"Dealer OS answered {e.code} to the push." + (f" {detail}" if detail else "")) from e except urllib.error.URLError as e: raise DealerOsPushError( f"Could not reach Dealer OS at {base_url}: {e.reason}") from e global LAST_BODY_BYTES, LAST_WIRE_BYTES LAST_BODY_BYTES = len(body) LAST_WIRE_BYTES = len(wire) if not raw: return {} try: return json.loads(raw.decode("utf-8")) except ValueError: return {"raw": raw.decode("utf-8", "replace")[:200]}