"""Lasting links for the documents Dealer OS's Purchases page shows (2026-09-02). The Mac Studio keeps every purchase's service history and V5 scans as FILES on its own disk (data/purchase_docs//...), because the platform links they came from either expire within the hour (Motorway's signed document CDN) or never existed as links at all (Carwow's zip bundle). A file on the Mac's disk is nothing to Dealer OS, so before a push each one not yet uploaded is sent, once, to Dealer OS's own file store: PUT {DEALER_OS_BASE_URL}/api/bidbrain/documents?reg=&name= Authorization: Bearer , Content-Type: , body: the raw bytes and the reply {"url": "https://..."} is the lasting link, remembered here against the local path (dealer_os_files table) so the same file is never sent twice. The push then carries that link in place of the local path. Until Dealer OS has that route (a 404), nothing is uploaded and the push simply carries no link for that file, as today. Photos are not handled here: a car's photo_urls are already lasting CDN links. """ import json import mimetypes import os import sqlite3 import urllib.error import urllib.parse import urllib.request from bidbrain import db MAX_PER_PUSH = 150 # After this many failures in a row the batch stops: Dealer OS's own store # is down, not one bad file, and 900 more tries would only hold up the push. MAX_CONSECUTIVE_FAILURES = 3 # Dealer OS refuses anything over about 4.5 MB (413); an image above this # is downsized to a JPEG with a 2000 px longer edge first, plenty for a # V5 or a car photo. A PDF that big cannot be shrunk here and is skipped. MAX_FILE_BYTES = 4_000_000 DOWNSIZE_EDGE = 2000 LOCAL_LIST_KEYS = ("service_history_photos", "v5_photos", "document_paths", "v5_local_paths", "photo_urls") LOCAL_SINGLE_KEYS = ("photo_url",) IMAGE_EXT = (".jpg", ".jpeg", ".png", ".heic", ".webp", ".tif", ".tiff") class DocumentUploadError(RuntimeError): """An upload that did not land, in plain words. missing_route is True when Dealer OS has no documents route yet.""" missing_route = False def _ensure(conn): conn.execute("""CREATE TABLE IF NOT EXISTS dealer_os_files ( local_path TEXT PRIMARY KEY, url TEXT NOT NULL, uploaded_at TEXT NOT NULL)""") def known_urls(path: str = None) -> dict: """{local_path: url} for every file already uploaded.""" conn = db.connect(path or db.DB_PATH) try: _ensure(conn) return {r[0]: r[1] for r in conn.execute("SELECT local_path, url FROM dealer_os_files")} finally: conn.close() def remember(local_path, url, path: str = None) -> None: conn = db.connect(path or db.DB_PATH) try: _ensure(conn) conn.execute("INSERT OR REPLACE INTO dealer_os_files (local_path, url, uploaded_at) VALUES (?,?,?)", (local_path, url, db._utcnow())) conn.commit() finally: conn.close() def is_local(value) -> bool: return isinstance(value, str) and value.startswith("/") def _downsized(local_path): """A JPEG copy of an image with a 2000 px longer edge, via macOS's own sips, in a temp file the caller removes. None if sips cannot do it.""" import subprocess import tempfile out = tempfile.mktemp(suffix=".jpg") r = subprocess.run(["sips", "-s", "format", "jpeg", "-s", "formatOptions", "80", "-Z", str(DOWNSIZE_EDGE), local_path, "--out", out], capture_output=True, text=True) if r.returncode != 0 or not os.path.exists(out): return None return out def upload(local_path, reg, base_url, api_key, timeout=60, opener=None) -> str: """Send one file, return its lasting url. Raises DocumentUploadError. Sets upload.downsized to True when the file had to be shrunk first.""" upload.downsized = False send_path = local_path tmp = None size = os.path.getsize(local_path) ext = os.path.splitext(local_path)[1].lower() if size > MAX_FILE_BYTES: if ext in IMAGE_EXT: tmp = _downsized(local_path) if not tmp or os.path.getsize(tmp) > MAX_FILE_BYTES: if tmp: os.remove(tmp) raise DocumentUploadError(f"{os.path.basename(local_path)} is {size / 1e6:.1f} MB and could not be made small enough to send.") send_path = tmp upload.downsized = True try: with open(send_path, "rb") as f: body = f.read() finally: if tmp: os.remove(tmp) ctype = mimetypes.guess_type(send_path)[0] or "application/octet-stream" if ctype not in ("image/jpeg", "image/png", "application/pdf"): raise DocumentUploadError(f"{os.path.basename(local_path)} is a {ctype} file, which Dealer OS does not take.") name = os.path.basename(local_path) if upload.downsized: name = os.path.splitext(name)[0] + ".jpg" q = urllib.parse.urlencode({"reg": reg or "", "name": name}) req = urllib.request.Request( base_url.rstrip("/") + "/api/bidbrain/documents?" + q, data=body, method="PUT", headers={"Content-Type": ctype, "Authorization": "Bearer " + api_key}) do_open = opener or urllib.request.urlopen try: with do_open(req, timeout=timeout) as r: reply = json.loads(r.read().decode("utf-8") or "{}") except urllib.error.HTTPError as e: if e.code in (401, 403, 404): # A Dealer OS without the route answers 401 from its own login # check (the path is not in its public exceptions), not 404, so # both read as "not built yet", never as a bad token: the same # token has just been accepted by the push itself. err = DocumentUploadError("Dealer OS has no documents route open to this Mac yet.") err.missing_route = True raise err from e raise DocumentUploadError(f"Dealer OS answered {e.code} to a document upload.") from e except (urllib.error.URLError, OSError) as e: raise DocumentUploadError(f"Could not reach Dealer OS to send a document: {e}") from e url = reply.get("url") if isinstance(reply, dict) else None if not url: raise DocumentUploadError("Dealer OS took the file but sent no link back.") return url def sync_documents(purchases, base_url, api_key, path: str = None, opener=None, limit=MAX_PER_PUSH) -> dict: """Upload every local document in these purchase rows that has no lasting link yet, up to limit per call. Returns counts. Stops at the first missing route or refused token (nothing else would succeed).""" have = known_urls(path) sent = skipped = failed = downsized = 0 streak = 0 note = "" for r in purchases: todo = [] for key in LOCAL_LIST_KEYS: todo.extend(r.get(key) or []) for key in LOCAL_SINGLE_KEYS: if r.get(key): todo.append(r.get(key)) for p in todo: if not is_local(p) or p in have: continue if not os.path.exists(p): skipped += 1 continue if limit is not None and sent >= limit: return {"sent": sent, "skipped": skipped, "failed": failed, "downsized": downsized, "note": "more next push"} try: url = upload(p, r.get("reg") or "", base_url, api_key, opener=opener) except DocumentUploadError as e: if e.missing_route: return {"sent": sent, "skipped": skipped, "failed": failed, "downsized": downsized, "note": str(e)} failed += 1 streak += 1 note = str(e) if streak >= MAX_CONSECUTIVE_FAILURES: note = f"stopped after {streak} failures in a row, Dealer OS is not storing files right now: {e}" return {"sent": sent, "skipped": skipped, "failed": failed, "downsized": downsized, "note": note} continue remember(p, url, path) have[p] = url sent += 1 streak = 0 if upload.downsized: downsized += 1 return {"sent": sent, "skipped": skipped, "failed": failed, "downsized": downsized, "note": note} def with_links(purchases, path: str = None) -> list: """Copies of the rows with each local document path replaced by its lasting link, or dropped when none exists yet. Non local entries (a CDN link) are kept as they are.""" have = known_urls(path) out = [] for r in purchases: d = dict(r) for key in LOCAL_LIST_KEYS: vals = d.get(key) if not isinstance(vals, list): continue d[key] = [have.get(v) if is_local(v) else v for v in vals] d[key] = [v for v in d[key] if v] for key in LOCAL_SINGLE_KEYS: v = d.get(key) if is_local(v): d[key] = have.get(v) out.append(d) return out