"""Private, job-scoped Playwright reader. No model calls or arbitrary JS API.""" import argparse import asyncio from collections import OrderedDict from datetime import datetime, timezone import hashlib import hmac import ipaddress import json import os from pathlib import Path import re import socket import time from urllib.parse import urlsplit, urlunsplit def normalize_url(value): p = urlsplit(value) if p.scheme not in {"http", "https"} or not p.hostname or p.username or p.password: raise ValueError("public_http_url_required") if p.port not in {None, 80, 443}: raise ValueError("nonstandard_port_not_allowed") return urlunsplit((p.scheme.lower(), p.netloc.lower(), p.path or "/", p.query, "")) def public_host(host): if host.lower() in {"localhost", "metadata.google.internal"} or host.endswith(".local"): return False try: try: records = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM) except socket.gaierror as exc: if exc.errno != socket.EAI_AGAIN: raise # Retry transient DNS before dispatching any request to the site. time.sleep(0.25) records = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM) addresses = [x[4][0] for x in records] return bool(addresses) and all(ipaddress.ip_address(a).is_global for a in addresses) except (ValueError, OSError): return False def page_chunk(page, text_offset=0, link_offset=0): """Fit the existing controller's 16 KiB result budget without silent loss.""" if not 0 <= text_offset <= len(page["text"]) or not 0 <= link_offset <= len(page["links"]): raise ValueError("offset_out_of_range") text = page["text"][text_offset:text_offset + 5000] links = page["links"][link_offset:link_offset + 10] def payload(): t_end, l_end = text_offset + len(text), link_offset + len(links) return {**{k:v for k,v in page.items() if k not in {"text", "links"}}, "text": text, "links": links, "text_offset": text_offset, "link_offset": link_offset, "total_text_chars": len(page["text"]), "total_links": len(page["links"]), "next_text_offset": t_end if t_end < len(page["text"]) else None, "next_link_offset": l_end if l_end < len(page["links"]) else None} result = payload() while len(json.dumps(result, ensure_ascii=False).encode()) > 14000: if len(text) > 1000: text = text[:len(text)//2] elif links: links = links[:-1] else: raise ValueError("page_metadata_exceeds_budget") result = payload() return result def proxy_options(value): if not value: return None from urllib.parse import unquote p = urlsplit(value) if p.scheme not in {"http", "https", "socks5"} or not p.hostname: raise ValueError("invalid_browser_proxy") server = f"{p.scheme}://{p.hostname}" + (f":{p.port}" if p.port else "") result = {"server":server, "bypass":"localhost,127.0.0.1,::1"} if p.username: result["username"] = unquote(p.username) if p.password: result["password"] = unquote(p.password) return result def is_access_challenge(title, text): probe = (title + " " + text[:500]).lower() if any(marker in probe for marker in ("verify you are human", "checking your browser", "just a moment...")): return True heading = title.partition("|")[0].strip().casefold() # Some HTTP 200 interstitials identify themselves in both the title and body. return (heading == "verifying your browser" and any(line.strip().casefold() == heading for line in text[:500].splitlines()) and "complete the check below to continue" in text[:500].casefold()) def inline_anchor_targets(text, anchors): """Annotate unambiguous visible labels without changing the independent link list.""" remaining = min(8192, 250000 - len(text)) if remaining <= 0: return text inserts = [] for label, url in anchors: if len(inserts) >= 64: break start = text.find(label) if not label or start < 0 or text.find(label, start + 1) >= 0: continue end = start + len(label) if any(start < prior_end and prior_start < end for prior_start, prior_end, _ in inserts): continue suffix = " (" + url + ")" if len(suffix) <= remaining: inserts.append((start, end, suffix)) remaining -= len(suffix) for _, end, suffix in sorted(inserts, reverse=True): text = text[:end] + suffix + text[end:] return text _LINKS_JS = "els => els.slice(0, 3000).map(e => ({url:e.href,text:(e.innerText||e.getAttribute('aria-label')||'').trim().slice(0,160)}))" _INLINE_LINKS_JS = """els => els.slice(0, 3000).map(e => { const label = (e.innerText || '').trim(); return {url:e.href, text:(label || e.getAttribute('aria-label') || '').trim().slice(0,160), inline:label.length > 0 && label.length <= 160 && e.checkVisibility({visibilityProperty:true, opacityProperty:true}) && !e.closest('nav, header, footer, aside, [role=navigation], [role=banner], [role=contentinfo]') && !(e.getAttribute('href') || '').trim().startsWith('#')}; })""" def build_app(state_dir, allowed_hosts=(), *, inline_links=False): from contextlib import asynccontextmanager from fastapi import FastAPI, Request, HTTPException from playwright.async_api import async_playwright token = os.environ.get("SGAR_BROWSER_TOKEN", "") if len(token) < 24: raise ValueError("browser_token_required") state = Path(state_dir).resolve() state.mkdir(parents=True, exist_ok=False, mode=0o700) pages = OrderedDict() lock = asyncio.Lock() runtime = {} allowed = {host.lower() for host in allowed_hosts} @asynccontextmanager async def lifespan(app): async with async_playwright() as pw: proxy = proxy_options(os.environ.get("SGAR_TOOL_HTTPS_PROXY")) browser = await pw.chromium.launch(headless=True, proxy=proxy) runtime["browser"] = browser runtime["context"] = await browser.new_context(accept_downloads=False, service_workers="block") try: yield finally: await runtime["context"].close() await browser.close() app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) def auth(request): if not hmac.compare_digest(request.headers.get("authorization", ""), "Bearer " + token): raise HTTPException(status_code=401, detail="unauthorized") @app.get("/health") async def health(request: Request): auth(request) return {"ready": bool(runtime.get("browser")), "cached_pages": len(pages)} @app.post("/reset") async def reset(request: Request): auth(request) async with lock: await runtime["context"].close() runtime["context"] = await runtime["browser"].new_context(accept_downloads=False, service_workers="block") pages.clear() return {"status":"success"} @app.post("/read") async def read(request: Request): auth(request) result = await read_page(request) if result.get("status") == "error": fields = ("error", "http_status", "error_class") print(json.dumps({"event": "page_read_failed", "at": datetime.now(timezone.utc).isoformat(), **result.pop("_diagnostic", {}), **{k: result[k] for k in fields if k in result}}), flush=True) return result async def read_page(request: Request): try: if int(request.headers.get("content-length", "0")) > 8192: raise ValueError("request_too_large") body = await request.json() raw_url = body.get("url", "") if not isinstance(raw_url, str) or len(raw_url) > 4096: raise ValueError("invalid_url") url = normalize_url(raw_url) host = urlsplit(url).hostname if allowed and host not in allowed: raise ValueError("navigation_outside_allowed_hosts") if not await asyncio.to_thread(public_host, host): raise ValueError("public_host_required") offsets = [body.get(k, 0) for k in ["text_offset", "link_offset"]] if any(type(x) is not int or x < 0 for x in offsets): raise ValueError("invalid_offset") except (ValueError, TypeError, AttributeError): return {"status":"error", "error":"invalid_or_disallowed_request"} async with lock: if url not in pages: if any(offsets): return {"status":"error", "error":"snapshot_expired_restart_at_offset_zero"} tab = await runtime["context"].new_page() resolved = {} navigation_blocked = False async def route_guard(route): nonlocal navigation_blocked try: u = normalize_url(route.request.url) h = urlsplit(u).hostname if h not in resolved: resolved[h] = await asyncio.to_thread(public_host, h) permitted = resolved[h] if route.request.is_navigation_request() and allowed: permitted = permitted and h in allowed if not permitted: if route.request.is_navigation_request(): navigation_blocked = True await route.abort() elif route.request.resource_type in {"media", "font"}: await route.abort() else: await route.continue_() except (ValueError, OSError): if route.request.is_navigation_request(): navigation_blocked = True await route.abort() await tab.route("**/*", route_guard) phase = "navigation" try: response = await tab.goto(url, wait_until="domcontentloaded", timeout=30000) await tab.wait_for_timeout(800) status = response.status if response else None if status is None or status >= 400: return {"status":"error", "error":"page_http_error", "http_status":status} # DOM innerText keeps visible tables and rendered text. It # does not require a second, incompatible extraction stack. phase = "body" text = await tab.locator("body").inner_text(timeout=5000) phase = "links" links = await tab.locator("a[href]").evaluate_all(_INLINE_LINKS_JS if inline_links else _LINKS_JS) valid, seen = [], set() anchors = [] for item in links: try: link = normalize_url(item["url"]) except ValueError: continue if len(link) > 2048: continue if allowed and urlsplit(link).hostname not in allowed: continue if inline_links and item["inline"] and len(anchors) < 256: anchors.append((item["text"], link)) if link in seen: continue seen.add(link) valid.append({"url":link,"text":item["text"]}) if not text.strip(): return {"status":"error", "error":"empty_readable_content"} phase = "metadata" final_url = normalize_url(tab.url) title = (await tab.title())[:250] if is_access_challenge(title, text): return {"status":"error", "error":"site_access_challenge"} if inline_links: text = inline_anchor_targets(text, anchors) page = {"status":"success", "tool":"browser_read_page", "requested_url":url, "final_url":final_url, "title":title, "http_status":status, "fetched_at":datetime.now(timezone.utc).isoformat(), "page_id":hashlib.sha256((final_url + text).encode()).hexdigest()[:20], "content_truncated":len(text)>250000, "links_truncated":len(links)>=3000, "text":text[:250000], "links":valid, "rendered":True} pages[url] = page while len(pages)>40: pages.popitem(last=False) snapshot = state/(page["page_id"]+'.json') snapshot.write_text(json.dumps(page,ensure_ascii=False),encoding="utf-8") snapshot.chmod(0o600) except Exception as exc: # Keep URLs, page content, and raw exception messages out of the log. code = re.search(r"\bnet::(ERR_[A-Z0-9_]+)\b", str(exc)) return {"status":"error", "error":"browser_read_failed", "error_class":type(exc).__name__, "_diagnostic":{"phase":phase, "net_error":code.group(1) if code else None, "navigation_blocked":navigation_blocked}} finally: await tab.close() try: return page_chunk(pages[url], *offsets) except ValueError: return {"status":"error", "error":"invalid_page_offset_or_size"} return app def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--port", type=int, required=True) parser.add_argument("--state-dir", required=True) parser.add_argument("--allowed-host", action="append", default=[]) parser.add_argument("--inline-links", action="store_true") args = parser.parse_args() if not os.environ.get("SLURM_JOB_ID"): raise RuntimeError("allocated_job_required") import uvicorn uvicorn.run(build_app(args.state_dir,args.allowed_host,inline_links=args.inline_links),host="127.0.0.1",port=args.port, access_log=False,log_level="warning") if __name__ == "__main__": main()