diff --git a/backends/exllamav3/admission.py b/backends/exllamav3/admission.py new file mode 100644 index 00000000..e9a26951 --- /dev/null +++ b/backends/exllamav3/admission.py @@ -0,0 +1,608 @@ +"""Lane admission control for the tabbyAPI exllamav3 backend. + +LANE-ADMISSION-v1 + +A pure-stdlib request-class gate that sits *above* the exllamav3 enqueue +point in ``backends/exllamav3/model.py``. exllamav3 has no per-job priority +and its requeue path re-grants a long job the slot it just freed, so class +separation ("short queries jump the line, long agentic sessions never share +the slot with each other") can only be expressed before the job is built. +The gate holds requests instead of rejecting them: a held request always +eventually proceeds (``hold_timeout`` expiry is fail-open). + +No tabbyAPI or third-party imports: the module must import cleanly with a +bare ``python3`` so it is unit-testable on a workstation with no engine +deps. ``logger`` is injected by the caller (tabbyAPI passes its own) so this +module never configures logging itself. + +Config block (all keys optional; omitting it entirely, or ``{}``, disables +the feature — the gate is then provably a no-op): + + admission: + classes: + long: + prompt_tokens_gte: 32768 # optional predicate + max_new_tokens_gte: null # optional predicate + max_concurrent: 1 + short: + max_concurrent: 2 + default: short # class when no rule matches + hold_timeout: 1800 # seconds; 0 = wait forever +""" + +from __future__ import annotations + +import asyncio +import logging +import math +import time +from collections import deque + +DEFAULT_HOLD_TIMEOUT = 1800.0 + +# Recognised per-class predicates. A key present with value ``None`` is an +# *absent* predicate (the YAML example spells it out to document intent); +# a class with no present predicates matches nothing — use ``default`` for +# the catch-all. +_PREDICATES = ("prompt_tokens_gte", "max_new_tokens_gte") +_CLASS_KEYS = frozenset(_PREDICATES) | {"max_concurrent"} +_CONFIG_KEYS = frozenset({"classes", "default", "hold_timeout"}) + + +def _is_mapping(obj): + return isinstance(obj, dict) + + +def _require_int(value, key): + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"admission: {key} must be an integer, got {value!r}") + return value + + +def _require_nonneg_int(value, key): + _require_int(value, key) + if value < 0: + raise ValueError(f"admission: {key} must be >= 0, got {value!r}") + return value + + +class _ClassRule: + __slots__ = ("name", "prompt_tokens_gte", "max_new_tokens_gte", "max_concurrent") + + def __init__(self, name, body): + if not _is_mapping(body): + raise ValueError(f"admission: classes.{name} must be a mapping, got {body!r}") + for key in body: + if key not in _CLASS_KEYS: + raise ValueError(f"admission: classes.{name} has unknown key {key!r}") + self.name = name + self.max_concurrent = body.get("max_concurrent", 1) + if isinstance(self.max_concurrent, bool) or not isinstance(self.max_concurrent, int): + raise ValueError( + f"admission: classes.{name}.max_concurrent must be an integer, " + f"got {self.max_concurrent!r}" + ) + if self.max_concurrent < 1: + raise ValueError( + f"admission: classes.{name}.max_concurrent must be >= 1, " + f"got {self.max_concurrent!r}" + ) + self.prompt_tokens_gte = body.get("prompt_tokens_gte") + self.max_new_tokens_gte = body.get("max_new_tokens_gte") + if self.prompt_tokens_gte is not None: + _require_nonneg_int(self.prompt_tokens_gte, f"classes.{name}.prompt_tokens_gte") + if self.max_new_tokens_gte is not None: + _require_nonneg_int(self.max_new_tokens_gte, f"classes.{name}.max_new_tokens_gte") + + def matches(self, prompt_tokens, max_new_tokens): + """True iff every *present* predicate matches. Anded. Inclusive bounds. + + A class with no present predicates *matches nothing* (ruling 5) — it + is a capacity reservation, not a catch-all; use ``default`` for that. + ``max_new_tokens`` of ``None`` (client sent no limit) never matches a + class that declares ``max_new_tokens_gte``. + """ + matched_any = False + if self.prompt_tokens_gte is not None: + if prompt_tokens < self.prompt_tokens_gte: + return False + matched_any = True + if self.max_new_tokens_gte is not None: + if max_new_tokens is None or max_new_tokens < self.max_new_tokens_gte: + return False + matched_any = True + return matched_any + + +class _NoopCM: + """Async context manager that never awaits and never touches state.""" + + async def __aenter__(self): + return None + + async def __aexit__(self, *exc_info): + return False + + +_NOOP_CM = _NoopCM() + + +class _Holder: + """One in-flight ``async with gate.acquire(...)`` — the slot handle. + + Lifecycle: queued in the class FIFO as a future; granted by a releasing + holder or :meth:`LaneGate.abort_all`; releases (or hands on) its permit + on *every* exit path — normal, exception, cancellation, timeout — and on a + failed entry too (:meth:`__aenter__` raising, where Python never calls + :meth:`__aexit__`). Both routes go through :meth:`_release_owned`, which is + idempotent. + """ + + __slots__ = ( + "gate", + "cls", + "request_id", + "prompt_tokens", + "max_new_tokens", + "fut", + "counted", + "permit", + "admitted", + "t_enqueued", + "t_granted", + "abort_event", + ) + # ``permit`` means this holder OWNS a semaphore permit and must hand it + # on / return it at exit. Fail-open wakes (hold_timeout expiry, abort_all) + # set admitted WITHOUT a permit — they proceed but never release one, so + # the semaphore count can never exceed its limit. + + def __init__(self, gate, cls, request_id, prompt_tokens, max_new_tokens, abort_event=None): + self.gate = gate + self.cls = cls + self.request_id = request_id + self.prompt_tokens = prompt_tokens + self.max_new_tokens = max_new_tokens + self.fut = asyncio.Future() + self.counted = False # counted in gate._waiting + self.permit = False # consumed a semaphore permit (holds a slot) + self.admitted = False # __aexit__ must free the slot + self.t_enqueued = 0.0 + self.t_granted = 0.0 + self.abort_event = abort_event + + async def __aenter__(self): + try: + return await self._enter() + except BaseException: + # ``async with`` does NOT call __aexit__ when __aenter__ raises, so + # whatever this holder already owns — a permit spent by a grant, + # the ``_held`` counter bumped by _note_grant() — would be stranded + # for the life of the process. One occurrence permanently shrinks + # the lane, and on a ``max_concurrent: 1`` lane it disables it. + # Hand back exactly what is owned, then let the original exception + # propagate untouched (never swallowed). + self._release_owned() + raise + + async def _enter(self): + gate = self.gate + if not gate._enabled: + return self + abort_event = self.abort_event + if abort_event is not None and abort_event.is_set(): + # The client is already gone. Raise the same error the endpoint + # would see from DisconnectHandler.poll() mid-generation (SPEC §5) + # — before queueing, so this request never consumes a slot. + raise asyncio.CancelledError() + queue = gate._queues[self.cls] + # Pre-grant fast path: an idle permit is taken directly; the FIFO is + # only formed once the lane is saturated (FIFO fairness applies among + # actual waiters). + if not queue and not gate._sems[self.cls].locked(): + await gate._sems[self.cls].acquire() + self.permit = True + self._note_grant(0.0) + return self + queue.append(self.fut) + self.counted = True + gate._waiting[self.cls] = gate._waiting.get(self.cls, 0) + 1 + self.t_enqueued = gate._clock() + abort_fut = None + try: + timeout = gate._hold_timeout + if abort_event is None: + if timeout and timeout > 0: + try: + # NOTE: asyncio.wait_for cancels self.fut on expiry, so + # a grant racing the timer shows up as a *cancelled* + # future here; the permit is reconciled below, never + # stranded. + await asyncio.wait_for(self.fut, timeout) + except asyncio.TimeoutError: + if self.fut.done() and not self.fut.cancelled(): + # A wake landed before the timer's cancel of the + # future. result True -> a real grant (permit was + # consumed by the releaser on our behalf); + # result False -> an abort_all fail-open wake. + self.permit = bool(self.fut.result()) + self._note_grant(gate._clock() - self.t_enqueued) + return self + # No permit exists for us (a grant could not have landed: + # wait_for cancelled the future first). Give up the queue + # position and proceed fail-open with a warning — admitted + # but permit-free, so our exit releases nothing. + if self.fut in queue: + queue.remove(self.fut) + self._warn_fail_open(timeout) + self._note_grant(timeout) + return self + else: + # hold_timeout == 0: wait forever, no fail-open timer. + await self.fut + else: + # SPEC §3/§5 — race the FIFO wake and the fail-open timer + # against the disconnect signal, so a request that vanishes + # while *queued* reaches the endpoint through the same path as + # one that vanishes mid-generation. + abort_fut = asyncio.ensure_future(abort_event.wait()) + done, _ = await asyncio.wait( + (self.fut, abort_fut), + timeout=(timeout if timeout and timeout > 0 else None), + return_when=asyncio.FIRST_COMPLETED, + ) + if abort_event.is_set(): + # Abandon the hold, permit-free. Raising here (rather than + # returning) makes the handler below do the giving-up: a + # grant that landed in the same tick re-enters the FIFO + # instead of being spent on a request with no client. + raise asyncio.CancelledError() + if self.fut in done: + self.permit = bool(self.fut.result()) + self._note_grant(gate._clock() - self.t_enqueued) + return self + # Timer expired with neither a grant nor an abort: the same + # fail-open exit as the branch above (SPEC §4 — hold, never + # reject; a vanished client is the only early way out). + if self.fut in queue: + queue.remove(self.fut) + self._warn_fail_open(timeout) + self._note_grant(timeout) + return self + # Woken via the FIFO. result True -> a real grant: we consumed + # the releaser's permit. result False -> abort_all fail-open wake: + # admitted, but we own no permit. + self.permit = bool(self.fut.result()) + self._note_grant(gate._clock() - self.t_enqueued) + return self + except BaseException: + # Cancellation (or any other exception) while waiting: give up the + # queue position and any permit that nonetheless landed, so a + # cancelled waiter neither consumes nor leaks a slot. Consult the + # wake *flag*: a False wake (abort_all / fail-open) owns no permit, + # so releasing one here would over-release and silently un-limit + # the lane; a True wake is an in-transit permit and must re-enter + # the FIFO, because a bare Semaphore.release() never wakes a + # queued waiter (only _pass_permit/abort_all drain the queue). + if self.fut.done() and not self.fut.cancelled() and self.fut.result(): + gate._pass_permit(self.cls) + # That in-transit permit is back in the lane now; forget the + # ownership flag so the entry-failure cleanup in __aenter__ + # cannot hand the same permit back a second time. + self.permit = False + if self.fut in queue: + queue.remove(self.fut) + if self.counted: + self.counted = False + gate._dec_waiting(self.cls) + raise + finally: + if abort_fut is not None: + # The disconnect waiter is ours; never leave it pending on the + # caller's event after this holder has settled. + abort_fut.cancel() + if self.counted: + self.counted = False + gate._dec_waiting(self.cls) + + def _warn_fail_open(self, timeout): + """One advisory line per fail-open grant (SPEC §4: never reject).""" + # One formatted string: the injected logger is tabbyAPI's XLogger, + # whose info/warning take (msg, log_extra: dict | None, details), not + # printf *args — extra positional values TypeError (verified against + # common.logger.XLogger on the pinned image). + self.gate._logger.warning( + f"admission: request_id={self.request_id} class={self.cls} " + f"waited >{timeout:.1f}s for a {self.cls} slot " + f"(prompt_tokens={self.prompt_tokens} " + f"max_new_tokens={self.max_new_tokens}); " + f"proceeding fail-open, never rejected", + ) + + def _note_grant(self, waited): + self.t_granted = self.gate._clock() + self.gate._held[self.cls] = self.gate._held.get(self.cls, 0) + 1 + self.admitted = True + self.gate._logger.info( + f"admission: granted request_id={self.request_id} " + f"class={self.cls} prompt_tokens={self.prompt_tokens} " + f"waited={waited:.3f}s (held={self.gate._held[self.cls]})", + ) + + def _release_owned(self): + """Return everything this holder currently owns. Idempotent. + + The single exit path for *both* :meth:`__aexit__` and the failed-entry + path in :meth:`__aenter__`: each owned thing is handed back once and + its flag cleared, so calling this again — or calling it after the + normal exit — releases nothing further. A permit-free fail-open wake + owns no permit, so it returns only its ``_held`` count. + """ + gate = self.gate + if not gate._enabled: + return + if self.admitted: + self.admitted = False + gate._dec_held(self.cls) + if self.permit: + # Hand the slot straight to the next live waiter (FIFO), else + # return the permit to the semaphore. Never release more than + # was granted. + self.permit = False + gate._pass_permit(self.cls) + + async def __aexit__(self, *exc_info): + self._release_owned() + return False + + +class LaneGate: + """Request-class admission gate. Construct via :meth:`from_config`.""" + + def __init__(self): + self._enabled = False + self._logger = logging.getLogger("admission") + self._rules = {} # name -> _ClassRule, declaration order + self._default = "default" # sentinel for the disabled gate + self._hold_timeout = DEFAULT_HOLD_TIMEOUT + self._max_batch_size = None + self._sems = {} # class -> asyncio.Semaphore (loop-bound) + self._queues = {} # class -> deque of waiter futures (FIFO) + self._held = {} # class -> int + self._waiting = {} # class -> int + self._lane_loop = None # loop the semaphores above belong to + + @classmethod + def from_config(cls, raw, *, max_batch_size=None, logger=None): + """Build a gate from the ``admission`` config block. + + ``None`` or ``{}`` yields a **disabled** gate. Raises ``ValueError`` + naming the offending key for malformed *config* — validation happens + here, at config-load time only; nothing in this module raises at + request time. ``logger`` defaults to ``logging.getLogger("admission")`` + so tabbyAPI can inject its own while this module stays stdlib-only. + """ + gate = cls() + gate._logger = logger if logger is not None else logging.getLogger("admission") + if raw is None or (isinstance(raw, dict) and not raw): + gate._enabled = False + return gate + if not _is_mapping(raw): + raise ValueError(f"admission: config must be a mapping, got {raw!r}") + for key in raw: + if key not in _CONFIG_KEYS: + raise ValueError(f"admission: unknown key {key!r}") + + classes = raw.get("classes") + if classes is None or (isinstance(classes, dict) and not classes): + raise ValueError(f"admission: classes must be a non-empty mapping, got {classes!r}") + if not _is_mapping(classes): + raise ValueError(f"admission: classes must be a mapping, got {classes!r}") + # Ordered mapping: preserve declaration order — first match wins. + for name, body in classes.items(): + gate._rules[name] = _ClassRule(name, body) + + default = raw.get("default") + if default is None: + # No explicit default: fall back to the first declared class. + gate._default = next(iter(gate._rules)) + else: + if default not in gate._rules: + raise ValueError( + f"admission: default {default!r} is not a declared class " + f"(known: {list(gate._rules)})" + ) + gate._default = default + + hold_timeout = raw.get("hold_timeout", DEFAULT_HOLD_TIMEOUT) + if hold_timeout is None: + hold_timeout = DEFAULT_HOLD_TIMEOUT + if isinstance(hold_timeout, bool) or not isinstance(hold_timeout, (int, float)): + raise ValueError(f"admission: hold_timeout must be a number, got {hold_timeout!r}") + if hold_timeout < 0: + raise ValueError(f"admission: hold_timeout must be >= 0, got {hold_timeout!r}") + gate._hold_timeout = float(hold_timeout) + + gate._max_batch_size = max_batch_size + # Oversubscription is advisory, never fatal: class capacities that + # exceed the engine's max_batch_size are legal (the engine queues), + # but usually a misconfiguration — say so once and keep working. + if max_batch_size is not None: + total = sum(rule.max_concurrent for rule in gate._rules.values()) + if total > max_batch_size: + gate._logger.warning( + f"admission: sum of class max_concurrent ({total}) " + f"exceeds max_batch_size ({max_batch_size}); advisory " + f"only, gate stays enabled", + ) + gate._enabled = True + return gate + + @property + def enabled(self) -> bool: + return self._enabled + + def classify(self, prompt_tokens, max_new_tokens): + """First class whose present predicates all match, else ``default``. + + Never raises: garbage request-side values (negative token counts, + non-numbers) degrade to the default class with a warning. Bad + *config* is rejected at :meth:`from_config`, not here. + """ + if not isinstance(prompt_tokens, (int, float)) or isinstance(prompt_tokens, bool): + self._logger.warning( + f"admission: non-numeric prompt_tokens {prompt_tokens!r}; using default class", + ) + prompt_tokens = -1 + if max_new_tokens is not None and ( + isinstance(max_new_tokens, bool) or not isinstance(max_new_tokens, (int, float)) + ): + self._logger.warning( + f"admission: non-numeric max_new_tokens {max_new_tokens!r}; treating as no limit", + ) + max_new_tokens = None + # NaN/inf defeat `<` comparisons (nan < x is False, which would + # *match* a gte predicate): route them to the default class instead + # of letting them masquerade as a long request. + if isinstance(prompt_tokens, float) and not math.isfinite(prompt_tokens): + self._logger.warning( + f"admission: non-finite prompt_tokens {prompt_tokens!r}; using default class", + ) + prompt_tokens = -1 + if isinstance(max_new_tokens, float) and not math.isfinite(max_new_tokens): + self._logger.warning( + f"admission: non-finite max_new_tokens {max_new_tokens!r}; treating as no limit", + ) + max_new_tokens = None + for rule in self._rules.values(): + if rule.matches(prompt_tokens, max_new_tokens): + return rule.name + return self._default + + def acquire( + self, cls, request_id, *, prompt_tokens=None, max_new_tokens=None, abort_event=None + ): + """``async with gate.acquire(cls, request_id): ...`` + + Awaits capacity for *one slot of class ``cls``*; holds it for the + body of the ``async with``. Never raises on timeout (warns and + proceeds); releases on every exit path, including cancellation. + The disabled gate returns a no-op CM that never awaits. + + ``abort_event`` is the tabbyAPI ``DisconnectHandler.abort_event`` + (SPEC §3/§5): already set on entry or fired while queued → the hold is + abandoned *without* taking a permit and ``asyncio.CancelledError`` is + raised (the in-tree representation of a vanished client — no new + exception class); a grant landing in the same tick is handed back + through the FIFO, not consumed. It is ignored while the gate is + disabled (SPEC §8). The three keyword args are additive optionals: the + required call shape stays ``acquire(cls, request_id)``. + """ + if not self._enabled: + return _NOOP_CM + if cls not in self._rules: + # Request-time badness must never raise: park in the default + # lane, warn once per unknown name is acceptable noise (rare). + self._logger.warning( + f"admission: request_id={request_id} asked for undeclared " + f"class {cls!r}; using default {self._default!r}", + ) + cls = self._default + self._lanes_for(asyncio.get_running_loop()) + self._logger.info( + f"admission: hold request_id={request_id} class={cls} " + f"prompt_tokens={prompt_tokens} " + f"max_new_tokens={max_new_tokens}", + ) + return _Holder( + self, cls, request_id, prompt_tokens, max_new_tokens, abort_event=abort_event + ) + + def held(self): + """Snapshot of in-flight holders per class (``{}`` when idle).""" + return dict(self._held) + + def waiting(self): + """Snapshot of queued waiters per class (``{}`` when idle).""" + return dict(self._waiting) + + def abort_all(self): + """Release every waiter immediately, FIFO order (used by unload). + + Waiters are *not* rejected — each proceeds fail-open (with a warning) + so a cancelling unload cannot turn held requests into client errors. + No-op when nothing is waiting. Wakes at most one release per waiter + already queued, in queue (arrival) order; wake-ups own no permit, so + the semaphore count can never exceed its limit. + """ + if not self._enabled: + return + try: + self._lanes_for(asyncio.get_running_loop()) + except RuntimeError: + pass # no running loop: operate on whatever queues exist + for queue in self._queues.values(): + while queue: + fut = queue.popleft() + if not fut.done(): + # False = fail-open wake (admitted, owns no permit) — + # never release more than was granted. + fut.set_result(False) + # A done future is a timed-out/cancelled waiter already on + # its way out of the queue; it holds no queue position. + + # -- internals --------------------------------------------------------- + + def _lanes_for(self, loop): + """Build the loop-bound semaphores/queues on first use. + + ``from_config`` may run at config-load time with no event loop, so + the primitives are created lazily on the first request (and rebuilt + if the loop changes — a test-only scenario; production binds one + loop and never rebinds). + """ + if self._lane_loop is loop: + return + self._sems = { + name: asyncio.Semaphore(rule.max_concurrent) for name, rule in self._rules.items() + } + self._queues = {name: deque() for name in self._rules} + self._held = {} + self._waiting = {} + self._lane_loop = loop + + def _clock(self): + try: + return asyncio.get_running_loop().time() + except RuntimeError: + return time.monotonic() + + def _dec_waiting(self, cls): + n = self._waiting.get(cls, 0) - 1 + if n <= 0: + self._waiting.pop(cls, None) + else: + self._waiting[cls] = n + + def _dec_held(self, cls): + n = self._held.get(cls, 0) - 1 + if n <= 0: + self._held.pop(cls, None) + else: + self._held[cls] = n + + def _pass_permit(self, cls): + """Hand the freed permit to the next live FIFO waiter, else return it. + + Never calls ``Semaphore.release`` for a permit that a waiter has not + consumed, so the semaphore count can never exceed its limit. + """ + queue = self._queues[cls] + while queue: + fut = queue.popleft() + if not fut.done(): + fut.set_result(True) + return + # done == already timed out or cancelled; that holder releases + # any permit it raced into itself. + self._sems[cls].release() diff --git a/backends/exllamav3/model.py b/backends/exllamav3/model.py index 63972f9b..c3d1fb78 100644 --- a/backends/exllamav3/model.py +++ b/backends/exllamav3/model.py @@ -27,6 +27,7 @@ Tokenizer, ) from exllamav3.cache import CacheLayer_quant +from backends.exllamav3.admission import LaneGate from backends.exllamav3.grammar import ExLlamaV3Grammar, MessageGate from backends.exllamav3.sampler import ( @@ -289,6 +290,9 @@ class ExllamaV3Container: chunk_size: int = 2048 max_rq_tokens: Optional[int] = 2048 max_batch_size: Optional[int] = None + # Request-class admission gate (lanes): None until __init__ creates the + # disabled gate, then replaced at every load from the admission block. + admission: Optional[LaneGate] = None draft_num_tokens: Optional[int] = None dynamic_draft: Optional[bool] = False draft_confidence: float = 0.4 @@ -311,6 +315,8 @@ def __init__(self): self.load_condition = asyncio.Condition() self.autosplit_reserve = [96 / 1024] self.encode_cache = OrderedDict() + # Disabled gate until a load supplies an admission block. + self.admission = LaneGate.from_config(None) # Required methods @classmethod @@ -697,6 +703,16 @@ async def create(cls, model_directory: pathlib.Path, hf_model: HFModel, **kwargs output_chunking = unwrap(kwargs.get("output_chunking"), True) self.max_rq_tokens = self.chunk_size if output_chunking else None + # Request-class admission (lanes): from_config validates the block at + # load and defers every loop-bound primitive to the first acquire, so + # no event loop is needed here. A missing or empty admission block + # leaves the gate disabled and the request path untouched. + self.admission = LaneGate.from_config( + unwrap(kwargs.get("admission"), None), + max_batch_size=self.max_batch_size, + logger=xlogger, + ) + # Warm up kernels and graphs after loading, sized from the settings above self.warmup_enabled = unwrap(kwargs.get("warmup"), False) @@ -1036,6 +1052,16 @@ async def wait_for_jobs(self, skip_wait: bool = False): skip_wait: If True, cancel jobs immediately instead of waiting. """ + # Lane holds are not jobs: active_job_ids means "job the engine is + # running", and a request queued on its lane has no entry there, so + # the cancel and drain loops below cannot reach it. Release the holds + # whenever the model is going away, ahead of the generator check so a + # hold is released even on a teardown whose generator is already gone. + # A held client that then proceeds into a tearing-down engine hits the + # existing not-loaded error path; the alternative is sleeping until + # hold_timeout and failing into the same path much later. + self.admission.abort_all() + if not self.generator: return @@ -2542,266 +2568,292 @@ async def generate_gen( in_reasoning = phases is not None and phases.reasoning generation = {} - try: - job = AsyncJob( - self.generator, - sampler=reasoning_sampler if in_reasoning else sampler, - input_ids=input_ids, - max_new_tokens=max_tokens, - min_new_tokens=unwrap(params.min_tokens, 0), - token_healing=( - unwrap(params.token_healing, False) - and self._can_heal(input_ids[0], stop_conditions) - ), - decode_special_tokens=True, - stop_conditions=stop_conditions, - banned_strings=reasoning_banned if in_reasoning else content_banned, - embeddings=mm_embeddings_content, - return_top_tokens=params.top_logprobs, - return_probs=bool(params.logprobs) or bool(params.top_logprobs), - return_logits=return_logits, - max_rq_tokens=max_rq_tokens, - stop_on_loop=bool(params.loop_detect), - filters=( - [] - if in_reasoning and not (phases.engine_trigger or phases.gated) - else grammar_handler.filters - ), - ) - except AssertionError as ex: - # The engine's own page check; the preflight above mirrors it, but a request - # it misses must still reach the client as a context-length error and not - # as a generic failure - if "cannot be enqueued" not in str(ex): - raise - raise ContextLengthExceededError( - f"{ex}. The request exceeds the available context size " - "(this model's maximum context length)." - ) from ex - self.active_job_ids[request_id] = job - if phases is not None: - phases.job = job - self.job_phases[request_id] = phases - await disconnect_handler.add_cleanup_task(id(job), job.cancel, ()) - job_status = status_display.add_job(request_id, label, context_len) - job_status.is_waiting = lambda: self._job_pending(job) - job_status.prompt = prompt - - async def cancel_from_status(): - # Cancelling wakes the consumer below even while the job waits in - # the engine's queue; the loop then ends without an EOS result - job_status.cancel_requested = True - await job.cancel() + # Classify on the limit the client asked for: max_tokens above + # is the engine-resolved value, which auto-expands an absent + # limit to the whole remaining window and would file every + # open-ended request into the longest lane. + client_max_new = params.max_tokens if unwrap(params.max_tokens, 0) > 0 else None + # Lane admission: hold a slot for this request's class before the + # job is built, and hold it until every exit path below has run. + # The split can only be expressed here: exllamav3 re-queues a + # long job at the head of its pending list, so nothing inside + # the engine can keep a short request from being overtaken. The + # async with frees the slot on every exit, including the + # AssertionError mapped to ContextLengthExceededError below and + # any cancellation, so no path can strand a permit. The handler + # may be None on some call paths, so its abort event is read + # defensively. A disabled gate's context manager never awaits: + # with no admission block in the yml, this block is inert. + async with self.admission.acquire( + self.admission.classify(context_len, client_max_new), + request_id, + prompt_tokens=context_len, + max_new_tokens=client_max_new, + abort_event=disconnect_handler.abort_event if disconnect_handler else None, + ): + try: + job = AsyncJob( + self.generator, + sampler=reasoning_sampler if in_reasoning else sampler, + input_ids=input_ids, + max_new_tokens=max_tokens, + min_new_tokens=unwrap(params.min_tokens, 0), + token_healing=( + unwrap(params.token_healing, False) + and self._can_heal(input_ids[0], stop_conditions) + ), + decode_special_tokens=True, + stop_conditions=stop_conditions, + banned_strings=reasoning_banned if in_reasoning else content_banned, + embeddings=mm_embeddings_content, + return_top_tokens=params.top_logprobs, + return_probs=bool(params.logprobs) or bool(params.top_logprobs), + return_logits=return_logits, + max_rq_tokens=max_rq_tokens, + stop_on_loop=bool(params.loop_detect), + filters=( + [] + if in_reasoning and not (phases.engine_trigger or phases.gated) + else grammar_handler.filters + ), + ) + except AssertionError as ex: + # The engine's own page check; the preflight above mirrors it, but a request + # it misses must still reach the client as a context-length error and not + # as a generic failure + if "cannot be enqueued" not in str(ex): + raise + raise ContextLengthExceededError( + f"{ex}. The request exceeds the available context size " + "(this model's maximum context length)." + ) from ex + self.active_job_ids[request_id] = job + if phases is not None: + phases.job = job + self.job_phases[request_id] = phases + await disconnect_handler.add_cleanup_task(id(job), job.cancel, ()) + job_status = status_display.add_job(request_id, label, context_len) + job_status.is_waiting = lambda: self._job_pending(job) + job_status.prompt = prompt + + async def cancel_from_status(): + # Cancelling wakes the consumer below even while the job waits in + # the engine's queue; the loop then ends without an EOS result + job_status.cancel_requested = True + await job.cancel() - job_status.cancel = cancel_from_status + job_status.cancel = cancel_from_status - generated_tokens = 0 - full_response = "" - metrics_result = {} - # Only the OAI request types carry these flags; Kobold requests don't - return_progress = bool(getattr(params, "return_progress", False)) - stop_after_output = bool(getattr(params, "stop_after_output", False)) - stopped_at_output = False - prefill_start_time: float | None = None - prefill_cached_tokens = 0 - - # Get the generation status once it's ready - try: - async for result in job: - await disconnect_handler.poll() + generated_tokens = 0 + full_response = "" + metrics_result = {} + # Only the OAI request types carry these flags; Kobold requests don't + return_progress = bool(getattr(params, "return_progress", False)) + stop_after_output = bool(getattr(params, "stop_after_output", False)) + stopped_at_output = False + prefill_start_time: float | None = None + prefill_cached_tokens = 0 - stage = result.get("stage") - if stage == "started": - # The started event carries no counts; the job knows how - # much of its prompt was found in the cache at allocation - prefill_cached_tokens = self._job_cached_tokens(job) - job_status.started(prefill_cached_tokens) - prefill_start_time = time.time() - elif stage == "prefill": - job_status.prefill( - result.get("curr_progress", 0), result.get("max_progress", 0) - ) - if return_progress and prefill_start_time is not None: - # Time since the engine began prefill for this job. The - # started event and the first prefill event often arrive - # in the same batch, so the event stamp lags by a chunk - started_at = getattr(job.job, "time_first_prefill", None) - yield { - "_prefill_progress": { - "total": result.get("max_progress", 0), - "cache": prefill_cached_tokens, - "processed": result.get("curr_progress", 0), - "time_ms": int( - (time.time() - (started_at or prefill_start_time)) * 1000 - ), + # Get the generation status once it's ready + try: + async for result in job: + await disconnect_handler.poll() + + stage = result.get("stage") + if stage == "started": + # The started event carries no counts; the job knows how + # much of its prompt was found in the cache at allocation + prefill_cached_tokens = self._job_cached_tokens(job) + job_status.started(prefill_cached_tokens) + prefill_start_time = time.time() + elif stage == "prefill": + job_status.prefill( + result.get("curr_progress", 0), result.get("max_progress", 0) + ) + if return_progress and prefill_start_time is not None: + # Time since the engine began prefill for this job. The + # started event and the first prefill event often arrive + # in the same batch, so the event stamp lags by a chunk + started_at = getattr(job.job, "time_first_prefill", None) + yield { + "_prefill_progress": { + "total": result.get("max_progress", 0), + "cache": prefill_cached_tokens, + "processed": result.get("curr_progress", 0), + "time_ms": int( + (time.time() - (started_at or prefill_start_time)) * 1000 + ), + } } - } - # The generator can produce several results per iteration - # (speculative decoding), while this consumer may only get one - # scheduling slot per iteration whenever an await above or - # downstream actually suspends. Merge everything already - # queued so streaming keeps pace with generation instead of - # backing up until the generator goes idle. - if result.get("stage") == "streaming" and not result.get("eos"): - span = [result] - while not job.queue.empty(): - nxt = job.queue.get_nowait() - if isinstance(nxt, Exception): - raise nxt - if not isinstance(nxt, dict): - # Cancellation sentinel. A cancel from the status page - # still sends what was generated; the loop ends next - if job_status.cancel_requested: + # The generator can produce several results per iteration + # (speculative decoding), while this consumer may only get one + # scheduling slot per iteration whenever an await above or + # downstream actually suspends. Merge everything already + # queued so streaming keeps pace with generation instead of + # backing up until the generator goes idle. + if result.get("stage") == "streaming" and not result.get("eos"): + span = [result] + while not job.queue.empty(): + nxt = job.queue.get_nowait() + if isinstance(nxt, Exception): + raise nxt + if not isinstance(nxt, dict): + # Cancellation sentinel. A cancel from the status page + # still sends what was generated; the loop ends next + if job_status.cancel_requested: + break + raise CancelledError("Job cancelled while draining results") + if nxt.get("stage") != "streaming": + continue + span.append(nxt) + if nxt.get("eos"): break - raise CancelledError("Job cancelled while draining results") - if nxt.get("stage") != "streaming": - continue - span.append(nxt) - if nxt.get("eos"): - break - if len(span) > 1: - result = _merge_stream_results(span) - - chunk = unwrap(result.get("text"), "") - if chunk: - chunk_tokens = result.get("token_ids") - if chunk_tokens is None: - chunk_tokens = self.tokenizer.encode(chunk) - full_response += chunk - job_status.append_text(chunk) - - # Extract token IDs as a plain list for downstream consumers - if isinstance(chunk_tokens, torch.Tensor): - token_id_list = chunk_tokens.flatten().tolist() - generated_tokens += len(token_id_list) - elif isinstance(chunk_tokens, tuple): - first = chunk_tokens[0] - if isinstance(first, torch.Tensor): - token_id_list = first.flatten().tolist() + if len(span) > 1: + result = _merge_stream_results(span) + + chunk = unwrap(result.get("text"), "") + if chunk: + chunk_tokens = result.get("token_ids") + if chunk_tokens is None: + chunk_tokens = self.tokenizer.encode(chunk) + full_response += chunk + job_status.append_text(chunk) + + # Extract token IDs as a plain list for downstream consumers + if isinstance(chunk_tokens, torch.Tensor): + token_id_list = chunk_tokens.flatten().tolist() + generated_tokens += len(token_id_list) + elif isinstance(chunk_tokens, tuple): + first = chunk_tokens[0] + if isinstance(first, torch.Tensor): + token_id_list = first.flatten().tolist() + else: + token_id_list = list(first) + generated_tokens += len(token_id_list) else: - token_id_list = list(first) - generated_tokens += len(token_id_list) - else: - token_id_list = list(chunk_tokens) - generated_tokens += len(token_id_list) - - # Increase penalty range to generated token amount - # TODO: - # if auto_scale_penalty_range: - # gen_settings.token_repetition_range = generated_tokens - - generation = { - "request_id": request_id, - "text": chunk, - "token_ids": token_id_list, - "prompt_tokens": context_len, - "generated_tokens": generated_tokens, - "offset": len(full_response), - } + token_id_list = list(chunk_tokens) + generated_tokens += len(token_id_list) + + # Increase penalty range to generated token amount + # TODO: + # if auto_scale_penalty_range: + # gen_settings.token_repetition_range = generated_tokens + + generation = { + "request_id": request_id, + "text": chunk, + "token_ids": token_id_list, + "prompt_tokens": context_len, + "generated_tokens": generated_tokens, + "offset": len(full_response), + } - # Raw logits for the generated positions, when requested - if result.get("logits") is not None: - generation["logits"] = result["logits"] + # Raw logits for the generated positions, when requested + if result.get("logits") is not None: + generation["logits"] = result["logits"] - if params.logprobs > 0: - self.handle_logprobs(result, generation) + if params.logprobs > 0: + self.handle_logprobs(result, generation) - job_status.generated(generated_tokens) - yield generation + job_status.generated(generated_tokens) + yield generation - if result.get("eos"): - xlogger.debug("EOS result received from generator", result) - finish_chunk = self.handle_finish_chunk( - result, request_id, full_response, label - ) - await disconnect_handler.finish(id(job)) + if result.get("eos"): + xlogger.debug("EOS result received from generator", result) + finish_chunk = self.handle_finish_chunk( + result, request_id, full_response, label + ) + await disconnect_handler.finish(id(job)) + + # Save the final result for metrics logging + metrics_result = finish_chunk + job_status.finish_reason = finish_chunk["finish_reason"] + + if ( + finish_chunk["finish_reason"] == "repetition" + and params.loop_detect_error + ): + raise LoopDetectedError( + "Generation failed: the model started repeating itself and " + f"loop detection stopped it after {finish_chunk['gen_tokens']} " + "tokens" + ) - # Save the final result for metrics logging - metrics_result = finish_chunk - job_status.finish_reason = finish_chunk["finish_reason"] + yield finish_chunk + break - if finish_chunk["finish_reason"] == "repetition" and params.loop_detect_error: - raise LoopDetectedError( - "Generation failed: the model started repeating itself and " - f"loop detection stopped it after {finish_chunk['gen_tokens']} " - "tokens" + if chunk and stop_after_output: + # Text is out and the job has no EOS of its own to reach; + # the loop ends and the response finishes below + stopped_at_output = True + await job.cancel() + else: + # The job ended without an EOS result. Cancelled from the status + # page or stopped at its first output: finish the response with + # what was generated. Otherwise the request itself was cancelled + # and nothing is sent + if job_status.cancel_requested or stopped_at_output: + if job_status.cancel_requested: + xlogger.warning(f"{label}: cancelled from the status page") + finish_chunk = self.handle_finish_chunk( + { + "eos_reason": ( + "cancelled" if job_status.cancel_requested else "max_new_tokens" + ), + "prompt_tokens": context_len, + "cached_tokens": prefill_cached_tokens, + "new_tokens": generated_tokens, + **self._job_times(job), + }, + request_id, + full_response, + label, ) + await disconnect_handler.finish(id(job)) + metrics_result = finish_chunk + job_status.finish_reason = finish_chunk["finish_reason"] + yield finish_chunk - yield finish_chunk - break - - if chunk and stop_after_output: - # Text is out and the job has no EOS of its own to reach; - # the loop ends and the response finishes below - stopped_at_output = True + except CancelledError: + if not job.cancelled: await job.cancel() - else: - # The job ended without an EOS result. Cancelled from the status - # page or stopped at its first output: finish the response with - # what was generated. Otherwise the request itself was cancelled - # and nothing is sent - if job_status.cancel_requested or stopped_at_output: - if job_status.cancel_requested: - xlogger.warning(f"{label}: cancelled from the status page") - finish_chunk = self.handle_finish_chunk( - { - "eos_reason": ( - "cancelled" if job_status.cancel_requested else "max_new_tokens" - ), - "prompt_tokens": context_len, - "cached_tokens": prefill_cached_tokens, - "new_tokens": generated_tokens, - **self._job_times(job), - }, - request_id, - full_response, - label, - ) - await disconnect_handler.finish(id(job)) - metrics_result = finish_chunk - job_status.finish_reason = finish_chunk["finish_reason"] - yield finish_chunk - - except CancelledError: - if not job.cancelled: - await job.cancel() - except LoopDetectedError: - # The job ended normally in the engine; only the response fails - raise - - except Exception as ex: - await self._recover_from_generation_error(ex, job) + except LoopDetectedError: + # The job ended normally in the engine; only the response fails + raise - if isinstance(ex, AssertionError) and "cannot be enqueued" in str(ex): - raise ContextLengthExceededError( - f"{str(ex)}. The request exceeds the available context size." - ) from ex + except Exception as ex: + await self._recover_from_generation_error(ex, job) - raise ex - finally: - status_display.remove_job(request_id) - self.job_phases.pop(request_id, None) + if isinstance(ex, AssertionError) and "cannot be enqueued" in str(ex): + raise ContextLengthExceededError( + f"{str(ex)}. The request exceeds the available context size." + ) from ex - # Log generation options to console - # Some options are too large, so log the args instead - log_generation_params( - label, - request_id=request_id, - bos_token_id=self.bos_token_id, - eos_token_id=eos_tokens, - prompt=prompt, - **params.model_dump(exclude={"prompt"}), - # auto_scale_penalty_range=auto_scale_penalty_range, # TODO - ) + raise ex + finally: + status_display.remove_job(request_id) + self.job_phases.pop(request_id, None) - # Log the metrics if present - if metrics_result: - log_metrics( + # Log generation options to console + # Some options are too large, so log the args instead + log_generation_params( label, - metrics_result, - context_len, - self.max_seq_len, + request_id=request_id, + bos_token_id=self.bos_token_id, + eos_token_id=eos_tokens, + prompt=prompt, + **params.model_dump(exclude={"prompt"}), + # auto_scale_penalty_range=auto_scale_penalty_range, # TODO ) - MetricsManager.record_finish(metrics_result) + + # Log the metrics if present + if metrics_result: + log_metrics( + label, + metrics_result, + context_len, + self.max_seq_len, + ) + MetricsManager.record_finish(metrics_result) diff --git a/common/config_models.py b/common/config_models.py index e7b0b2cc..01742169 100644 --- a/common/config_models.py +++ b/common/config_models.py @@ -6,7 +6,7 @@ PrivateAttr, field_validator, ) -from typing import List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union CACHE_SIZES = Literal["FP16", "Q8", "Q6", "Q4"] @@ -444,6 +444,20 @@ class ModelConfig(BaseConfigModel): ), ge=1, ) + admission: Optional[Dict[str, Any]] = Field( + None, + description=( + "Request-class admission control (default: None, disabled).\n" + "Splits requests into classes by their declared size and holds a lane\n" + "slot per class before the job reaches the engine, so small queries\n" + "keep their latency while long agentic sessions never share the\n" + "engine with each other. Keys: classes (per-class predicates\n" + "prompt_tokens_gte / max_new_tokens_gte plus max_concurrent, first\n" + "match wins), default, hold_timeout in seconds. A held request is\n" + "never rejected, it waits for its lane and always eventually\n" + "proceeds. Used by EXL3 models only.\n" + ), + ) recurrent_checkpoint_interval: Optional[int] = Field( None, description=( diff --git a/tests/test_admission.py b/tests/test_admission.py new file mode 100644 index 00000000..6ff5205d --- /dev/null +++ b/tests/test_admission.py @@ -0,0 +1,1342 @@ +"""Tests for the exllamav3 lane-admission gate. + +Stdlib unittest only — runs offline: + python3 -m unittest tests.test_admission +""" + +import asyncio +import logging +import unittest + +from backends.exllamav3 import admission + + +# Reference config mirroring the plan's SPEC §2 example, with an extra +# catch-all so `default` and first-declared fallback can be told apart. +SPEC_CONFIG = { + "classes": { + "long": {"prompt_tokens_gte": 32768, "max_new_tokens_gte": None, "max_concurrent": 1}, + "mid": {"prompt_tokens_gte": 4096, "max_new_tokens_gte": 1024, "max_concurrent": 1}, + "short": {"max_concurrent": 2}, + }, + "default": "short", + "hold_timeout": 1800, +} + + +class DisabledGateTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.mod = admission + + def test_from_config_none_is_disabled(self): + gate = self.mod.LaneGate.from_config(None, max_batch_size=None) + self.assertFalse(gate.enabled) + + def test_default_logger_is_named_admission(self): + # Ruling 2: logger=None means logging.getLogger("admission"). + gate = self.mod.LaneGate.from_config(None, max_batch_size=None) + self.assertEqual(gate._logger.name, "admission") + gate2 = self.mod.LaneGate.from_config(SPEC_CONFIG, max_batch_size=2) + self.assertEqual(gate2._logger.name, "admission") + + def test_disabled_abort_all_held_waiting_are_noops(self): + gate = self.mod.LaneGate.from_config({}, max_batch_size=1) + gate.abort_all() + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + + def test_from_config_empty_is_disabled(self): + gate = self.mod.LaneGate.from_config({}, max_batch_size=None) + self.assertFalse(gate.enabled) + + async def test_disabled_acquire_never_awaits_and_keeps_no_state(self): + gate = self.mod.LaneGate.from_config(None, max_batch_size=1) + + # Swap the module's asyncio namespace for one that raises on ANY + # attribute access: if the disabled `async with` completes, the gate + # provably never touched the event loop (binding: "no await" when + # unconfigured — the zero-behavior-change / mergeability property). + class _ForbiddenAsyncio: + def __getattr__(self, name): + raise AssertionError(f"disabled gate must not use asyncio.{name}") + + real_asyncio = self.mod.asyncio + self.mod.asyncio = _ForbiddenAsyncio() + try: + async with gate.acquire("long", "req-1"): + pass + cls = gate.classify(100_000, None) + finally: + self.mod.asyncio = real_asyncio + self.assertIsInstance(cls, str) + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + # abort_all on a disabled/idle gate must be a harmless no-op. + gate.abort_all() + + +class ClassificationTests(unittest.TestCase): + def setUp(self): + self.mod = admission + + def test_enabled_for_real_config(self): + gate = self.mod.LaneGate.from_config(SPEC_CONFIG, max_batch_size=2) + self.assertTrue(gate.enabled) + + def test_boundary_inclusive(self): + gate = self.mod.LaneGate.from_config(SPEC_CONFIG, max_batch_size=2) + self.assertEqual(gate.classify(32768, None), "long") + self.assertEqual(gate.classify(32767, None), "short") # via default + self.assertEqual(gate.classify(100_000, None), "long") + + def test_declaration_order_preserved_first_match_wins(self): + # Alphabetical sorting would put "zzz" first and swallow everything; + # insertion order must win instead. + cfg = { + "classes": { + "first": {"prompt_tokens_gte": 32768}, + "zzz": {"max_concurrent": 1}, + }, + } + gate = self.mod.LaneGate.from_config(cfg, max_batch_size=2) + self.assertEqual(gate.classify(40000, None), "first") + + def test_default_fallback(self): + gate = self.mod.LaneGate.from_config(SPEC_CONFIG, max_batch_size=2) + self.assertEqual(gate.classify(10, None), "short") + + def test_default_absent_falls_back_to_first_declared_class(self): + cfg = { + "classes": { + "long": {"prompt_tokens_gte": 32768}, + "short": {"max_concurrent": 2}, + }, + # deliberately no "default" key + } + gate = self.mod.LaneGate.from_config(cfg, max_batch_size=2) + self.assertEqual(gate.classify(10, None), "long") + + def test_both_predicates_are_anded(self): + gate = self.mod.LaneGate.from_config(SPEC_CONFIG, max_batch_size=2) + # mid needs BOTH prompt>=4096 AND max_new>=1024 + self.assertEqual(gate.classify(4096, 1024), "mid") + self.assertEqual(gate.classify(4096, 1023), "short") # tokens too small + self.assertEqual(gate.classify(4095, 1024), "short") # prompt too small + + def test_null_predicate_is_absent_predicate(self): + # long declares max_new_tokens_gte: null -> that predicate is absent, + # so a tiny max_new_tokens must not stop a 40k prompt from classing long. + gate = self.mod.LaneGate.from_config(SPEC_CONFIG, max_batch_size=2) + self.assertEqual(gate.classify(40000, 1), "long") + + def test_none_max_new_tokens_never_matches_class_declaring_it(self): + cfg = { + "classes": { + "capped": {"max_new_tokens_gte": 256, "max_concurrent": 1}, + "other": {"max_concurrent": 1}, + }, + "default": "other", + } + gate = self.mod.LaneGate.from_config(cfg, max_batch_size=2) + self.assertEqual(gate.classify(10, None), "other") + self.assertEqual(gate.classify(10, 255), "other") + self.assertEqual(gate.classify(10, 256), "capped") + + def test_predicateless_class_matches_nothing(self): + # "bare" declares no predicates: it must NOT act as a catch-all, so + # a small request goes to `default` (mid), not to "bare". + cfg = { + "classes": { + "long": {"prompt_tokens_gte": 32768, "max_concurrent": 1}, + "bare": {"max_concurrent": 2}, + "mid": {"prompt_tokens_gte": 64, "max_concurrent": 1}, + }, + "default": "mid", + } + gate = self.mod.LaneGate.from_config(cfg, max_batch_size=2) + self.assertEqual(gate.classify(100, None), "mid") # matched mid, not bare + self.assertEqual(gate.classify(10, None), "mid") # via default, not bare + self.assertEqual(gate.classify(40000, None), "long") + + def test_max_concurrent_defaults_to_one(self): + cfg = {"classes": {"solo": {"max_concurrent": 1}, "lone": {}}} + gate = self.mod.LaneGate.from_config(cfg, max_batch_size=2) + # "lone" omitted max_concurrent -> 1; observable via semaphores in + # GateTests; here just ensure classify never raises on it. + self.assertEqual(gate.classify(10, None), "solo") # default = first class + + +class _Capture(logging.Handler): + def __init__(self): + super().__init__() + self.records = [] + + def emit(self, record): + self.records.append(record) + + def text(self): + return "\n".join(r.getMessage() for r in self.records) + + +def make_gate(mod, cfg, logger, max_batch_size=None): + return mod.LaneGate.from_config(cfg, max_batch_size=max_batch_size, logger=logger) + + +def holding(gate, cls, rid, held_evt, release_evt): + """Task body: acquire, signal `held_evt`, block until `release_evt`.""" + + async def body(): + async with gate.acquire(cls, rid): + held_evt.set() + await release_evt.wait() + + return asyncio.get_running_loop().create_task(body()) + + +async def _until(predicate, poll=0.005): + """Poll `predicate` until true; wrap in wait_for so a regression FAILS.""" + while not predicate(): + await asyncio.sleep(poll) + + +class _GrantBoomLogger: + """Injected logger stand-in that raises on the grant line of chosen requests. + + The gate never configures logging: the caller injects its own logger + (Task 3 injects tabbyAPI's), so the grant line is the first place a + foreign logger can blow up — a custom logger object, a formatter error, + an ``extra`` key collision. ``boom_ids`` names which requests fail, so a + test can hold a lane normally and fail only the grant under examination. + The surface mirrors exactly what this module calls (``info`` / + ``warning`` / ``debug``); nothing of the module under test is patched or + mocked. + """ + + BOOM_ON = "admission: granted" + + def __init__(self, boom_ids=("*",)): + self.boom_ids = set(boom_ids) + self.messages = [] + + @staticmethod + def _text(msg, args): + # Same lazy-format contract logging.Logger.info() has — the module + # under test always passes %-style args, never a pre-formatted string. + return msg % args if args else str(msg) + + def info(self, msg, *args): + text = self._text(msg, args) + self.messages.append(text) + if not text.startswith(self.BOOM_ON): + return + if "*" in self.boom_ids or any(f"request_id={rid} class=" in text for rid in self.boom_ids): + raise RuntimeError(f"injected logger failed on the grant line: {text}") + + def warning(self, msg, *args): + self.messages.append(self._text(msg, args)) + + def debug(self, msg, *args): + self.messages.append(self._text(msg, args)) + + +async def _acquire_once(gate, cls, rid): + """One full hold that exits immediately — propagates any __aenter__ error.""" + async with gate.acquire(cls, rid): + pass + + +class GateTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.mod = admission + self.log = _Capture() + self.logger = logging.getLogger("test.admission.gate") + self.logger.handlers = [self.log] + self.logger.setLevel(logging.DEBUG) + self.logger.propagate = False + + async def test_same_class_semaphore_of_one_blocks_second(self): + cfg = {"classes": {"long": {"max_concurrent": 1}}, "default": "long"} + gate = make_gate(self.mod, cfg, self.logger) + h1, r1, h2, r2 = (asyncio.Event() for _ in range(4)) + t1 = holding(gate, "long", "req-1", h1, r1) + await h1.wait() + self.assertEqual(gate.held(), {"long": 1}) + t2 = holding(gate, "long", "req-2", h2, r2) + await asyncio.sleep(0.01) + self.assertFalse(h2.is_set(), "second long request must be held back") + self.assertEqual(gate.waiting(), {"long": 1}) + self.assertEqual(gate.held(), {"long": 1}) + r1.set() + await t1 + await asyncio.wait_for(h2.wait(), timeout=1) + self.assertEqual(gate.held(), {"long": 1}) # slot moved, not duplicated + r2.set() + await t2 + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + + async def test_short_class_never_blocked_by_long(self): + cfg = { + "classes": { + "long": {"prompt_tokens_gte": 32768, "max_concurrent": 1}, + "short": {"max_concurrent": 2}, + }, + "default": "short", + } + gate = make_gate(self.mod, cfg, self.logger, max_batch_size=2) + h1, r1, h2, r2 = (asyncio.Event() for _ in range(4)) + t1 = holding(gate, "long", "req-1", h1, r1) + await h1.wait() + # The 2k query classes short and must be admitted immediately while + # the 100k agentic session holds the long lane. + self.assertEqual(gate.classify(2000, None), "short") + t2 = holding(gate, "short", "req-2", h2, r2) + await asyncio.wait_for(h2.wait(), timeout=1) + self.assertEqual(gate.held(), {"long": 1, "short": 1}) + r1.set() + r2.set() + await asyncio.gather(t1, t2) + self.assertEqual(gate.held(), {}) + + async def test_two_shorts_admit_third_blocks(self): + cfg = { + "classes": { + "long": {"prompt_tokens_gte": 32768, "max_concurrent": 1}, + "short": {"max_concurrent": 2}, + }, + "default": "short", + } + gate = make_gate(self.mod, cfg, self.logger, max_batch_size=3) + evts = [asyncio.Event() for _ in range(6)] + h1, r1, h2, r2, h3, r3 = evts + t1 = holding(gate, "short", "a", h1, r1) + await h1.wait() + t2 = holding(gate, "short", "b", h2, r2) + await h2.wait() + self.assertEqual(gate.held(), {"short": 2}) + t3 = holding(gate, "short", "c", h3, r3) + await asyncio.sleep(0.01) + self.assertFalse(h3.is_set()) + self.assertEqual(gate.waiting(), {"short": 1}) + r1.set() + await t1 + await asyncio.wait_for(h3.wait(), timeout=1) + r2.set() + r3.set() + await asyncio.gather(t2, t3) + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + + async def test_fifo_order_within_class(self): + cfg = {"classes": {"solo": {"max_concurrent": 1}}, "default": "solo"} + gate = make_gate(self.mod, cfg, self.logger) + order = [] + release = asyncio.Event() + first_done = asyncio.Event() + + async def holder(rid): + async with gate.acquire("solo", rid): + order.append(rid) + if rid == "first": + first_done.set() + await release.wait() + + t0 = asyncio.get_running_loop().create_task(holder("first")) + await first_done.wait() + tails = [asyncio.get_running_loop().create_task(holder(f"w{i}")) for i in range(3)] + await asyncio.sleep(0.01) + self.assertEqual(gate.waiting(), {"solo": 3}) + release.set() + await asyncio.gather(t0, *tails) + self.assertEqual(order, ["first", "w0", "w1", "w2"]) + + async def test_grant_and_hold_log_lines(self): + cfg = {"classes": {"solo": {"max_concurrent": 1}}, "default": "solo"} + gate = make_gate(self.mod, cfg, self.logger) + async with gate.acquire("solo", "req-42", prompt_tokens=1234): + pass + text = self.log.text() + self.assertIn("req-42", text) + self.assertIn("solo", text) + self.assertIn("1234", text) + self.assertIn("waited", text) + + +class HoldTimeoutTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.mod = admission + self.log = _Capture() + self.logger = logging.getLogger("test.admission.timeout") + self.logger.handlers = [self.log] + self.logger.setLevel(logging.DEBUG) + self.logger.propagate = False + + async def test_timeout_proceeds_with_warning_never_raises(self): + cfg = { + "classes": {"solo": {"max_concurrent": 1}}, + "default": "solo", + "hold_timeout": 0.05, + } + gate = make_gate(self.mod, cfg, self.logger) + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "solo", "holder", h1, r1) + await h1.wait() + h2, r2 = asyncio.Event(), asyncio.Event() + t2 = holding(gate, "solo", "waiter", h2, r2) + # Fail-open: the waiter proceeds on its own after the timeout. + await asyncio.wait_for(h2.wait(), timeout=2) + text = self.log.text() + self.assertIn("waiter", text) + self.assertIn("fail-open", text) + self.assertEqual(gate.held(), {"solo": 2}) # holder + fail-open waiter + r1.set() + r2.set() + await asyncio.gather(t1, t2) + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + + async def test_timeout_expiry_does_not_consume_slot_for_exits(self): + # After a fail-open expiry, releasing the real holder must let a + # NEW request in normally (no stranded permit). + cfg = { + "classes": {"solo": {"max_concurrent": 1}}, + "default": "solo", + "hold_timeout": 0.05, + } + gate = make_gate(self.mod, cfg, self.logger) + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "solo", "holder", h1, r1) + await h1.wait() + h2, r2 = asyncio.Event(), asyncio.Event() + t2 = holding(gate, "solo", "expired", h2, r2) + # Bounded: a fail-open regression must FAIL this test, not hang CI. + await asyncio.wait_for(h2.wait(), timeout=2) + r1.set() + await t1 + h3, r3 = asyncio.Event(), asyncio.Event() + t3 = holding(gate, "solo", "fresh", h3, r3) + await asyncio.wait_for(h3.wait(), timeout=2) # admitted by permit, not timeout + r2.set() + r3.set() + await asyncio.gather(t2, t3) + self.assertEqual(gate.waiting(), {}) + + async def test_zero_hold_timeout_waits_forever(self): + cfg = { + "classes": {"solo": {"max_concurrent": 1}}, + "default": "solo", + "hold_timeout": 0, + } + gate = make_gate(self.mod, cfg, self.logger) + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "solo", "holder", h1, r1) + await h1.wait() + h2, r2 = asyncio.Event(), asyncio.Event() + t2 = holding(gate, "solo", "patient", h2, r2) + await asyncio.sleep(0.1) + self.assertFalse(h2.is_set(), "hold_timeout 0 must not fail open") + fail_opens = [ + r + for r in self.log.records + if "fail-open" in r.getMessage() and "patient" in r.getMessage() + ] + self.assertEqual(fail_opens, [], "hold_timeout 0 must arm no fail-open timer") + self.assertEqual(gate.waiting(), {"solo": 1}) + r1.set() + await asyncio.wait_for(h2.wait(), timeout=2) # eventually granted + r2.set() + await asyncio.gather(t1, t2) + self.assertEqual(gate.held(), {}) + + async def test_default_hold_timeout_is_1800(self): + gate = make_gate(self.mod, {"classes": {"s": {}}}, self.logger) + self.assertEqual(gate._hold_timeout, 1800.0) + gate2 = make_gate(self.mod, {"classes": {"s": {}}, "hold_timeout": None}, self.logger) + self.assertEqual(gate2._hold_timeout, 1800.0) + + +class CancellationTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.mod = admission + self.log = _Capture() + self.logger = logging.getLogger("test.admission.cancel") + self.logger.handlers = [self.log] + self.logger.setLevel(logging.DEBUG) + self.logger.propagate = False + + def _gate(self, cfg=None): + cfg = cfg or {"classes": {"solo": {"max_concurrent": 1}}, "default": "solo"} + return make_gate(self.mod, cfg, self.logger) + + async def test_cancel_while_held_frees_slot(self): + gate = self._gate() + body_entered = asyncio.Event() + + async def leaky(): + async with gate.acquire("solo", "victim"): + body_entered.set() + await asyncio.sleep(10) # cancelled here, mid-hold + + t = asyncio.get_running_loop().create_task(leaky()) + await body_entered.wait() + self.assertEqual(gate.held(), {"solo": 1}) + t.cancel() + with self.assertRaises(asyncio.CancelledError): + await t + self.assertEqual(gate.held(), {}, "cancellation must release the slot") + self.assertEqual(gate.waiting(), {}) + # The slot is really free: a new request is admitted without waiting. + h, r = asyncio.Event(), asyncio.Event() + t2 = holding(gate, "solo", "next", h, r) + await asyncio.wait_for(h.wait(), timeout=1) + r.set() + await t2 + + async def test_cancel_while_body_raises_frees_slot(self): + gate = self._gate() + + async def raiser(): + async with gate.acquire("solo", "boom"): + raise RuntimeError("handler blew up") + + with self.assertRaises(RuntimeError): + await raiser() + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + + async def test_cancel_while_waiting_leaves_no_trace_and_no_leak(self): + gate = self._gate() + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "solo", "blocker", h1, r1) + await h1.wait() + + async def queued(): + async with gate.acquire("solo", "queued"): + pass # never reached + + t2 = asyncio.get_running_loop().create_task(queued()) + await asyncio.sleep(0.01) + self.assertEqual(gate.waiting(), {"solo": 1}) + t2.cancel() + with self.assertRaises(asyncio.CancelledError): + await t2 + self.assertEqual(gate.waiting(), {}, "cancelled waiter must leave waiting() at 0") + self.assertEqual(gate.held(), {"solo": 1}) # only the blocker remains + # No slot was consumed by the cancelled waiter: when the blocker + # exits, the next arrival is admitted immediately. + r1.set() + await t1 + self.assertEqual(gate.held(), {}) + h3, r3 = asyncio.Event(), asyncio.Event() + t3 = holding(gate, "solo", "after", h3, r3) + await asyncio.wait_for(h3.wait(), timeout=1) + r3.set() + await t3 + self.assertEqual(gate.held(), {}) + + async def test_cancel_waiting_with_two_waiters_preserves_queue(self): + gate = self._gate() + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "solo", "blocker", h1, r1) + await h1.wait() + order = [] + done = asyncio.Event() + + async def queued(rid): + async with gate.acquire("solo", rid): + order.append(rid) + done.set() + await asyncio.sleep(10) + + tm = asyncio.get_running_loop().create_task(queued("middle")) + tl = asyncio.get_running_loop().create_task(queued("last")) + await asyncio.sleep(0.01) + self.assertEqual(gate.waiting(), {"solo": 2}) + tm.cancel() + with self.assertRaises(asyncio.CancelledError): + await tm + self.assertEqual(gate.waiting(), {"solo": 1}) + r1.set() + await asyncio.wait_for(done.wait(), timeout=1) + self.assertEqual(order, ["last"], "queue must survive a middle cancellation") + tl.cancel() + await asyncio.gather(tl, return_exceptions=True) + await t1 + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + + async def test_cancel_after_grant_hands_permit_to_queue(self): + # The grant LANDED (fut resolved True) but the waiter was cancelled in + # the one-tick window before it could resume. That permit is in transit + # and must go back *through the FIFO*: releasing it straight into the + # Semaphore strands the next waiter, because only _pass_permit()/ + # abort_all() drain the queue — the lane would wedge with a full + # semaphore and a queued request nobody ever wakes. + gate = self._gate( + { + "classes": {"solo": {"max_concurrent": 1}}, + "default": "solo", + "hold_timeout": 0, # wait forever: the wedge must be permanent + } + ) + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "solo", "holder", h1, r1) + await h1.wait() + woke = set() + + async def queued(rid): + async with gate.acquire("solo", rid): + woke.add(rid) + await asyncio.sleep(10) + + t2 = asyncio.get_running_loop().create_task(queued("granted")) + t3 = asyncio.get_running_loop().create_task(queued("next")) + await asyncio.sleep(0.01) + self.assertEqual(gate.waiting(), {"solo": 2}) + fut_granted, fut_next = list(gate._queues["solo"]) # FIFO order + r1.set() # holder exits -> permit handed to t2 + await asyncio.sleep(0) # t1's step ran; the grant is in flight to t2 + self.assertTrue( + fut_granted.done() and fut_granted.result() is True, + "the grant must have landed on t2's future", + ) + self.assertFalse(t2.done(), "grant must still be in transit to t2") + self.assertEqual(len(gate._queues["solo"]), 1, "t2 dequeued, t3 still queued") + self.assertFalse(fut_next.done(), "t3 must still be pending at cancel time") + t2.cancel() # cancel inside the in-transit window + with self.assertRaises(asyncio.CancelledError): + await t2 + self.assertNotIn("granted", woke, "cancelled waiter must not reach its body") + # The permit must arrive at t3 via the queue, not idle in the semaphore. + await asyncio.wait_for(_until(lambda: "next" in woke), timeout=1) + self.assertEqual(gate.held(), {"solo": 1}) + self.assertEqual(gate.waiting(), {}) + self.assertLessEqual(gate._sems["solo"]._value, 1, "semaphore must never exceed its limit") + self.assertEqual(gate._sems["solo"]._value, 0, "t3 consumed the permit") + t3.cancel() + await asyncio.gather(t1, t3, return_exceptions=True) + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + self.assertEqual(gate._sems["solo"]._value, 1) + + async def test_cancel_after_abort_all_wake_releases_no_permit(self): + # abort_all() wakes a waiter with result False: admitted, but owning NO + # permit. Cancelling that waiter in the one-tick window before it + # resumes must therefore release nothing — else the lane silently stops + # limiting (a permit that was never owned enters the Semaphore). + gate = self._gate() + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "solo", "holder", h1, r1) + await h1.wait() + woke2 = asyncio.Event() + + async def queued(): + async with gate.acquire("solo", "aborted"): + woke2.set() + await asyncio.sleep(10) + + t2 = asyncio.get_running_loop().create_task(queued()) + await asyncio.sleep(0.01) + self.assertEqual(gate.waiting(), {"solo": 1}) + gate.abort_all() # fail-open wake (result False), owns no permit + t2.cancel() # ... before the wake can be delivered + with self.assertRaises(asyncio.CancelledError): + await t2 + self.assertFalse(woke2.is_set()) + self.assertEqual(gate.waiting(), {}) + self.assertEqual(gate._sems["solo"]._value, 0, "permit-free wake must release nothing") + # The lane still limits: a third request must NOT be admitted while the + # original holder is still inside its body. + h3, r3 = asyncio.Event(), asyncio.Event() + t3 = holding(gate, "solo", "third", h3, r3) + await asyncio.sleep(0.05) + self.assertFalse(h3.is_set(), "over-release admitted a second holder") + self.assertEqual(gate.held(), {"solo": 1}) + r1.set() + await t1 + await asyncio.wait_for(h3.wait(), timeout=1) # admitted normally now + self.assertEqual(gate.held(), {"solo": 1}) + r3.set() + await t3 + self.assertEqual(gate.held(), {}) + self.assertEqual(gate._sems["solo"]._value, 1) + + +class AbortEventTests(unittest.IsolatedAsyncioTestCase): + """SPEC §3/§5 — ``abort_event`` is the queued-path twin of poll().""" + + def setUp(self): + self.mod = admission + self.log = _Capture() + self.logger = logging.getLogger("test.admission.abort_event") + self.logger.handlers = [self.log] + self.logger.setLevel(logging.DEBUG) + self.logger.propagate = False + + def _gate(self, cfg=None): + cfg = cfg or {"classes": {"solo": {"max_concurrent": 1}}, "default": "solo"} + return make_gate(self.mod, cfg, self.logger) + + async def test_abort_set_on_entry_raises_without_acquiring(self): + gate = self._gate() + ev = asyncio.Event() + ev.set() # client already gone when the request reached the gate + with self.assertRaises(asyncio.CancelledError): + async with gate.acquire("solo", "gone", abort_event=ev): + self.fail("body must not run for a vanished client") + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + self.assertEqual(gate._sems["solo"]._value, 1, "no permit may be taken") + + async def test_abort_while_queued_abandons_hold_without_permit(self): + gate = self._gate() + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "solo", "holder", h1, r1) + await h1.wait() + ev = asyncio.Event() + entered = asyncio.Event() + + async def queued(): + async with gate.acquire("solo", "queued", abort_event=ev): + entered.set() + await asyncio.sleep(10) + + t2 = asyncio.get_running_loop().create_task(queued()) + await asyncio.sleep(0.01) + self.assertEqual(gate.waiting(), {"solo": 1}) + ev.set() # client disconnects while queued + with self.assertRaises(asyncio.CancelledError): + await t2 + self.assertFalse(entered.is_set(), "abandoned hold must not reach its body") + self.assertEqual(gate.waiting(), {}, "aborted waiter must leave waiting() at 0") + self.assertEqual(gate.held(), {"solo": 1}) # only the real holder + self.assertEqual(gate._sems["solo"]._value, 0, "abandoning must take/release no permit") + # The lane still limits: a new request queues behind the holder. + h3, r3 = asyncio.Event(), asyncio.Event() + t3 = holding(gate, "solo", "third", h3, r3) + await asyncio.sleep(0.05) + self.assertFalse(h3.is_set()) + self.assertEqual(gate.waiting(), {"solo": 1}) + r1.set() + await t1 + await asyncio.wait_for(h3.wait(), timeout=1) # admitted on the holder's exit + r3.set() + await t3 + self.assertEqual(gate.held(), {}) + self.assertEqual(gate._sems["solo"]._value, 1) + + async def test_abort_after_grant_before_body_returns_permit_to_queue(self): + # Grant and disconnect land in the same tick: the abort wins, and the + # in-transit permit must go back through the FIFO — the in-transit bug + # class of the cancel-after-grant case, reached through abort_event. + gate = self._gate( + { + "classes": {"solo": {"max_concurrent": 1}}, + "default": "solo", + "hold_timeout": 0, + } + ) + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "solo", "holder", h1, r1) + await h1.wait() + ev = asyncio.Event() + woke = set() + + async def queued(rid, abort=None): + async with gate.acquire("solo", rid, abort_event=abort): + woke.add(rid) + await asyncio.sleep(10) + + t2 = asyncio.get_running_loop().create_task(queued("granted", ev)) + t3 = asyncio.get_running_loop().create_task(queued("next")) + await asyncio.sleep(0.01) + self.assertEqual(gate.waiting(), {"solo": 2}) + r1.set() # holder exits -> permit handed to t2 + ev.set() # client vanishes before t2 resumes + with self.assertRaises(asyncio.CancelledError): + await t2 + self.assertNotIn("granted", woke) + await asyncio.wait_for(_until(lambda: "next" in woke), timeout=1) + self.assertEqual(gate.held(), {"solo": 1}) + self.assertEqual(gate.waiting(), {}) + self.assertLessEqual(gate._sems["solo"]._value, 1) + self.assertEqual(gate._sems["solo"]._value, 0, "t3 consumed the permit") + t3.cancel() + await asyncio.gather(t1, t3, return_exceptions=True) + self.assertEqual(gate.held(), {}) + self.assertEqual(gate._sems["solo"]._value, 1) + + async def test_none_abort_event_and_positional_call_keep_today_behaviour(self): + gate = self._gate() + async with gate.acquire("solo", "plain"): # SPEC §3 required call shape + self.assertEqual(gate.held(), {"solo": 1}) + async with gate.acquire("solo", "none", prompt_tokens=7, abort_event=None): + self.assertEqual(gate.held(), {"solo": 1}) + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + self.assertEqual(gate._sems["solo"]._value, 1) + + async def test_task_cancel_while_racing_abort_event_uses_the_same_giveup(self): + # Fix 1 + Fix 2 together: the task is *cancelled* (not aborted) while it + # sits in the grant/abort race with the event attached and an in-transit + # grant. The give-up path must be identical — permit back through the + # FIFO, and no leaked disconnect-waiter task. + gate = self._gate( + { + "classes": {"solo": {"max_concurrent": 1}}, + "default": "solo", + "hold_timeout": 0, + } + ) + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "solo", "holder", h1, r1) + await h1.wait() + woke = set() + ev = asyncio.Event() # never set: the client is cancelled, not aborted + + async def queued(rid, abort=None): + async with gate.acquire("solo", rid, abort_event=abort): + woke.add(rid) + await asyncio.sleep(10) + + t2 = asyncio.get_running_loop().create_task(queued("granted", ev)) + t3 = asyncio.get_running_loop().create_task(queued("next")) + await asyncio.sleep(0.01) + self.assertEqual(gate.waiting(), {"solo": 2}) + r1.set() + await asyncio.sleep(0) + self.assertFalse(t2.done(), "grant must still be in transit to t2") + t2.cancel() + with self.assertRaises(asyncio.CancelledError): + await t2 + await asyncio.wait_for(_until(lambda: "next" in woke), timeout=1) + self.assertNotIn("granted", woke) + self.assertEqual(gate.held(), {"solo": 1}) + self.assertEqual(gate.waiting(), {}) + self.assertEqual(gate._sems["solo"]._value, 0) + self.assertFalse( + getattr(ev, "_waiters", []), + "cancelled holder must leave no waiter registered on the caller's event", + ) + t3.cancel() + await asyncio.gather(t1, t3, return_exceptions=True) + self.assertEqual(gate.held(), {}) + self.assertEqual(gate._sems["solo"]._value, 1) + + async def test_abort_event_still_fails_open_on_hold_timeout(self): + # SPEC §4: an attached disconnect signal must not turn the fail-open + # timer into a rejection path — expiry still proceeds, permit-free. + gate = self._gate( + { + "classes": {"solo": {"max_concurrent": 1}}, + "default": "solo", + "hold_timeout": 0.05, + } + ) + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "solo", "holder", h1, r1) + await h1.wait() + ev = asyncio.Event() # never set + entered, leave = asyncio.Event(), asyncio.Event() + + async def queued(): + async with gate.acquire("solo", "patient", abort_event=ev): + entered.set() + await leave.wait() + + t2 = asyncio.get_running_loop().create_task(queued()) + await asyncio.wait_for(entered.wait(), timeout=2) + text = self.log.text() + self.assertIn("patient", text) + self.assertIn("fail-open", text) + self.assertEqual(gate.held(), {"solo": 2}) # holder + permit-free waiter + r1.set() + await t1 + self.assertEqual(gate._sems["solo"]._value, 1, "fail-open exit releases nothing") + leave.set() + await t2 + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + self.assertEqual(gate._sems["solo"]._value, 1) + + async def test_disabled_gate_ignores_abort_event_entirely(self): + # SPEC §8: unconfigured means provably no behaviour change — the + # no-op CM neither inspects nor awaits the event (a vanished client is + # still caught by DisconnectHandler.poll() mid-generation). + gate = self.mod.LaneGate.from_config(None, max_batch_size=None, logger=self.logger) + ev = asyncio.Event() + ev.set() + async with gate.acquire("solo", "req", abort_event=ev): + pass + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + + +class AbortAllTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.mod = admission + self.log = _Capture() + self.logger = logging.getLogger("test.admission.abort") + self.logger.handlers = [self.log] + self.logger.setLevel(logging.DEBUG) + self.logger.propagate = False + + def _gate(self, n=1, timeout=30): + cfg = { + "classes": { + "long": {"prompt_tokens_gte": 32768, "max_concurrent": n}, + "short": {"max_concurrent": 2}, + }, + "default": "short", + "hold_timeout": timeout, + } + return make_gate(self.mod, cfg, self.logger, max_batch_size=2) + + async def test_abort_all_noop_when_idle(self): + gate = self._gate() + gate.abort_all() # must not raise + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + + async def test_abort_all_releases_waiters_in_fifo_order(self): + gate = self._gate() + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "long", "blocker", h1, r1) + await h1.wait() + woke = [] + release = asyncio.Event() + + async def waiter(rid): + async with gate.acquire("long", rid): + woke.append(rid) + await release.wait() + + waiters = [] + for i in range(3): + waiters.append(asyncio.get_running_loop().create_task(waiter(f"w{i}"))) + await asyncio.sleep(0.005) # establish strict FIFO queue order + self.assertEqual(gate.waiting(), {"long": 3}) + gate.abort_all() + # Observe the wake directly (the waiters then park on `release`, so + # awaiting their tasks here would deadlock the test itself). + for i in range(3): + for _ in range(200): + if len(woke) > i: + break + await asyncio.sleep(0.005) + self.assertGreater(len(woke), i, f"waiter w{i} must be released") + self.assertEqual(woke, ["w0", "w1", "w2"], "FIFO release order") + self.assertEqual(gate.waiting(), {}) + release.set() + r1.set() + await asyncio.gather(t1, *waiters) + self.assertEqual(gate.waiting(), {}) + self.assertEqual(gate.held(), {}) + # Every wake rode a real or handed-down permit — never an over-release: + # the semaphore must end exactly at its limit. + self.assertEqual(gate._sems["long"]._value, 1) + + async def test_abort_all_only_affects_waiters_not_holders(self): + gate = self._gate() + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "long", "blocker", h1, r1) + await h1.wait() + gate.abort_all() # nothing waiting -> harmless + self.assertEqual(gate.held(), {"long": 1}) + self.assertFalse(t1.done()) + r1.set() + await t1 + + async def test_abort_all_does_not_exceed_granted_permits(self): + # Two long slots, two holders, one waiter: abort wakes exactly the + # waiter; the semaphore count never exceeds its limit at any point. + gate = self._gate(n=2) + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "long", "h1", h1, r1) + await h1.wait() + h2, r2 = asyncio.Event(), asyncio.Event() + t2 = holding(gate, "long", "h2", h2, r2) + await h2.wait() + self.assertEqual(gate.waiting(), {}) # fast path, no queue + release = asyncio.Event() + + async def waiter(rid): + async with gate.acquire("long", rid): + await release.wait() + + w = asyncio.get_running_loop().create_task(waiter("w")) + await asyncio.sleep(0.01) + self.assertEqual(gate.waiting(), {"long": 1}) + self.assertLessEqual(gate._sems["long"]._value, 2) + release.set() # the waiter parks on this once woken + gate.abort_all() + await asyncio.wait_for(w, timeout=1) # woken through the release, not rejected + self.assertEqual(gate._sems["long"]._value, 0) # never over-released + r1.set() + r2.set() + await asyncio.gather(t1, t2) + self.assertEqual(gate._sems["long"]._value, 2) # back to limit exactly + + +class FromConfigValidationTests(unittest.TestCase): + """Config-load-time errors: ValueError naming the offending key.""" + + def setUp(self): + self.mod = admission + + def _expect(self, raw, key_fragment): + with self.assertRaises(ValueError) as cm: + self.mod.LaneGate.from_config(raw, max_batch_size=None) + msg = str(cm.exception) + self.assertIn(key_fragment, msg, f"message must name {key_fragment!r}: {msg}") + + def test_non_mapping_raw(self): + self._expect(["not", "a", "mapping"], "admission") + self._expect(42, "admission") + + def test_non_mapping_classes(self): + self._expect({"classes": ["long", "short"]}, "classes") + + def test_empty_classes(self): + self._expect({"classes": {}}, "classes") + self._expect({"classes": None}, "classes") + + def test_negative_max_concurrent(self): + self._expect( + {"classes": {"long": {"max_concurrent": -1}}}, + "classes.long.max_concurrent", + ) + + def test_zero_max_concurrent(self): + self._expect( + {"classes": {"long": {"max_concurrent": 0}}}, + "classes.long.max_concurrent", + ) + + def test_non_int_max_concurrent(self): + self._expect( + {"classes": {"long": {"max_concurrent": "two"}}}, + "classes.long.max_concurrent", + ) + self._expect( + {"classes": {"long": {"max_concurrent": 1.5}}}, + "classes.long.max_concurrent", + ) + + def test_unknown_default(self): + self._expect( + {"classes": {"long": {}}, "default": "nope"}, + "default", + ) + + def test_negative_hold_timeout(self): + self._expect({"classes": {"long": {}}, "hold_timeout": -5}, "hold_timeout") + + def test_non_mapping_class_body(self): + self._expect({"classes": {"long": 7}}, "classes.long") + + def test_negative_predicate(self): + self._expect( + {"classes": {"long": {"prompt_tokens_gte": -1}}}, + "classes.long.prompt_tokens_gte", + ) + + def test_unknown_key_names_the_key(self): + self._expect({"classes": {"long": {}}, "typo_key": 1}, "typo_key") + self._expect({"classes": {"long": {"hold_timout": 3}}}, "hold_timout") + + +class RequestTimeResilienceTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.mod = admission + self.log = _Capture() + self.logger = logging.getLogger("test.admission.resilient") + self.logger.handlers = [self.log] + self.logger.setLevel(logging.DEBUG) + self.logger.propagate = False + + def _gate(self): + cfg = { + "classes": { + "long": {"prompt_tokens_gte": 32768, "max_concurrent": 1}, + "short": {"max_concurrent": 2}, + }, + "default": "short", + } + return make_gate(self.mod, cfg, self.logger, max_batch_size=2) + + async def test_negative_prompt_tokens_classes_default_no_raise(self): + gate = self._gate() + self.assertEqual(gate.classify(-5, None), "short") + self.assertEqual(gate.classify(0, -7), "short") + + async def test_garbage_request_values_never_raise(self): + gate = self._gate() + self.assertEqual(gate.classify("lots", None), "short") + self.assertEqual(gate.classify(None, "many"), "short") + self.assertEqual(gate.classify(float("nan"), None), "short") # NaN < x is False... + # ...but a NaN must never match the long lane: + self.assertNotEqual(gate.classify(float("nan"), None), "long") + + async def test_acquire_undeclared_class_falls_to_default_no_raise(self): + gate = self._gate() + async with gate.acquire("nonexistent", "req-x"): + self.assertEqual(gate.held(), {"short": 1}) + self.assertEqual(gate.held(), {}) + self.assertIn("nonexistent", self.log.text()) + + async def test_hold_and_release_balances_under_mixed_exits(self): + gate = self._gate() + for i in range(6): + async with gate.acquire("short", f"r{i}", prompt_tokens=10 * i): + pass + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + self.assertEqual(gate._sems["short"]._value, 2) + + +class EntryFailureTests(unittest.IsolatedAsyncioTestCase): + """``__aenter__`` must be *failure-safe*: everything it already owns is + handed back when it raises, because Python never calls ``__aexit__`` for a + failed ``async with`` entry. Without that, a permit spent by a grant is + stranded for the life of the process — on a ``max_concurrent: 1`` lane, one + occurrence disables the lane forever. + + Trigger: the injected logger raising on the grant line (see + ``_GrantBoomLogger``). Entry is driven both on the pre-grant fast path and + through the FIFO, because the two paths reach the grant line from + different places (outside the try / inside it). + """ + + def setUp(self): + self.mod = admission + + @staticmethod + def _cfg(cls="solo", n=1): + return {"classes": {cls: {"max_concurrent": n}}, "default": cls} + + async def test_grant_log_failure_on_fast_path_returns_permit(self): + logger = _GrantBoomLogger({"req-1"}) + gate = make_gate(self.mod, self._cfg(), logger) + with self.assertRaises(RuntimeError): + async with gate.acquire("solo", "req-1"): + self.fail("body must not run when the grant line raises") + self.assertEqual( + gate._sems["solo"]._value, + 1, + "the spent permit must be returned when __aenter__ raises", + ) + self.assertEqual(gate.held(), {}, "failed entry must not stay counted held") + self.assertEqual(gate.waiting(), {}) + # The lane is still whole: the next request is admitted without waiting. + h2, r2 = asyncio.Event(), asyncio.Event() + t2 = holding(gate, "solo", "req-2", h2, r2) + await asyncio.wait_for(h2.wait(), timeout=1) + self.assertEqual(gate.held(), {"solo": 1}) + r2.set() + await t2 + self.assertEqual(gate.held(), {}) + self.assertEqual(gate._sems["solo"]._value, 1) + + async def test_grant_log_failure_on_queued_path_keeps_lane_alive(self): + logger = _GrantBoomLogger({"queued"}) + gate = make_gate(self.mod, self._cfg(), logger) + h1, r1 = asyncio.Event(), asyncio.Event() + t1 = holding(gate, "solo", "holder", h1, r1) + await h1.wait() + # t2 must genuinely wait, then be granted by t1's release. + t2 = asyncio.get_running_loop().create_task(_acquire_once(gate, "solo", "queued")) + await asyncio.sleep(0.01) + self.assertEqual(gate.waiting(), {"solo": 1}) + r1.set() # the release grants to t2 ... + with self.assertRaises(RuntimeError): + await t2 # ... and the grant line raises inside its entry + self.assertEqual(gate.waiting(), {}) + self.assertEqual(gate.held(), {}, "failed entry must not stay counted held") + self.assertLessEqual(gate._sems["solo"]._value, 1, "semaphore must never exceed its limit") + # A later request is admitted normally AND the lane still limits to 1: + # an over-release would silently admit a second holder. + h3, r3 = asyncio.Event(), asyncio.Event() + t3 = holding(gate, "solo", "third", h3, r3) + await asyncio.wait_for(h3.wait(), timeout=1) + self.assertEqual(gate.held(), {"solo": 1}) + h4, r4 = asyncio.Event(), asyncio.Event() + t4 = holding(gate, "solo", "fourth", h4, r4) + await asyncio.sleep(0.05) + self.assertFalse(h4.is_set(), "over-release would admit a second holder") + self.assertEqual(gate._sems["solo"]._value, 0) + r3.set() + await asyncio.wait_for(h4.wait(), timeout=1) + r4.set() + await asyncio.gather(t1, t3, t4) + self.assertEqual(gate.held(), {}) + self.assertEqual(gate.waiting(), {}) + self.assertEqual(gate._sems["solo"]._value, 1) + + async def test_entry_failure_then_aexit_releases_nothing_more(self): + logger = _GrantBoomLogger({"req-1"}) + gate = make_gate(self.mod, self._cfg(), logger) + holder = gate.acquire("solo", "req-1") + with self.assertRaises(RuntimeError): + await holder.__aenter__() + self.assertEqual(gate._sems["solo"]._value, 1, "permit already returned") + self.assertFalse(holder.permit, "ownership flag must be cleared") + self.assertFalse(holder.admitted) + # `async with` never calls __aexit__ after a failed entry, but a caller + # that does (or a second cleanup pass) must not push a permit back in. + await holder.__aexit__(None, None, None) + holder._release_owned() + self.assertEqual(gate._sems["solo"]._value, 1, "must stay at the class limit exactly") + self.assertEqual(gate.held(), {}) + + +class OversubscriptionTests(unittest.TestCase): + def setUp(self): + self.mod = admission + self.log = _Capture() + self.logger = logging.getLogger("test.admission.oversub") + self.logger.handlers = [self.log] + self.logger.setLevel(logging.DEBUG) + self.logger.propagate = False + + def test_oversubscription_warns_once_and_keeps_working(self): + cfg = { + "classes": { + "long": {"max_concurrent": 2}, + "short": {"max_concurrent": 2}, + }, + "default": "short", + } + gate = make_gate(self.mod, cfg, self.logger, max_batch_size=1) + self.assertTrue(gate.enabled) + warns = [r for r in self.log.records if r.levelno >= logging.WARNING] + self.assertEqual(len(warns), 1, "exactly one advisory line") + self.assertIn("max_batch_size", warns[0].getMessage()) + # Advisory, never fatal: classification still works. + self.assertEqual(gate.classify(10, None), "short") + + def test_no_warning_when_within_batch_size(self): + cfg = { + "classes": { + "long": {"max_concurrent": 1}, + "short": {"max_concurrent": 1}, + }, + "default": "short", + } + gate = make_gate(self.mod, cfg, self.logger, max_batch_size=2) + warns = [r for r in self.log.records if r.levelno >= logging.WARNING] + self.assertEqual(warns, []) + + def test_no_warning_without_max_batch_size(self): + cfg = {"classes": {"long": {"max_concurrent": 4}}} + gate = make_gate(self.mod, cfg, self.logger, max_batch_size=None) + warns = [r for r in self.log.records if r.levelno >= logging.WARNING] + self.assertEqual(warns, []) + + +class _XLoggerStrict: + """Mimics tabbyAPI's ``common.logger.XLogger`` — the logger the model.py + patch injects into the gate in production. + + Unlike stdlib ``logging.Logger``, its ``info``/``warning`` take NO + printf-style ``*args``: the second positional is a dict. Every gate log + call must therefore be ONE formatted string. Production incident + 2026-11-14: the 6-positional ``info`` call at admission.py:527 raised + ``TypeError: XLogger.info() takes from 2 to 4 positional arguments but 6 + were given`` and 500'd every chat request. The offline suite's stdlib + logger accepted the same call, which is why it missed.""" + + def __init__(self): + self.messages = [] + + def info(self, log_message, log_extra=None, details=None): + self.messages.append(("info", log_message)) + + def warning(self, log_message, log_extra=None, details=None): + self.messages.append(("warning", log_message)) + + +class XLoggerSignatureTests(unittest.IsolatedAsyncioTestCase): + """Every log-producing path of the gate, driven through a logger with + tabbyAPI's real signature. A TypeError here is a 500 on every request.""" + + @classmethod + def setUpClass(cls): + cls.mod = admission + + def _gate(self, cfg, xlog, max_batch_size=2): + return self.mod.LaneGate.from_config(cfg, max_batch_size=max_batch_size, logger=xlog) + + CFG = { + "classes": { + "long": {"max_concurrent": 1, "prompt_tokens_gte": 10000}, + "short": {"max_concurrent": 1}, + }, + "default": "short", + } + + async def test_hold_and_grant_lines_survive(self): + xlog = _XLoggerStrict() + gate = self._gate(self.CFG, xlog) + async with gate.acquire("short", "r1"): + pass + levels = [l for l, _ in xlog.messages] + self.assertIn("info", levels, "hold and/or grant line must log") + text = " ".join(m for _, m in xlog.messages) + self.assertIn("granted", text) + + async def test_unknown_class_warning_survives(self): + xlog = _XLoggerStrict() + gate = self._gate(self.CFG, xlog) + async with gate.acquire("bogus", "r2"): + pass + self.assertTrue( + any(l == "warning" for l, _ in xlog.messages), "unknown class must warn without raising" + ) + + async def test_nonfinite_classification_warning_survives(self): + xlog = _XLoggerStrict() + gate = self._gate(self.CFG, xlog) + cls = gate.classify(float("nan"), None) + self.assertEqual(cls, "short", "NaN must fall to the default class") + self.assertTrue(any(l == "warning" for l, _ in xlog.messages)) + + async def test_timeout_fail_open_warning_survives(self): + xlog = _XLoggerStrict() + cfg = dict(self.CFG, hold_timeout=0.05) + cfg = {**self.CFG, "hold_timeout": 0.05} + gate = self._gate(cfg, xlog) + release = asyncio.Event() + + async def holder(): + async with gate.acquire("short", "h1"): + await release.wait() + + t = asyncio.get_running_loop().create_task(holder()) + for _ in range(100): + if gate.held(): + break + await asyncio.sleep(0.005) + try: + async with gate.acquire("short", "r3"): + pass + finally: + release.set() + await t + self.assertTrue( + any("fail-open" in m for _, m in xlog.messages), + "the hold_timeout warning must log without raising", + ) + + def test_advisory_warning_survives(self): + xlog = _XLoggerStrict() + cfg = { + "classes": {"long": {"max_concurrent": 2}, "short": {"max_concurrent": 1}}, + "default": "short", + } + self._gate(cfg, xlog, max_batch_size=2) + self.assertTrue(any(l == "warning" for l, _ in xlog.messages)) + + +if __name__ == "__main__": + unittest.main()