Life of a Request in vLLM V1

File: vllm/v1/core/sched/scheduler.py

The V1 Scheduler uses a unified token budget that covers both prefill and decode tokens. There is no separate "prefill phase" vs "decode phase" -- each request simply has num_computed_tokens that it tries to catch up to num_tokens_with_spec.

class Scheduler(SchedulerInterface):
    def __init__(self, vllm_config, kv_cache_config, ...):
        # Scheduling constraints
        self.max_num_running_reqs = self.scheduler_config.max_num_seqs
        self.max_num_scheduled_tokens = (
            self.scheduler_config.max_num_scheduled_tokens
            if self.scheduler_config.max_num_scheduled_tokens
            else self.scheduler_config.max_num_batched_tokens
        )

        # Priority queues
        self.waiting = create_request_queue(self.policy)
        self.running: list[Request] = []

        # Create the KV cache manager
        self.kv_cache_manager = KVCacheManager(
            kv_cache_config=kv_cache_config,
            max_model_len=self.max_model_len,
            enable_caching=self.cache_config.enable_prefix_caching,
        )

schedule() -- The Scheduling Algorithm

def schedule(self) -> SchedulerOutput:
    # NOTE(woosuk) on the scheduling algorithm:
    # There's no "decoding phase" nor "prefill phase" in the scheduler.
    # Each request just has num_computed_tokens and num_tokens_with_spec.
    # At each step, the scheduler tries to assign tokens to the requests
    # so that each request's num_computed_tokens can catch up its
    # num_tokens_with_spec.

    token_budget = self.max_num_scheduled_tokens

    # First, schedule the RUNNING requests
    req_index = 0
    while req_index < len(self.running) and token_budget > 0:
        request = self.running[req_index]

        num_new_tokens = (
            request.num_tokens_with_spec
            + request.num_output_placeholders
            - request.num_computed_tokens
        )
        num_new_tokens = min(num_new_tokens, token_budget)

        # Allocate KV cache blocks for new tokens
        new_blocks = self.kv_cache_manager.allocate_slots(
            request, num_new_tokens,
            num_lookahead_tokens=self.num_lookahead_tokens,
        )

        if new_blocks is None:
            # Out of memory -- preempt lowest-priority request
            preempted_req = self.running.pop()
            self._preempt_request(preempted_req, ...)
            continue

        # Schedule the request
        num_scheduled_tokens[request_id] = num_new_tokens
        token_budget -= num_new_tokens
        req_index += 1

    # Then, schedule WAITING requests with remaining budget
    # (similar loop over self.waiting queue)
    ...

File: vllm/v1/core/kv_cache_manager.py

The KVCacheManager is responsible for allocating and freeing blocks of GPU memory for KV caches. It supports prefix caching through block hashing.

class KVCacheManager:
    def __init__(self, kv_cache_config, max_model_len, ...):
        self.coordinator = get_kv_cache_coordinator(
            kv_cache_config=kv_cache_config,
            max_model_len=self.max_model_len,
            enable_caching=self.enable_caching,
        )
        self.block_pool = self.coordinator.block_pool

    def get_computed_blocks(self, request) -> tuple[KVCacheBlocks, int]:
        """Find prefix cache hits for a request."""
        if not self.enable_caching:
            return self.empty_kv_cache_blocks, 0

        max_cache_hit_length = request.num_tokens - 1
        computed_blocks, num_new_computed_tokens = (
            self.coordinator.find_longest_cache_hit(
                request.block_hashes, max_cache_hit_length
            )
        )
        return self.create_kv_cache_blocks(computed_blocks), num_new_computed_tokens

File: vllm/v1/worker/gpu_worker.py -- Worker

The Worker manages the GPU device, initializes distributed communication, and hosts the GPUModelRunner. In a single-GPU setup, it runs inside the EngineCore process. With tensor parallelism, separate worker processes handle each GPU.

class Worker(WorkerBase):
    def __init__(self, vllm_config, local_rank, rank, ...):
        super().__init__(
            vllm_config=vllm_config,
            local_rank=local_rank,
            rank=rank,
            distributed_init_method=distributed_init_method,
            is_driver_worker=is_driver_worker,
        )

File: vllm/v1/worker/gpu_model_runner.py -- GPUModelRunner

The GPUModelRunner is the most complex component. It manages the persistent InputBatch, builds attention metadata, runs the model forward pass, and samples output tokens.

class GPUModelRunner(
    LoRAModelRunnerMixin, KVConnectorModelRunnerMixin, ECConnectorModelRunnerMixin
):
    def __init__(self, vllm_config, device):
        self.max_num_tokens = scheduler_config.max_num_batched_tokens
        self.max_num_reqs = scheduler_config.max_num_seqs
        self.device = device
        self.dtype = self.model_config.dtype
        ...

execute_model() -- The GPU Forward Pass

def execute_model(
    self, scheduler_output: SchedulerOutput,
    intermediate_tensors: IntermediateTensors | None = None,
) -> ModelRunnerOutput | AsyncModelRunnerOutput | IntermediateTensors | None:

    num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens

    # 1. Update persistent batch states from SchedulerOutput
    deferred_state_corrections_fn = self._update_states(scheduler_output)

    if not num_scheduled_tokens:
        return EMPTY_MODEL_RUNNER_OUTPUT

    num_reqs = self.input_batch.num_reqs
    req_ids = self.input_batch.req_ids

    # 2. Prepare inputs (build attention metadata, positions, etc.)
    logits_indices, spec_decode_metadata = self._prepare_inputs(
        scheduler_output, num_scheduled_tokens_np,
    )

    # 3. Determine CUDA graph mode and padding
    (cudagraph_mode, batch_desc, should_ubatch,
     num_tokens_across_dp, cudagraph_stats,
    ) = self._determine_batch_execution_and_padding(
        num_tokens=num_tokens_unpadded,
        num_reqs=num_reqs,
        num_scheduled_tokens_np=num_scheduled_tokens_np,
        max_num_scheduled_tokens=max_num_scheduled_tokens,
        ...
    )

    # 4. Build attention metadata per layer
    # 5. Run the actual model forward pass
    # 6. Sample tokens from logits
    # 7. Return ModelRunnerOutput

14.3 Unified Token Budget (Chunked Prefill)

The scheduler makes no distinction between "prefill tokens" and "decode tokens". Both consume from the same token budget per step. This enables:

• Mixed batching: Prefill and decode requests run in the same step
• Chunked prefill: Long prompts are split across multiple steps, sharing budget with decode requests
• Speculative decoding: Draft tokens are just more tokens in the same budget

# From the schedule() docstring:
# There's no "decoding phase" nor "prefill phase" in the scheduler.
# Each request just has the num_computed_tokens and
# num_tokens_with_spec. At each step, the scheduler tries to assign
# tokens to the requests so that each request's num_computed_tokens
# can catch up its num_tokens_with_spec.

msgspec Structs with gc=False

All cross-process data structures use msgspec.Struct with gc=False to disable Python's cyclic garbage collector tracking. Combined with array_like=True for compact serialization and omit_defaults=True to skip default-valued fields, this minimizes both serialization overhead and GC pause times.

GC Heap Freezing

After initialization, EngineCore calls freeze_gc_heap() to mark all existing objects as permanent. This prevents the GC from scanning startup allocations (model weights, config objects) during collection cycles, significantly reducing GC pause times.

# At the end of EngineCore.__init__():

# Mark the startup heap as static so that it's ignored by GC.
# Reduces pause times of oldest generation collections.
freeze_gc_heap()
# Enable environment variable cache
enable_envs_cache()

Zero-Copy Buffer Reuse in Output Thread

The output I/O thread reuses serialization buffers via encode_into(outputs, buffer) and tracks ZMQ message delivery with zmq.MessageTracker. Buffers are only reclaimed when ZMQ confirms the message has been sent, preventing premature deallocation of backing memory for numpy arrays and tensors.

# Output thread buffer management:
reuse_buffers: list[bytearray] = []
pending = deque[tuple[zmq.MessageTracker, Any, bytearray]]()

# Reclaim buffers that zmq is finished with
while pending and pending[-1][0].done:
    reuse_buffers.append(pending.pop()[2])

buffer = reuse_buffers.pop() if reuse_buffers else bytearray()
buffers = encoder.encode_into(outputs, buffer)
tracker = socket.send_multipart(buffers, copy=False, track=True)

Async Output Copy Stream

GPU-to-CPU copies of sampled tokens are performed on a separate CUDA stream, overlapping the copy with the next forward pass. The AsyncGPUModelRunnerOutput wrapper lazily materializes the CPU result only when needed.

# vllm/v1/worker/gpu_model_runner.py
class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
    def __init__(self, model_runner_output, sampled_token_ids, ...):
        # Initiate copy on a separate stream (non-blocking)
        with torch.cuda.stream(async_output_copy_stream):
            async_output_copy_stream.wait_stream(default_stream)
            self.sampled_token_ids_cpu = self._sampled_token_ids.to(
                "cpu", non_blocking=True
            )
            self.async_copy_ready_event.record()

    def get_output(self) -> ModelRunnerOutput:
        """Blocks until copy is finished, then returns."""
        self.async_copy_ready_event.synchronize()
        # Convert to Python lists
        valid_sampled_token_ids = self.sampled_token_ids_cpu.tolist()
        ...

Output Processing Chunking

The API process splits output processing into chunks of VLLM_V1_OUTPUT_PROC_CHUNK_SIZE to avoid blocking the asyncio event loop when there are many concurrent requests. Between chunks, it yields control with await asyncio.sleep(0).

# In output_handler():
chunk_size = envs.VLLM_V1_OUTPUT_PROC_CHUNK_SIZE

for start in range(0, num_outputs, chunk_size):
    end = start + chunk_size
    outputs_slice = engine_core_outputs[start:end]
    processed_outputs = output_processor.process_outputs(outputs_slice, ...)
    # Allow other asyncio tasks to run between chunks
    if end < num_outputs:
        await asyncio.sleep(0)