Skip to content

vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler

Classes:

OffloadingConnectorScheduler

Implementation of Scheduler side methods

Methods:

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
class OffloadingConnectorScheduler:
    """Implementation of Scheduler side methods"""

    _offloading_enabled = True

    def __init__(
        self,
        spec: OffloadingSpec,
        vllm_config: VllmConfig,
        kv_cache_config: KVCacheConfig,
    ):
        self.config = SchedulerOffloadConfig.from_spec(
            spec, vllm_config, kv_cache_config
        )
        self._group_configs_by_id = {
            config.group_idx: config for config in self.config.kv_group_configs
        }
        self.manager: OffloadingManager = spec.get_manager()
        # Under async_init the engine serves without offloading until every
        # worker has reported ready.
        self._offloading_enabled = not spec.async_init
        self._ready_ranks: set[int] = set()
        if not self._offloading_enabled:
            logger.info("KV offloading is off until every worker is ready")
        self._connector_stats = OffloadingConnectorStats()
        self.num_hash_lookahead_tokens = vllm_config.num_hash_lookahead_tokens
        self.use_lookahead_block_hashes = self.num_hash_lookahead_tokens > 0

        # The static config facts ride on the first stats payload of this
        # process. A Prometheus child holds the values it received, so one
        # payload is enough.
        self._info_sent = False

        full_attention_groups: list[int] = []
        sliding_window_groups: list[int] = []
        for config_group_idx, group_config in enumerate(self.config.kv_group_configs):
            if group_config.sliding_window_size_in_chunks is None:
                full_attention_groups.append(config_group_idx)
            else:
                sliding_window_groups.append(config_group_idx)

        # sort sliding window groups by window size in decreasing order
        def _sliding_window_sort_key(i: int) -> int:
            val = self.config.kv_group_configs[i].sliding_window_size_in_chunks
            assert val is not None
            return val

        sliding_window_groups.sort(key=_sliding_window_sort_key, reverse=True)

        # used by _lookup
        self._full_attention_groups: tuple[int, ...] = tuple(full_attention_groups)
        self._sliding_window_groups: tuple[int, ...] = tuple(sliding_window_groups)
        self._lookup_groups = self._full_attention_groups + self._sliding_window_groups
        self._mamba_align_size: int | None = resolve_mamba_align_size(
            spec, kv_cache_config
        )

        self._req_status: dict[ReqId, RequestOffloadState] = {}
        self._current_batch_load_jobs: dict[int, TransferJob] = {}
        # Requests loading synchronously in the current engine step
        self._current_batch_sync_loads: set[ReqId] = set()
        self._current_batch_sync_load_bytes = 0
        self._current_batch_jobs_to_flush: set[int] = set()
        # GPU block IDs allocated in the current engine step
        self._current_batch_allocated_block_ids: set[int] = set()
        # if GPU prefix caching is enabled,
        # Track loaded chunks to avoid redundant loads.
        self._chunks_being_loaded: set[OffloadKey] | None = (
            set() if vllm_config.cache_config.enable_prefix_caching else None
        )

        # Job ID counter shared by loads and stores.
        self._job_counter: int = 0
        # Threshold value for stale jobs. All job ids >= _stale_job_threshold are
        # active jobs.
        self._stale_job_threshold: int = 0
        # Jobs flushed by a cache reset -> worker reports still due. Their
        # chunks are released at once, but a worker may still write them (a
        # late DMA) until it reports; the manager learns through
        # on_reset_flushed() once every one has.
        self._reset_fence: dict[int, int] = {}
        self._reset_fence_since = 0.0
        self._job_statuses: dict[int, TransferJobStatus] = {}

        # block_id -> pending store job_ids. Used to track jobs that needs
        # flushing in case a block is re-allocated by the KV cache manager.
        # Populated only for finished requests (running-request blocks are
        # protected by their ref_cnt) and for sliding window blocks (which can
        # be freed before a request finishes).
        self._block_id_to_pending_jobs: dict[int, set[int]] = {}

        self._events_tracker = OffloadingEventsTracker(
            spec.kv_events_config,
            {
                group.group_idx: group.kv_event_group_spec
                for group in self.config.kv_group_configs
            },
            num_hash_lookahead_tokens=vllm_config.num_hash_lookahead_tokens,
        )

        handler_classes: list[type[StoreTriggerHandler]] = []
        if self.config.supports_handoff:
            handler_classes.append(OnHandoffStoreHandler)
        handler_classes.append(OnComputeStoreHandler)
        self._store_handlers: dict[StoreTrigger, StoreTriggerHandler] = {
            handler_cls.trigger: handler_cls(self.config, self._events_tracker)
            for handler_cls in handler_classes
        }

    # Holds the manager lock for the current schedule(), or None between steps.
    # Read and written only by the scheduler thread, so it needs no
    # synchronization of its own. Declared on the class so an instance built
    # without __init__ still reads None rather than raising.
    _step_region: ExitStack | None = None

    # ------------------------------------------------------------------
    # Manager exclusion
    #
    # The manager is entered from the scheduler thread and, for managers that
    # run one, from a tier-poll thread. Manager methods never take their
    # own lock, so the exclusion boundary lives here. manager.lock may be a
    # no-op for a manager that needs no exclusion; it is entered regardless.
    #
    # schedule() holds one region for the whole step: opened by the first
    # manager-touching hook and closed at the end of build_connector_meta.
    # That span is required, not tidiness -- get_num_new_matched_tokens
    # establishes lookup() HITs that update_state_after_alloc later pins with
    # prepare_load(), and prepare_store() evicts after both. Releasing between
    # them would let another thread evict a block the step already treated as
    # present.
    #
    # Hooks outside schedule() use a short region instead, so the manager is
    # free while the engine runs the model.
    #
    # The step region outlives any single call, so it cannot be a with-block;
    # an ExitStack carries the entered lock from open to close instead.
    # ------------------------------------------------------------------

    def _acquire_step(self) -> None:
        """Open the step-scoped region. Idempotent within a step."""
        if self._step_region is not None:
            return
        region = ExitStack()
        region.enter_context(self.manager.lock)
        self._step_region = region

    def _release_step(self) -> None:
        """Close the step-scoped region, letting other threads in."""
        region = self._step_region
        if region is None:
            return
        self._step_region = None
        region.close()

    @contextmanager
    def _manager_locked(self) -> Generator[None]:
        """Hold the manager for the duration of one hook outside schedule().

        None of these hooks can legitimately run inside the step-scoped region:
        on_new_request comes from add_request, has_pending_push_work from the
        engine's liveness check before schedule(), the rest from output
        processing or a client RPC. So finding the region open here means a
        previous schedule() raised between _acquire_step() and the release in
        build_connector_meta. Recover instead of deadlocking.
        """
        if self._step_region is not None:
            logger.error(
                "Offloading manager lock was left held by a previous "
                "schedule(); releasing it before continuing."
            )
            self._release_step()
        with self.manager.lock:
            yield

    def _maybe_observe_lookup_async_delay(
        self, req_status: RequestOffloadState
    ) -> None:
        start_time = req_status.deferred_lookup_start_time
        if start_time is None:
            return
        req_status.deferred_lookup_start_time = None
        self._connector_stats.observe_histogram(
            _ConnectorMetricName.LOOKUP_ASYNC_DELAY,
            time.monotonic() - start_time,
        )

    def _generate_job_id(self) -> int:
        job_id = self._job_counter
        self._job_counter += 1
        return job_id

    def _remove_pending_job(self, job_id: int, block_ids: list[int] | None) -> None:
        for bid in block_ids or ():
            pending = self._block_id_to_pending_jobs[bid]
            pending.remove(job_id)
            if not pending:
                del self._block_id_to_pending_jobs[bid]

    def _remove_request(self, req_id: ReqId) -> None:
        """Remove the state of a request, and tell each store handler."""
        del self._req_status[req_id]
        for handler in self._store_handlers.values():
            handler.on_request_removed(req_id)

    def _maximal_prefix_lookup(
        self,
        keys: Iterable[OffloadKey],
        req_context: ReqContext,
        req: Request,
        group_config: GroupOffloadConfig,
        start_chunk_idx: int,
    ) -> int | None:
        """Return the number of consecutive offloaded chunks from the start,
        or None if the backend deferred a lookup."""
        hit_count = 0
        defer_lookup = False
        for local_idx, key in enumerate(keys):
            result = self.manager.lookup(key, req_context)
            match result:
                case LookupResult.HIT | LookupResult.HIT_PENDING:
                    self._events_tracker.record_lookup(
                        req,
                        group_config,
                        start_chunk_idx + local_idx,
                        key,
                    )
                    hit_count += 1
                    defer_lookup |= result is LookupResult.HIT_PENDING
                case LookupResult.RETRY:
                    # Don't break: keep scanning to let manager kick off
                    # async lookups (until a miss is detected).
                    defer_lookup = True
                case LookupResult.MISS:
                    break
        return hit_count if not defer_lookup else None

    def _sliding_window_lookup(
        self,
        keys: Sequence[OffloadKey],
        sliding_window_size: int,
        req_context: ReqContext,
        initial_window_size: int | None = None,
    ) -> int | None:
        """Return the end index (in `keys`) of the last run of
        `sliding_window_size` consecutive hits, scanning from the end.
        The first run may need a larger window for a partial rightmost chunk.
        Returns 0 on miss, None if the backend deferred a lookup."""
        defer_lookup = False
        pending_in_window = False
        consecutive_hits = 0
        required_window = initial_window_size or sliding_window_size
        for idx in range(len(keys) - 1, -1, -1):
            match self.manager.lookup(keys[idx], req_context):
                case LookupResult.HIT:
                    consecutive_hits += 1
                case LookupResult.HIT_PENDING:
                    # Block is in cache, just not readable yet — counts
                    # as hit for the consecutive streak. Don't break:
                    # keep scanning to let manager kick off async lookups.
                    pending_in_window = True
                    consecutive_hits += 1
                case LookupResult.RETRY:
                    # Block location uncertain — does not count as hit.
                    # Don't break: keep scanning to let manager kick off
                    # async lookups.
                    defer_lookup = True
                    consecutive_hits = 0
                    pending_in_window = False
                    required_window = sliding_window_size
                case LookupResult.MISS:
                    consecutive_hits = 0
                    # This gap rules out the incomplete window to its right.
                    pending_in_window = False
                    required_window = sliding_window_size
            if consecutive_hits == required_window:
                return (
                    None if defer_lookup or pending_in_window else idx + required_window
                )
        return None if defer_lookup or pending_in_window else consecutive_hits

    def _lookup_complete_chunks(
        self,
        req_status: RequestOffloadState,
        max_num_new_tokens: int | None = None,
    ) -> tuple[int | None, int | None]:
        """Find how many tokens beyond num_locally_computed_tokens can be loaded.

        Iterates full-attention groups first (prefix lookup), then sliding-window
        groups (suffix lookup). Each group may tighten max_hit_size_tokens, which
        can invalidate an earlier group's result, so the loop re-runs when that
        happens until num_hit_tokens converges.

        Returns the number of hit tokens (None to defer) and the hit boundary
        the full-attention groups proved before any sliding-window/Mamba group
        narrowed it (None if not reached).
        """
        num_computed_tokens = req_status.num_locally_computed_tokens
        max_hit_size_tokens: int = req_status.req.num_tokens
        if self._sliding_window_groups:
            # the last prompt token has to be recomputed to get the logprobs
            # for sliding window attention, we must reduce by 1 to make sure
            # we still have a hit after reduction. Tighter caps below are
            # already supported boundaries and are not reduced.
            max_hit_size_tokens -= 1
        if max_num_new_tokens is not None:
            max_hit_size_tokens = min(
                max_hit_size_tokens, num_computed_tokens + max_num_new_tokens
            )
        if req_status.max_load_tokens is not None:
            max_hit_size_tokens = min(
                max_hit_size_tokens,
                num_computed_tokens + req_status.max_load_tokens,
            )
        if self._sliding_window_groups and self._mamba_align_size is not None:
            # Constrain hit-window to the mamba block size.
            max_hit_size_tokens = round_down(
                max_hit_size_tokens, self._mamba_align_size
            )

        num_hit_tokens: int = 0
        defer_lookup = False
        lookup_groups = self._lookup_groups
        for state in req_status.group_states:
            state.lookup_hit_end_chunk_idx = 0

        # Tracks which legacy EAGLE groups already popped their volatile tail
        # in this convergence iteration. A non-EAGLE constraint resets it.
        eagle_verified: set[int] = set()
        full_attention_hit_boundary: int | None = None
        while lookup_groups:
            looked_up_sliding_window: bool = False
            full_attention_deferred = False
            looked_up_full_attention = False
            groups_iter = iter(lookup_groups)
            lookup_groups = ()
            for config_group_idx in groups_iter:
                group_config: GroupOffloadConfig = self.config.kv_group_configs[
                    config_group_idx
                ]
                group_state: RequestGroupState = req_status.group_states[
                    config_group_idx
                ]
                tokens_per_chunk = group_config.tokens_per_chunk
                offload_keys = group_state.offload_keys

                num_keyed_tokens = (
                    len(req_status.req.block_hashes) * self.config.tokens_per_hash
                )
                assert len(offload_keys) >= num_keyed_tokens // tokens_per_chunk

                apply_eagle_drop = (
                    group_config.is_eagle_group and not self.use_lookahead_block_hashes
                )
                is_eagle_unverified = (
                    apply_eagle_drop and config_group_idx not in eagle_verified
                )

                # Constrain to a chunk-aligned boundary for this group.
                max_hit_size_tokens = min(
                    max_hit_size_tokens, len(offload_keys) * tokens_per_chunk
                )
                if req_status.max_load_tokens is not None:
                    max_hit_size_tokens = round_down(
                        max_hit_size_tokens, tokens_per_chunk
                    )
                if max_hit_size_tokens - num_computed_tokens < tokens_per_chunk:
                    # We can only load less than a chunk, so skip.
                    return 0, full_attention_hit_boundary

                sliding_window_size_in_chunks = (
                    group_config.sliding_window_size_in_chunks
                )
                # Zero cannot pin a junction, so a later confirmed hit may replace it.
                if (
                    sliding_window_size_in_chunks is not None
                    and not full_attention_hit_boundary
                    and not full_attention_deferred
                    and (looked_up_full_attention or not self._full_attention_groups)
                ):
                    # Groups before the first sparse group are full attention.
                    full_attention_hit_boundary = (
                        max_hit_size_tokens if self._full_attention_groups else 0
                    )

                # For legacy EAGLE groups, query one extra chunk that will be
                # popped. Widening applies to every group type: without it, the
                # pop below shrinks max_hit_size_tokens past what was queried,
                # which can push the confirmed boundary under a coarser
                # sibling group's chunk granularity and zero the whole
                # request's hit (issue #52735).
                query_max = max_hit_size_tokens
                if is_eagle_unverified:
                    query_max = min(
                        max_hit_size_tokens + tokens_per_chunk,
                        len(offload_keys) * tokens_per_chunk,
                    )

                num_chunks = min(cdiv(query_max, tokens_per_chunk), len(offload_keys))
                start_chunk_idx = num_computed_tokens // tokens_per_chunk
                offload_keys = offload_keys[start_chunk_idx:num_chunks]

                # end index (in the sliced offload_keys) up to which we
                # have backend-confirmed hits
                num_hit_chunks: int | None
                if sliding_window_size_in_chunks is None:
                    looked_up_full_attention = True
                    num_hit_chunks = self._maximal_prefix_lookup(
                        offload_keys,
                        req_status.req_context,
                        req_status.req,
                        group_config,
                        start_chunk_idx,
                    )
                    full_attention_deferred |= num_hit_chunks is None
                    group_state.lookup_hit_end_chunk_idx = start_chunk_idx + (
                        num_hit_chunks or 0
                    )
                else:
                    required_window = sliding_window_size_in_chunks
                    if is_eagle_unverified:
                        required_window += 1
                    candidate_end = min(
                        max_hit_size_tokens,
                        (num_chunks - int(is_eagle_unverified)) * tokens_per_chunk,
                    )
                    initial_window = group_config.load_window_size_in_chunks(
                        candidate_end
                    )
                    assert initial_window is not None
                    num_hit_chunks = self._sliding_window_lookup(
                        offload_keys,
                        required_window,
                        req_status.req_context,
                        initial_window + int(is_eagle_unverified),
                    )
                if num_hit_chunks == 0:
                    return 0, full_attention_hit_boundary

                if num_hit_chunks is None:
                    defer_lookup = True
                else:
                    if is_eagle_unverified:
                        num_hit_chunks -= 1
                        eagle_verified.add(config_group_idx)

                    max_hit_size_tokens = min(
                        max_hit_size_tokens,
                        tokens_per_chunk * (start_chunk_idx + num_hit_chunks),
                    )

                new_num_hit_tokens = max_hit_size_tokens - num_computed_tokens
                if new_num_hit_tokens < tokens_per_chunk:
                    # We can only load less than a chunk, so skip.
                    return 0, full_attention_hit_boundary

                if new_num_hit_tokens < num_hit_tokens:
                    if not apply_eagle_drop:
                        eagle_verified.clear()
                    if defer_lookup:
                        # make another iteration on all groups to check
                        # if we still need to defer lookup
                        defer_lookup = False
                        lookup_groups = self._lookup_groups
                    elif looked_up_sliding_window and not lookup_groups:
                        # we need another iteration to confirm previously looked up
                        # sliding window works with the new_num_hit_tokens
                        lookup_groups = self._sliding_window_groups

                looked_up_sliding_window |= sliding_window_size_in_chunks is not None
                num_hit_tokens = new_num_hit_tokens

        if defer_lookup:
            logger.debug(
                "Offloading manager delayed request %s as backend requested",
                req_status.req.request_id,
            )
            return None, full_attention_hit_boundary

        # Possibly delay the request if any hit chunk is already being loaded.
        if self._chunks_being_loaded:
            for group_config, group_state in zip(
                self.config.kv_group_configs, req_status.group_states
            ):
                tokens_per_chunk = group_config.tokens_per_chunk
                sliding_window_size_in_chunks = group_config.load_window_size_in_chunks(
                    num_computed_tokens + num_hit_tokens
                )
                offload_keys = group_state.offload_keys
                num_chunks = cdiv(
                    num_computed_tokens + num_hit_tokens, tokens_per_chunk
                )
                start_chunk_idx = num_computed_tokens // tokens_per_chunk
                offload_keys = offload_keys[start_chunk_idx:num_chunks]
                if sliding_window_size_in_chunks is not None:
                    offload_keys = offload_keys[-sliding_window_size_in_chunks:]
                if any(key in self._chunks_being_loaded for key in offload_keys):
                    # Hit chunks are being loaded, so delay the request.
                    logger.debug(
                        "Delaying request %s since some of its"
                        " chunks are already being loaded",
                        req_status.req.request_id,
                    )
                    return None, full_attention_hit_boundary

        logger.debug(
            "Request %s hit %s offloaded tokens after %s GPU hit tokens",
            req_status.req.request_id,
            num_hit_tokens,
            num_computed_tokens,
        )

        return num_hit_tokens, full_attention_hit_boundary

    def _lookup(
        self,
        req_status: RequestOffloadState,
        max_num_new_tokens: int | None = None,
    ) -> int | None:
        complete_hit, full_attention_hit_boundary = self._lookup_complete_chunks(
            req_status, max_num_new_tokens
        )
        num_hit_tokens = self._lookup_partial_tail_hit(
            req_status, complete_hit, full_attention_hit_boundary, max_num_new_tokens
        )
        if num_hit_tokens is not None and full_attention_hit_boundary is not None:
            self._maybe_register_shared_prefix_junction(
                req_status, full_attention_hit_boundary, num_hit_tokens
            )
        return num_hit_tokens

    def _lookup_partial_tail_hit(
        self,
        req_status: RequestOffloadState,
        complete_hit: int | None,
        full_attention_hit_boundary: int | None,
        max_num_new_tokens: int | None,
    ) -> int | None:
        """Extend a complete-chunk hit with a partial tail, if supported."""
        req_status.partial_tail_boundary = None
        if complete_hit is None or not self.config.supports_partial_tail:
            return complete_hit

        local_tokens = req_status.num_locally_computed_tokens
        complete_boundary = local_tokens + complete_hit
        tokens_per_hash = self.config.tokens_per_hash
        block_size = self.config.partial_tail_block_size
        max_eagle_proof_margin = self.config.max_eagle_proof_margin
        # A recurrent group holds state only at a producer's tail, so the
        # full-attention prefix, not the complete hit, bounds the search.
        anchor_boundary = max(complete_boundary, full_attention_hit_boundary or 0)
        anchor_pending = anchor_boundary > complete_boundary and any(
            key in (self._chunks_being_loaded or ())
            for group_config, group_state in zip(
                self.config.kv_group_configs, req_status.group_states
            )
            if not group_config.requires_cow_source
            for key in group_state.offload_keys[
                complete_boundary // block_size : anchor_boundary // block_size
            ]
        )
        if anchor_pending:
            anchor_boundary = complete_boundary
        # A partial tail carries its own EAGLE proof, so it may also lie in
        # the block after the chunk that the complete-chunk lookup pops.
        num_search_blocks = 2 if max_eagle_proof_margin else 1
        block_end = anchor_boundary + num_search_blocks * block_size
        max_boundary = min(
            req_status.req.num_prompt_tokens - 1,
            block_end - 1,
            len(req_status.req.block_hashes) * tokens_per_hash - max_eagle_proof_margin,
        )
        if max_num_new_tokens is not None:
            max_boundary = min(max_boundary, local_tokens + max_num_new_tokens)
        if req_status.max_load_tokens is not None:
            max_boundary = min(
                max_boundary,
                local_tokens + req_status.max_load_tokens,
            )
        max_boundary = round_down(max_boundary, tokens_per_hash)
        if max_boundary <= complete_boundary:
            return complete_hit

        # A boundary past the block of anchor_boundary also loads that
        # block's complete chunk. Look it up once, and only if it is needed.
        # Partial tails use one block per chunk, so a block index is a chunk
        # index.
        bridge_chunk_idx = anchor_boundary // block_size
        bridge_boundary = (bridge_chunk_idx + 1) * block_size
        # Bridge keys that the complete-chunk lookup did not confirm.
        unconfirmed_bridge_keys: list[tuple[GroupOffloadConfig, OffloadKey]] = []
        pending = anchor_pending
        if max_boundary >= bridge_boundary:
            for group_config, group_state in zip(
                self.config.kv_group_configs, req_status.group_states
            ):
                if group_config.requires_cow_source:
                    continue
                key = group_state.offload_keys[bridge_chunk_idx]
                pending |= key in (self._chunks_being_loaded or ())
                # An EAGLE group always confirmed the chunk that it popped.
                if group_state.lookup_hit_end_chunk_idx <= bridge_chunk_idx:
                    unconfirmed_bridge_keys.append((group_config, key))
            for _, key in unconfirmed_bridge_keys:
                result = self.manager.lookup(key, req_status.req_context)
                if result is LookupResult.MISS:
                    max_boundary = bridge_boundary - tokens_per_hash
                    break
                pending |= result is not LookupResult.HIT
        bridge_pending = pending
        # A tail of this request may lie below a sibling's longer full-attention
        # prefix. A block-aligned boundary there is a complete chunk, which the
        # complete-chunk lookup already missed.
        for boundary in range(max_boundary, complete_boundary, -tokens_per_hash):
            if boundary <= anchor_boundary and boundary % block_size == 0:
                continue
            loads_bridge = boundary >= bridge_boundary
            hit, boundary_pending = self._lookup_partial_tail(
                req_status, boundary, loads_bridge and bridge_pending
            )
            if hit:
                # The complete-chunk lookup recorded the confirmed bridge keys.
                if loads_bridge:
                    for group_config, key in unconfirmed_bridge_keys:
                        self._events_tracker.record_lookup(
                            req_status.req, group_config, bridge_chunk_idx, key
                        )
                req_status.partial_tail_boundary = boundary
                return boundary - local_tokens
            pending |= boundary_pending

        if pending and complete_hit == 0:
            return None
        return complete_hit

    def _lookup_partial_tail(
        self,
        req_status: RequestOffloadState,
        boundary: int,
        bridge_pending: bool,
    ) -> tuple[bool, bool]:
        """Look up every key a partial-tail hit at `boundary` loads.

        `bridge_pending` is True if a complete chunk that the hit also loads
        is not ready yet.
        Returns `(hit, pending)`. `hit` is True only if every key is ready.
        `pending` is True if no required key missed and a required key or
        bridge chunk is not ready yet.
        """
        req = req_status.req
        req_context = req_status.req_context
        pending = bridge_pending
        boundary_keys: list[tuple[GroupOffloadConfig, int, OffloadKey]] = []
        for config_group_idx, group_config in enumerate(self.config.kv_group_configs):
            key_tokens = self.config.partial_tail_key_tokens(config_group_idx, boundary)
            key = req_status.register_boundary_key(group_config.group_idx, key_tokens)
            # The offload manager can report HIT as soon as the source chunk
            # is ready, while its earlier GPU load is still in flight.  Do
            # not schedule the same partial-tail transfer twice.
            pending |= key in (self._chunks_being_loaded or ())
            result = self.manager.lookup(key, req_context)
            if result is LookupResult.MISS:
                return False, False
            pending |= result is not LookupResult.HIT
            boundary_keys.append((group_config, key_tokens, key))

        if pending:
            return False, True
        for group_config, key_tokens, key in boundary_keys:
            self._events_tracker.record_partial_lookup(
                req, group_config, key_tokens, key
            )
        return True, False

    def _maybe_register_shared_prefix_junction(
        self, req_status: RequestOffloadState, boundary: int, num_hit_tokens: int
    ) -> None:
        """Pin a shared-prefix junction where a sparse group cut the hit, as
        `get_computed_blocks` does for a local hit, so the recompute retains
        the sliding-window/Mamba state at the full-attention hit boundary."""
        if self.config.retention_interval is None:
            return
        if boundary <= req_status.num_locally_computed_tokens + num_hit_tokens:
            return
        request = req_status.req
        # Never lower a junction the local prefix cache already set.
        if boundary > request.shared_prefix_boundary:
            request.shared_prefix_boundary = boundary

    def on_new_request(self, request: Request) -> None:
        """Called when a new request is added to the scheduler."""
        req_context = _create_req_context(request, self)
        req_status = RequestOffloadState(
            config=self.config,
            req=request,
            req_context=req_context,
            offloading_context=RequestOffloadingContext(),
        )
        self._req_status[request.request_id] = req_status
        try:
            with self._manager_locked():
                req_status.offloading_context = self.manager.on_new_request(req_context)
        except Exception:
            del self._req_status[request.request_id]
            raise
        self._events_tracker.on_new_request(
            req_context, request, self.config.kv_group_configs
        )

    def get_num_new_matched_tokens(
        self,
        request: Request,
        num_computed_tokens: int,
        max_num_new_tokens: int | None = None,
    ) -> tuple[int | None, bool]:
        """Get number of new tokens that can be loaded beyond the
        num_computed_tokens.

        Args:
            request (Request): the request object.
            num_computed_tokens (int): the number of locally
                computed tokens for this request
            max_num_new_tokens (int | None): cap on the number of tokens that
                may be loaded beyond `num_computed_tokens`, if any.

        Returns:
            A tuple with the following elements:
                - The number of tokens that can be loaded beyond what is
                  already computed.
                  If None, it means that the connector needs more time to
                  determine the number of matched tokens, and the scheduler
                  should query for this request again later.
                - `True` if tokens will be loaded asynchronously
                  (between scheduler steps).

        """
        # First manager-touching hook of a step: opens the step-scoped region,
        # closed at the end of build_connector_meta.
        self._acquire_step()

        req_status = self._req_status[request.request_id]
        for group_state in req_status.group_states:
            group_state.block_ids.clear()
        for handler in self._store_handlers.values():
            handler.on_request_restarted(request.request_id)

        if req_status.transfer_jobs:
            logger.debug(
                "Delaying request %s since it still has in-flight transfers",
                request.request_id,
            )
            return None, False

        if request.resumable and self.use_lookahead_block_hashes:
            # Core may have recomputed block hashes after preemption.
            for group_state in req_status.group_states:
                group_state.offload_keys.clear()
        req_status.update_offload_keys()
        req_status.num_locally_computed_tokens = num_computed_tokens

        num_hit_tokens: int | None
        if (
            request.skip_reading_external_kv
            or req_status.load_failed
            or not self._offloading_enabled
        ):
            num_hit_tokens = 0
        else:
            lookup_start = time.monotonic()
            num_hit_tokens = self._lookup(req_status, max_num_new_tokens)
            self._connector_stats.observe_histogram(
                _ConnectorMetricName.LOOKUP_SYNC_DELAY,
                time.monotonic() - lookup_start,
            )
            if num_hit_tokens is None:
                if req_status.deferred_lookup_start_time is None:
                    req_status.deferred_lookup_start_time = lookup_start
            else:
                self._maybe_observe_lookup_async_delay(req_status)
        req_status.update_num_hit_chunks(num_computed_tokens + (num_hit_tokens or 0))

        sync_load = self._reserve_sync_load(
            req_status, num_computed_tokens, num_hit_tokens
        )
        return num_hit_tokens, bool(num_hit_tokens) and not sync_load

    def _reserve_sync_load(
        self,
        req_status: RequestOffloadState,
        num_computed_tokens: int,
        num_hit_tokens: int | None,
    ) -> bool:
        """Reserve the step's sync-load budget for a hit, if it fits."""
        req_id = req_status.req.request_id
        self._current_batch_sync_loads.discard(req_id)
        # Keep at least one token to compute, or the request can't be scheduled.
        if (
            not self.config.sync_load_budget_bytes
            or not num_hit_tokens
            or num_computed_tokens + num_hit_tokens >= req_status.req.num_tokens
        ):
            return False
        # Overestimates sliding-window and recurrent groups, which load less.
        num_bytes = num_hit_tokens * self.config.kv_bytes_per_token
        budget = self.config.sync_load_budget_bytes
        if self._current_batch_sync_load_bytes + num_bytes > budget:
            return False
        self._current_batch_sync_load_bytes += num_bytes
        self._current_batch_sync_loads.add(req_id)
        return True

    def get_external_cache_hit_sources(
        self,
        request: Request,
        num_external_tokens: int,
    ) -> dict[CacheHitSource, int]:
        """Split the accepted external hit by the tier each loaded key came from.

        Attention keys cover only their loaded token ranges; recurrent state
        covers the whole reused prefix. Overlaps report the outermost tier.
        Uncovered tokens, including omitted sliding-window history, are
        `external_unspecified`.
        """
        sources: dict[CacheHitSource, int] = {}
        if num_external_tokens == 0:
            return sources
        req_status = self._req_status[request.request_id]
        start = req_status.num_locally_computed_tokens
        end = start + num_external_tokens

        events: list[tuple[int, int, CacheHitSource]] = [
            (end, 0, CacheHitSource.EXTERNAL_UNSPECIFIED)
        ]
        for lo, hi, key in req_status.load_key_ranges:
            lo, hi = max(start, lo), min(end, hi)
            if lo >= hi:
                continue
            source = CacheHitSource(
                self.manager.get_load_source(key, req_status.req_context)
            )
            events.extend(((lo, 1, source), (hi, -1, source)))

        # Sweep endpoints instead of rescanning every range for each interval.
        active: Counter[CacheHitSource] = Counter()
        position = start
        for boundary, delta, source in sorted(events):
            if boundary > position:
                tiers = [tier for tier, count in active.items() if count > 0]
                tier = (
                    CacheHitSource.outermost(tiers)
                    if tiers
                    else CacheHitSource.EXTERNAL_UNSPECIFIED
                )
                sources[tier] = sources.get(tier, 0) + boundary - position
            active[source] += delta
            position = boundary
        return sources

    def _num_prefix_blocks(self, group_idx: int, local_prefix_tokens: int) -> int:
        """Return the number of shared prefix blocks in a full-attention group."""
        group_config = self._group_configs_by_id.get(group_idx)
        if (
            group_config is None
            or group_config.sliding_window_size_in_chunks is not None
        ):
            return 0
        return local_prefix_tokens // group_config.tokens_per_block

    def update_state_after_alloc(
        self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int
    ):
        req_status = self._req_status[request.request_id]
        # Include allocations made by other connectors in a MultiConnector.
        # Full-attention prefix hits were touched, not allocated.
        num_locally_computed_tokens = req_status.num_locally_computed_tokens
        for group_idx, group_blocks in enumerate(blocks.blocks):
            num_prefix_blocks = self._num_prefix_blocks(
                group_idx, num_locally_computed_tokens
            )
            self._current_batch_allocated_block_ids.update(
                block.block_id
                for block in group_blocks[num_prefix_blocks:]
                if block.block_id != 0
            )

        if num_external_tokens == 0:
            return

        # prepare_load() below pins keys that get_num_new_matched_tokens()
        # already saw as HIT, so both must sit inside the same region.
        self._acquire_step()

        req_status.loaded_from_offload = True
        sync_load = request.request_id in self._current_batch_sync_loads

        num_cached_tokens = num_locally_computed_tokens + num_external_tokens
        partial_tail_boundary = req_status.partial_tail_boundary
        if partial_tail_boundary is not None:
            assert partial_tail_boundary == num_cached_tokens

        keys_to_load: list[OffloadKey] = []
        req_status.load_key_ranges.clear()
        dst_block_ids: list[int] = []
        # per group
        group_sizes: list[int] = []
        block_indices: list[int] = []
        for config_group_idx, (group_config, group_state) in enumerate(
            zip(self.config.kv_group_configs, req_status.group_states)
        ):
            group_blocks = blocks.blocks[group_config.group_idx]
            tokens_per_block = group_config.tokens_per_block
            tokens_per_chunk = group_config.tokens_per_chunk
            offload_keys = group_state.offload_keys
            num_gpu_blocks = cdiv(num_cached_tokens, tokens_per_block)

            assert len(group_blocks) >= num_gpu_blocks
            # ``load_start_gpu_block_idx``: the index in ``group_blocks`` where the
            # load region begins -- a slice bound, not a count of computed blocks.
            # Scan from the computed boundary, not 0: sparse groups (Mamba,
            # SWA) legitimately hold non-null unhashed blocks below it.
            # Skip nulls (sentinel / out-of-retention padding).
            first_fresh_gpu_block_idx = cdiv(
                num_locally_computed_tokens, tokens_per_block
            )
            load_start_gpu_block_idx = num_gpu_blocks
            for i in range(first_fresh_gpu_block_idx, num_gpu_blocks):
                block = group_blocks[i]
                # Sync-load blocks are hashed at allocation: match by position.
                if not block.is_null and (sync_load or block.block_hash is None):
                    load_start_gpu_block_idx = i
                    break

            assert num_locally_computed_tokens % tokens_per_block == 0
            num_pending_gpu_blocks = num_gpu_blocks - load_start_gpu_block_idx

            if group_config.sliding_window_size_in_chunks is not None:
                assert (
                    num_pending_gpu_blocks
                    <= group_config.sliding_window_size_in_chunks
                    * self.config.blocks_per_chunk
                    + 1
                )

            num_chunks = cdiv(num_cached_tokens, tokens_per_chunk)
            if num_pending_gpu_blocks:
                start_chunk_idx = (
                    load_start_gpu_block_idx // self.config.blocks_per_chunk
                )
                end_chunk_idx = num_chunks - (partial_tail_boundary is not None)
                assert len(offload_keys) >= end_chunk_idx
                group_keys_to_load = offload_keys[start_chunk_idx:end_chunk_idx]
                if partial_tail_boundary is not None:
                    group_keys_to_load.append(
                        req_status.register_boundary_key(
                            group_config.group_idx,
                            self.config.partial_tail_key_tokens(
                                config_group_idx, partial_tail_boundary
                            ),
                        )
                    )
                keys_to_load.extend(group_keys_to_load)
                if issubclass(group_config.manager_cls, MambaManager):
                    req_status.load_key_ranges.extend(
                        (num_locally_computed_tokens, num_cached_tokens, key)
                        for key in group_keys_to_load
                    )
                else:
                    for chunk_idx, key in enumerate(
                        group_keys_to_load, start_chunk_idx
                    ):
                        range_start = max(
                            load_start_gpu_block_idx * tokens_per_block,
                            chunk_idx * tokens_per_chunk,
                        )
                        range_end = min(
                            num_cached_tokens, (chunk_idx + 1) * tokens_per_chunk
                        )
                        req_status.load_key_ranges.append((range_start, range_end, key))

            dst_block_ids.extend(
                block.block_id
                for block in group_blocks[load_start_gpu_block_idx:num_gpu_blocks]
            )
            group_sizes.append(num_pending_gpu_blocks)
            block_indices.append(load_start_gpu_block_idx)

        for handler in self._store_handlers.values():
            handler.on_load_scheduled(req_status, num_cached_tokens)

        src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context)
        dst_spec = GPULoadStoreSpec(
            dst_block_ids, group_sizes=group_sizes, block_indices=block_indices
        )

        load_job_id = self._generate_job_id()
        self._current_batch_load_jobs[load_job_id] = TransferJob(
            req_id=request.request_id,
            src_spec=src_spec,
            dst_spec=dst_spec,
            sync=sync_load,
        )
        # a load can only be issued when no other jobs are pending.
        assert not req_status.transfer_jobs
        req_status.transfer_jobs.add(load_job_id)
        self._job_statuses[load_job_id] = TransferJobStatus(
            req_id=request.request_id,
            pending_count=self.config.num_workers,
            keys=tuple(dict.fromkeys(keys_to_load)),
            is_store=False,
            is_sync_load=sync_load,
        )

        # A sync load completes before this step's forward pass.
        if self._chunks_being_loaded is not None and not sync_load:
            self._chunks_being_loaded.update(keys_to_load)
        req_status.partial_tail_boundary = None

    def _update_req_states(self, scheduler_output: SchedulerOutput) -> None:
        """Update request states from the Scheduler's output."""
        # new_block_ids_end[req_id][i] = end of pre-existing block_ids for
        # the i-th sliding window group (before this step's extend).
        # Used to detect sliding window blocks that got re-allocated.
        new_block_ids_end: dict[str, tuple[int, ...]] = {}

        for req_id, block_id_groups, is_full_table in yield_req_data(scheduler_output):
            req_status = self._req_status[req_id]
            req_status.update_offload_keys()

            if is_full_table:
                for group_state in req_status.group_states:
                    group_state.block_ids.clear()

            if block_id_groups:
                if self._sliding_window_groups:
                    new_block_ids_end[req_id] = tuple(
                        len(req_status.group_states[config_group_idx].block_ids)
                        for config_group_idx in self._sliding_window_groups
                    )
                req_status.update_block_id_groups(block_id_groups)
                local_prefix_tokens = 0
                if is_full_table:
                    local_prefix_tokens = req_status.num_locally_computed_tokens
                    # Core can shorten or discard the hit after connector lookup.
                    # Replay instead moves the execution start without dropping FA.
                    if not req_status.req.replay_start:
                        local_prefix_tokens = min(
                            local_prefix_tokens, req_status.req.num_computed_tokens
                        )
                for group_idx, group_blocks in enumerate(block_id_groups):
                    num_prefix_blocks = self._num_prefix_blocks(
                        group_idx, local_prefix_tokens
                    )
                    self._current_batch_allocated_block_ids.update(
                        bid for bid in group_blocks[num_prefix_blocks:] if bid != 0
                    )

        for copy in scheduler_output.kv_cache_block_copies or ():
            self._current_batch_allocated_block_ids.add(copy.dst_block_id)

        # Zero out stale block_ids in sliding window groups' pending-store
        # positions. Only sliding window groups can have stale entries (blocks
        # freed by remove_skipped_blocks then reallocated). Only positions in
        # [first pending chunk * bsf, end) need checking, where the first
        # pending chunk is the lowest over the handlers and end is the
        # pre-extend length: earlier positions were already offloaded, later
        # ones are fresh allocations from this step.
        if self._sliding_window_groups and self._current_batch_allocated_block_ids:
            blocks_per_chunk = self.config.blocks_per_chunk
            for req_id, req_status in self._req_status.items():
                ends = new_block_ids_end.get(req_id)
                for i, config_group_idx in enumerate(self._sliding_window_groups):
                    group_state = req_status.group_states[config_group_idx]
                    first_live_idxs = (
                        handler.first_live_chunk_idx(req_status, config_group_idx)
                        for handler in self._store_handlers.values()
                    )
                    start = (
                        min(
                            (idx for idx in first_live_idxs if idx is not None),
                            default=0,
                        )
                        * blocks_per_chunk
                    )
                    end = ends[i] if ends is not None else len(group_state.block_ids)
                    for j in range(start, end):
                        if (
                            group_state.block_ids[j]
                            in self._current_batch_allocated_block_ids
                        ):
                            group_state.block_ids[j] = 0

    def _track_store_job(
        self,
        req_id: ReqId,
        keys: Collection[OffloadKey],
        *,
        trigger: StoreTrigger,
        fenced_block_ids: list[int] | None,
        deferred_fence_block_ids: list[int] | None,
    ) -> int:
        """Give a store job a job ID, track it, and return the ID."""
        job_status = TransferJobStatus(
            req_id=req_id,
            pending_count=self.config.num_workers,
            keys=tuple(dict.fromkeys(keys)),
            is_store=True,
            deferred_fence_block_ids=deferred_fence_block_ids,
            fenced_block_ids=fenced_block_ids,
            trigger=trigger,
        )
        job_id = self._generate_job_id()
        req_status = self._req_status[req_id]
        # a store can only be issued when no load is pending.
        if req_status.transfer_jobs:
            any_jid = next(iter(req_status.transfer_jobs))
            any_job = self._job_statuses[any_jid]
            assert any_job.is_store or any_job.is_sync_load
        req_status.transfer_jobs.add(job_id)

        # Watch sliding window blocks as they may get evicted
        # before the request finishes
        for bid in job_status.fenced_block_ids or ():
            self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)

        self._job_statuses[job_id] = job_status
        logger.debug(
            "Request %s offloading %d chunks (job %d)",
            job_status.req_id,
            len(job_status.keys),
            job_id,
        )

        if req_status.req.is_finished():
            # Register non-sliding-window blocks for flush detection.
            for bid in job_status.deferred_fence_block_ids or ():
                self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
                if bid in self._current_batch_allocated_block_ids:
                    self._current_batch_jobs_to_flush.add(job_id)
        return job_id

    def build_connector_meta(
        self, scheduler_output: SchedulerOutput
    ) -> KVConnectorMetadata:
        # A step that scheduled nothing reaches build_connector_meta without
        # any earlier manager touch, so open the region here too.
        self._acquire_step()
        try:
            return self._build_connector_meta(scheduler_output)
        finally:
            # The step is over: hand the manager back.
            self._release_step()

    def _build_connector_meta(
        self, scheduler_output: SchedulerOutput
    ) -> KVConnectorMetadata:
        # Every step reaches here, including the empty steps a held fence
        # keeps the engine taking when no request is left.
        if (
            self._reset_fence
            and time.monotonic() - self._reset_fence_since > _RESET_FENCE_WARN_S
        ):
            logger.warning_once(
                "KV offloading: workers have not reported %d jobs a cache "
                "reset flushed for over %d s",
                len(self._reset_fence),
                _RESET_FENCE_WARN_S,
            )
        self._update_req_states(scheduler_output)
        schedule_end_context = ScheduleEndContext(
            new_req_ids=[req.req_id for req in scheduler_output.scheduled_new_reqs],
            preempted_req_ids=scheduler_output.preempted_req_ids or (),
        )

        # Flush jobs for preempted requests.
        for req_id in scheduler_output.preempted_req_ids or ():
            req_status = self._req_status.get(req_id)
            if req_status is None:
                continue
            # A cache reset can preempt and reload a request in the same step.
            # These new loads have not been submitted to the worker yet.
            jobs_to_flush = (
                req_status.transfer_jobs - self._current_batch_load_jobs.keys()
            )
            if not jobs_to_flush:
                continue
            any_jid = next(iter(jobs_to_flush))
            any_job = self._job_statuses[any_jid]
            assert any_job.is_store or any_job.is_sync_load
            self._current_batch_jobs_to_flush.update(jobs_to_flush)

        # Flush jobs that contain re-allocated blocks.
        if (
            self._block_id_to_pending_jobs
            and not self._block_id_to_pending_jobs.keys().isdisjoint(
                self._current_batch_allocated_block_ids
            )
        ):
            self._current_batch_jobs_to_flush.update(
                jid
                for bid in self._current_batch_allocated_block_ids
                if bid in self._block_id_to_pending_jobs
                for jid in self._block_id_to_pending_jobs[bid]
            )

        store_jobs: dict[int, TransferJob] = {}
        if self._offloading_enabled:
            # Chunks skipped while offloading was off are still offered:
            # stored indices have not moved.
            for handler in self._store_handlers.values():
                sink = _StoreJobSink(self, handler.trigger)
                handler.build_store_jobs(scheduler_output, self._req_status, sink)
                store_jobs |= sink.jobs
        meta = OffloadingConnectorMetadata(
            load_jobs=self._current_batch_load_jobs,
            store_jobs=store_jobs,
            jobs_to_flush=self._current_batch_jobs_to_flush,
            offloading_enabled=self._offloading_enabled,
        )

        # All prepare_store calls for finished requests have been issued.
        # Signal on_request_finished and clean up state where possible.
        for req_id in scheduler_output.finished_req_ids or ():
            req_status = self._req_status.get(req_id)
            if req_status is None:
                continue
            req_status.finished_signaled = True
            self.manager.on_request_finished(req_status.req_context)
            if not req_status.transfer_jobs:
                self._remove_request(req_id)
        self._current_batch_load_jobs = {}
        self._current_batch_sync_loads = set()
        self._current_batch_sync_load_bytes = 0
        self._current_batch_jobs_to_flush = set()
        self._current_batch_allocated_block_ids = set()

        # Last manager call of the step, after every prepare_store and
        # on_request_finished above. on_schedule_end is documented as running at
        # the end of the step, and callers rely on that to bound the step's
        # exclusive access to the manager.
        self.manager.on_schedule_end(schedule_end_context)
        return meta

    def has_pending_push_work(self) -> bool:
        """Whether the engine must keep stepping.

        While True, build_connector_meta() and update_connector_output()
        continue to be called even when no requests are scheduled.

        Must use a short region, never the step-scoped one. The engine asks this
        before schedule() and returns early when the answer is False, so a
        step-scoped acquire here would never reach build_connector_meta's
        release and would hold the manager for good -- which is precisely the
        state an idle engine settles into.

        A cache reset fence also keeps it stepping: the flush must reach the
        workers and their reports come back before on_reset_flushed().
        """
        if self._job_statuses or self._reset_fence:
            return True
        with self._manager_locked():
            return self.manager.has_pending_work()

    def update_connector_output(self, connector_output: KVConnectorOutput):
        """Update KVConnector state from worker-side connectors output.

        Args:
            connector_output (KVConnectorOutput): the worker-side
                connectors output.

        """
        with self._manager_locked():
            self._update_connector_output(connector_output)

    def _update_connector_output(self, connector_output: KVConnectorOutput):
        meta = connector_output.kv_connector_worker_meta
        if not isinstance(meta, OffloadingWorkerMetadata):
            assert meta is None
            meta = OffloadingWorkerMetadata()
        if not self._offloading_enabled and meta.ready_ranks:
            self._ready_ranks |= meta.ready_ranks
            if len(self._ready_ranks) >= self.config.num_workers:
                self._offloading_enabled = True
                logger.info(
                    "KV offloading enabled: all %d workers ready",
                    self.config.num_workers,
                )
        if not meta.transfer_stats.is_empty():
            transfer_stats = OffloadingConnectorStats()
            if not meta.transfer_stats.load.is_empty():
                transfer_stats.increase_counter(
                    _TransferMetricName.LOAD_BYTES,
                    meta.transfer_stats.load.bytes,
                )
                transfer_stats.increase_counter(
                    _TransferMetricName.LOAD_TIME,
                    meta.transfer_stats.load.time,
                )
                for size in meta.transfer_stats.load.sizes:
                    transfer_stats.observe_histogram(
                        _TransferMetricName.LOAD_SIZE, size
                    )
            if not meta.transfer_stats.store.is_empty():
                transfer_stats.increase_counter(
                    _TransferMetricName.STORE_BYTES,
                    meta.transfer_stats.store.bytes,
                )
                transfer_stats.increase_counter(
                    _TransferMetricName.STORE_TIME,
                    meta.transfer_stats.store.time,
                )
                for size in meta.transfer_stats.store.sizes:
                    transfer_stats.observe_histogram(
                        _TransferMetricName.STORE_SIZE, size
                    )
            self._connector_stats.aggregate(transfer_stats)

        for job_id, count in meta.completed_jobs.items():
            assert count > 0
            if job_id < self._stale_job_threshold:
                if job_id in self._reset_fence:
                    self._reset_fence[job_id] -= count
                    if self._reset_fence[job_id] <= 0:
                        del self._reset_fence[job_id]
                        if not self._reset_fence:
                            self.manager.on_reset_flushed()
                            logger.info(
                                "KV offloading: workers reported every job "
                                "the cache reset flushed"
                            )
                logger.debug(
                    "Skipping stale completed job %d (pre-reset counter: %d)",
                    job_id,
                    self._stale_job_threshold,
                )
                continue
            job_status = self._job_statuses[job_id]
            if job_id in meta.failed_jobs:
                job_status.failed = True
            job_status.pending_count -= count
            if job_status.pending_count > 0:
                continue
            assert job_status.pending_count == 0

            req_status = self._req_status[job_status.req_id]
            if job_status.is_store:
                if job_status.failed:
                    logger.warning(
                        "KV offload store job %d failed; dropping %d chunk(s) "
                        "from the CPU cache rather than publishing them.",
                        job_id,
                        len(job_status.keys),
                    )
                assert job_status.trigger is not None
                self.manager.complete_store(
                    job_status.keys,
                    req_status.req_context,
                    trigger=job_status.trigger,
                    success=not job_status.failed,
                )
            else:
                if job_status.failed:
                    logger.warning(
                        "KV offload load job %d failed; request %s will not "
                        "load from the offloaded cache again.",
                        job_id,
                        job_status.req_id,
                    )
                    req_status.load_failed = True
                self.manager.complete_load(job_status.keys, req_status.req_context)
                if self._chunks_being_loaded:
                    self._chunks_being_loaded.difference_update(job_status.keys)
            if self._block_id_to_pending_jobs:
                # Sliding window blocks are tracked from store creation
                # and must be cleaned up unconditionally.
                self._remove_pending_job(job_id, job_status.fenced_block_ids)
                # Non-sliding-window blocks are only tracked after
                # request_finished, so only clean up for finished requests.
                if req_status.req.is_finished():
                    self._remove_pending_job(
                        job_id, job_status.deferred_fence_block_ids
                    )

            del self._job_statuses[job_id]
            req_status.transfer_jobs.remove(job_id)
            if req_status.finished_signaled and not req_status.transfer_jobs:
                self._remove_request(job_status.req_id)

    def get_stats(self) -> OffloadingConnectorStats | None:
        stats: OffloadingConnectorStats | None = None
        if not self._connector_stats.is_empty():
            stats = self._connector_stats
            self._connector_stats = OffloadingConnectorStats()

        with self._manager_locked():
            manager_stats = self.manager.get_stats()
        if manager_stats is not None:
            if stats is None:
                stats = manager_stats
            else:
                stats.aggregate(manager_stats)

        if not self._info_sent:
            if stats is None:
                stats = OffloadingConnectorStats()
            # Sent even when the manager reports no facts, so that the metric
            # exists whenever offloading runs.
            stats.set_info(self.manager.config_info())
            self._info_sent = True

        return stats

    def request_finished(
        self,
        request: Request,
    ) -> tuple[bool, dict[str, Any] | None]:
        """Called when a request has finished, before its blocks are freed.

        Returns:
            True if the request is being saved/sent asynchronously and blocks
            should not be freed until the request_id is returned from
            get_finished().
            Optional KVTransferParams to be included in the request outputs
            returned by the engine.

        """
        req_status = self._req_status.get(request.request_id)
        req_context = (
            req_status.req_context
            if req_status is not None
            else _create_req_context(request, self)
        )
        # Freeze at logical completion, independent of deferred tier callbacks.
        req_context.set_state(
            _RequestTokens(
                request.num_computed_tokens,
                request.num_in_flight_tokens,
                request.num_prompt_tokens,
                request.num_tokens,
            )
        )

        if req_status is None:
            # Untracked request (offloading never started): no in-flight jobs,
            # nothing was deferred, so finalize immediately.
            with self._manager_locked():
                self.manager.on_new_request(req_context)
                self.manager.on_request_finished(req_context)
            return False, None

        self._maybe_observe_lookup_async_delay(req_status)

        # Update offload keys with final block hash so the store handlers can
        # create store jobs for the last block(s) on the next schedule step.
        req_status.update_offload_keys()

        # Keep req_status alive: the store handlers process finished_req_ids
        # on the next step, and the cleanup follows their store jobs.
        # Register deferred fences so future block reuse triggers a flush via
        # _block_id_to_pending_jobs.
        for job_id in req_status.transfer_jobs:
            job_status = self._job_statuses[job_id]
            for bid in job_status.deferred_fence_block_ids or ():
                self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)

        return False, None

    def take_events(self) -> Iterable[KVCacheEvent]:
        """Drain pending KV cache events.

        Complete metadata is available only when self-describing KV events
        are enabled, and only for full-attention groups. Other shapes retain
        the previous placeholder payload so consumers can ignore them.

        Yields:
            `BlockStored` or `BlockRemoved` events corresponding to
            the underlying `OffloadingEvent` stream.

        """
        # Drain the manager eagerly, inside the region. manager.take_events() is
        # itself a generator whose backing list is cleared once exhausted, so
        # iterating it lazily would both hold the manager for as long as the
        # consumer takes and let a concurrent writer append events that the
        # clear then discards unpublished.
        with self._manager_locked():
            manager_events = list(self.manager.take_events())
        # Still a generator, so callers testing the result for truthiness keep
        # seeing a non-empty object the way they do today.
        yield from self._events_tracker.take_events(manager_events)

    def reset_cache(self) -> None:
        """Reset the offloading manager cache, evicting all stored chunks."""
        # reset_cache cannot be called in the middle of a schedule step.
        # _current_batch_jobs_to_flush may still hold a previous reset's flush
        # set if no step ran since; the new ids are merged into it.
        assert not self._current_batch_load_jobs
        assert not self._current_batch_sync_loads
        assert not self._current_batch_allocated_block_ids

        # Held for the whole reset. manager.reset_cache() drains tiers by
        # polling them itself, so a tier-poll thread locked out here loses
        # nothing; it simply skips rounds until the reset completes.
        with self._manager_locked():
            self._reset_cache()

    def _reset_cache(self) -> None:
        # Flush all in-flight jobs
        self._current_batch_jobs_to_flush.update(self._job_statuses.keys())

        for req_id, status in list(self._req_status.items()):
            if status.req.is_finished():
                if not status.finished_signaled:
                    self.manager.on_request_finished(status.req_context)
                self._remove_request(req_id)

        # Reset offloading manager cache
        self.manager.reset_cache()

        for status in self._req_status.values():
            status.transfer_jobs.clear()
            status.partial_tail_boundary = None

        # Discard jobs and save job_counter to be able to discard worker responses
        self._stale_job_threshold = self._job_counter
        if self._job_statuses:
            self._reset_fence.update(
                (job_id, job.pending_count)
                for job_id, job in self._job_statuses.items()
            )
            self._reset_fence_since = time.monotonic()
            logger.info(
                "KV offloading: waiting for workers to report %d jobs "
                "flushed by the cache reset",
                len(self._reset_fence),
            )
        elif not self._reset_fence:
            self.manager.on_reset_flushed()
        self._job_statuses.clear()
        self._block_id_to_pending_jobs.clear()
        for handler in self._store_handlers.values():
            handler.reset()

        # The manager pool is empty; pending event payloads and announced
        # reference counts are stale.
        self._events_tracker.reset()

        # Note: _current_batch_jobs_to_flush is intentionally NOT cleared.
        # The load flush IDs collected above must be delivered to workers.
        if self._chunks_being_loaded is not None:
            self._chunks_being_loaded.clear()

    def shutdown(self) -> None:
        self.manager.shutdown()

_acquire_step()

Open the step-scoped region. Idempotent within a step.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _acquire_step(self) -> None:
    """Open the step-scoped region. Idempotent within a step."""
    if self._step_region is not None:
        return
    region = ExitStack()
    region.enter_context(self.manager.lock)
    self._step_region = region

_lookup_complete_chunks(req_status, max_num_new_tokens=None)

Find how many tokens beyond num_locally_computed_tokens can be loaded.

Iterates full-attention groups first (prefix lookup), then sliding-window groups (suffix lookup). Each group may tighten max_hit_size_tokens, which can invalidate an earlier group's result, so the loop re-runs when that happens until num_hit_tokens converges.

Returns the number of hit tokens (None to defer) and the hit boundary the full-attention groups proved before any sliding-window/Mamba group narrowed it (None if not reached).

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _lookup_complete_chunks(
    self,
    req_status: RequestOffloadState,
    max_num_new_tokens: int | None = None,
) -> tuple[int | None, int | None]:
    """Find how many tokens beyond num_locally_computed_tokens can be loaded.

    Iterates full-attention groups first (prefix lookup), then sliding-window
    groups (suffix lookup). Each group may tighten max_hit_size_tokens, which
    can invalidate an earlier group's result, so the loop re-runs when that
    happens until num_hit_tokens converges.

    Returns the number of hit tokens (None to defer) and the hit boundary
    the full-attention groups proved before any sliding-window/Mamba group
    narrowed it (None if not reached).
    """
    num_computed_tokens = req_status.num_locally_computed_tokens
    max_hit_size_tokens: int = req_status.req.num_tokens
    if self._sliding_window_groups:
        # the last prompt token has to be recomputed to get the logprobs
        # for sliding window attention, we must reduce by 1 to make sure
        # we still have a hit after reduction. Tighter caps below are
        # already supported boundaries and are not reduced.
        max_hit_size_tokens -= 1
    if max_num_new_tokens is not None:
        max_hit_size_tokens = min(
            max_hit_size_tokens, num_computed_tokens + max_num_new_tokens
        )
    if req_status.max_load_tokens is not None:
        max_hit_size_tokens = min(
            max_hit_size_tokens,
            num_computed_tokens + req_status.max_load_tokens,
        )
    if self._sliding_window_groups and self._mamba_align_size is not None:
        # Constrain hit-window to the mamba block size.
        max_hit_size_tokens = round_down(
            max_hit_size_tokens, self._mamba_align_size
        )

    num_hit_tokens: int = 0
    defer_lookup = False
    lookup_groups = self._lookup_groups
    for state in req_status.group_states:
        state.lookup_hit_end_chunk_idx = 0

    # Tracks which legacy EAGLE groups already popped their volatile tail
    # in this convergence iteration. A non-EAGLE constraint resets it.
    eagle_verified: set[int] = set()
    full_attention_hit_boundary: int | None = None
    while lookup_groups:
        looked_up_sliding_window: bool = False
        full_attention_deferred = False
        looked_up_full_attention = False
        groups_iter = iter(lookup_groups)
        lookup_groups = ()
        for config_group_idx in groups_iter:
            group_config: GroupOffloadConfig = self.config.kv_group_configs[
                config_group_idx
            ]
            group_state: RequestGroupState = req_status.group_states[
                config_group_idx
            ]
            tokens_per_chunk = group_config.tokens_per_chunk
            offload_keys = group_state.offload_keys

            num_keyed_tokens = (
                len(req_status.req.block_hashes) * self.config.tokens_per_hash
            )
            assert len(offload_keys) >= num_keyed_tokens // tokens_per_chunk

            apply_eagle_drop = (
                group_config.is_eagle_group and not self.use_lookahead_block_hashes
            )
            is_eagle_unverified = (
                apply_eagle_drop and config_group_idx not in eagle_verified
            )

            # Constrain to a chunk-aligned boundary for this group.
            max_hit_size_tokens = min(
                max_hit_size_tokens, len(offload_keys) * tokens_per_chunk
            )
            if req_status.max_load_tokens is not None:
                max_hit_size_tokens = round_down(
                    max_hit_size_tokens, tokens_per_chunk
                )
            if max_hit_size_tokens - num_computed_tokens < tokens_per_chunk:
                # We can only load less than a chunk, so skip.
                return 0, full_attention_hit_boundary

            sliding_window_size_in_chunks = (
                group_config.sliding_window_size_in_chunks
            )
            # Zero cannot pin a junction, so a later confirmed hit may replace it.
            if (
                sliding_window_size_in_chunks is not None
                and not full_attention_hit_boundary
                and not full_attention_deferred
                and (looked_up_full_attention or not self._full_attention_groups)
            ):
                # Groups before the first sparse group are full attention.
                full_attention_hit_boundary = (
                    max_hit_size_tokens if self._full_attention_groups else 0
                )

            # For legacy EAGLE groups, query one extra chunk that will be
            # popped. Widening applies to every group type: without it, the
            # pop below shrinks max_hit_size_tokens past what was queried,
            # which can push the confirmed boundary under a coarser
            # sibling group's chunk granularity and zero the whole
            # request's hit (issue #52735).
            query_max = max_hit_size_tokens
            if is_eagle_unverified:
                query_max = min(
                    max_hit_size_tokens + tokens_per_chunk,
                    len(offload_keys) * tokens_per_chunk,
                )

            num_chunks = min(cdiv(query_max, tokens_per_chunk), len(offload_keys))
            start_chunk_idx = num_computed_tokens // tokens_per_chunk
            offload_keys = offload_keys[start_chunk_idx:num_chunks]

            # end index (in the sliced offload_keys) up to which we
            # have backend-confirmed hits
            num_hit_chunks: int | None
            if sliding_window_size_in_chunks is None:
                looked_up_full_attention = True
                num_hit_chunks = self._maximal_prefix_lookup(
                    offload_keys,
                    req_status.req_context,
                    req_status.req,
                    group_config,
                    start_chunk_idx,
                )
                full_attention_deferred |= num_hit_chunks is None
                group_state.lookup_hit_end_chunk_idx = start_chunk_idx + (
                    num_hit_chunks or 0
                )
            else:
                required_window = sliding_window_size_in_chunks
                if is_eagle_unverified:
                    required_window += 1
                candidate_end = min(
                    max_hit_size_tokens,
                    (num_chunks - int(is_eagle_unverified)) * tokens_per_chunk,
                )
                initial_window = group_config.load_window_size_in_chunks(
                    candidate_end
                )
                assert initial_window is not None
                num_hit_chunks = self._sliding_window_lookup(
                    offload_keys,
                    required_window,
                    req_status.req_context,
                    initial_window + int(is_eagle_unverified),
                )
            if num_hit_chunks == 0:
                return 0, full_attention_hit_boundary

            if num_hit_chunks is None:
                defer_lookup = True
            else:
                if is_eagle_unverified:
                    num_hit_chunks -= 1
                    eagle_verified.add(config_group_idx)

                max_hit_size_tokens = min(
                    max_hit_size_tokens,
                    tokens_per_chunk * (start_chunk_idx + num_hit_chunks),
                )

            new_num_hit_tokens = max_hit_size_tokens - num_computed_tokens
            if new_num_hit_tokens < tokens_per_chunk:
                # We can only load less than a chunk, so skip.
                return 0, full_attention_hit_boundary

            if new_num_hit_tokens < num_hit_tokens:
                if not apply_eagle_drop:
                    eagle_verified.clear()
                if defer_lookup:
                    # make another iteration on all groups to check
                    # if we still need to defer lookup
                    defer_lookup = False
                    lookup_groups = self._lookup_groups
                elif looked_up_sliding_window and not lookup_groups:
                    # we need another iteration to confirm previously looked up
                    # sliding window works with the new_num_hit_tokens
                    lookup_groups = self._sliding_window_groups

            looked_up_sliding_window |= sliding_window_size_in_chunks is not None
            num_hit_tokens = new_num_hit_tokens

    if defer_lookup:
        logger.debug(
            "Offloading manager delayed request %s as backend requested",
            req_status.req.request_id,
        )
        return None, full_attention_hit_boundary

    # Possibly delay the request if any hit chunk is already being loaded.
    if self._chunks_being_loaded:
        for group_config, group_state in zip(
            self.config.kv_group_configs, req_status.group_states
        ):
            tokens_per_chunk = group_config.tokens_per_chunk
            sliding_window_size_in_chunks = group_config.load_window_size_in_chunks(
                num_computed_tokens + num_hit_tokens
            )
            offload_keys = group_state.offload_keys
            num_chunks = cdiv(
                num_computed_tokens + num_hit_tokens, tokens_per_chunk
            )
            start_chunk_idx = num_computed_tokens // tokens_per_chunk
            offload_keys = offload_keys[start_chunk_idx:num_chunks]
            if sliding_window_size_in_chunks is not None:
                offload_keys = offload_keys[-sliding_window_size_in_chunks:]
            if any(key in self._chunks_being_loaded for key in offload_keys):
                # Hit chunks are being loaded, so delay the request.
                logger.debug(
                    "Delaying request %s since some of its"
                    " chunks are already being loaded",
                    req_status.req.request_id,
                )
                return None, full_attention_hit_boundary

    logger.debug(
        "Request %s hit %s offloaded tokens after %s GPU hit tokens",
        req_status.req.request_id,
        num_hit_tokens,
        num_computed_tokens,
    )

    return num_hit_tokens, full_attention_hit_boundary

_lookup_partial_tail(req_status, boundary, bridge_pending)

Look up every key a partial-tail hit at boundary loads.

bridge_pending is True if a complete chunk that the hit also loads is not ready yet. Returns (hit, pending). hit is True only if every key is ready. pending is True if no required key missed and a required key or bridge chunk is not ready yet.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _lookup_partial_tail(
    self,
    req_status: RequestOffloadState,
    boundary: int,
    bridge_pending: bool,
) -> tuple[bool, bool]:
    """Look up every key a partial-tail hit at `boundary` loads.

    `bridge_pending` is True if a complete chunk that the hit also loads
    is not ready yet.
    Returns `(hit, pending)`. `hit` is True only if every key is ready.
    `pending` is True if no required key missed and a required key or
    bridge chunk is not ready yet.
    """
    req = req_status.req
    req_context = req_status.req_context
    pending = bridge_pending
    boundary_keys: list[tuple[GroupOffloadConfig, int, OffloadKey]] = []
    for config_group_idx, group_config in enumerate(self.config.kv_group_configs):
        key_tokens = self.config.partial_tail_key_tokens(config_group_idx, boundary)
        key = req_status.register_boundary_key(group_config.group_idx, key_tokens)
        # The offload manager can report HIT as soon as the source chunk
        # is ready, while its earlier GPU load is still in flight.  Do
        # not schedule the same partial-tail transfer twice.
        pending |= key in (self._chunks_being_loaded or ())
        result = self.manager.lookup(key, req_context)
        if result is LookupResult.MISS:
            return False, False
        pending |= result is not LookupResult.HIT
        boundary_keys.append((group_config, key_tokens, key))

    if pending:
        return False, True
    for group_config, key_tokens, key in boundary_keys:
        self._events_tracker.record_partial_lookup(
            req, group_config, key_tokens, key
        )
    return True, False

_lookup_partial_tail_hit(req_status, complete_hit, full_attention_hit_boundary, max_num_new_tokens)

Extend a complete-chunk hit with a partial tail, if supported.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _lookup_partial_tail_hit(
    self,
    req_status: RequestOffloadState,
    complete_hit: int | None,
    full_attention_hit_boundary: int | None,
    max_num_new_tokens: int | None,
) -> int | None:
    """Extend a complete-chunk hit with a partial tail, if supported."""
    req_status.partial_tail_boundary = None
    if complete_hit is None or not self.config.supports_partial_tail:
        return complete_hit

    local_tokens = req_status.num_locally_computed_tokens
    complete_boundary = local_tokens + complete_hit
    tokens_per_hash = self.config.tokens_per_hash
    block_size = self.config.partial_tail_block_size
    max_eagle_proof_margin = self.config.max_eagle_proof_margin
    # A recurrent group holds state only at a producer's tail, so the
    # full-attention prefix, not the complete hit, bounds the search.
    anchor_boundary = max(complete_boundary, full_attention_hit_boundary or 0)
    anchor_pending = anchor_boundary > complete_boundary and any(
        key in (self._chunks_being_loaded or ())
        for group_config, group_state in zip(
            self.config.kv_group_configs, req_status.group_states
        )
        if not group_config.requires_cow_source
        for key in group_state.offload_keys[
            complete_boundary // block_size : anchor_boundary // block_size
        ]
    )
    if anchor_pending:
        anchor_boundary = complete_boundary
    # A partial tail carries its own EAGLE proof, so it may also lie in
    # the block after the chunk that the complete-chunk lookup pops.
    num_search_blocks = 2 if max_eagle_proof_margin else 1
    block_end = anchor_boundary + num_search_blocks * block_size
    max_boundary = min(
        req_status.req.num_prompt_tokens - 1,
        block_end - 1,
        len(req_status.req.block_hashes) * tokens_per_hash - max_eagle_proof_margin,
    )
    if max_num_new_tokens is not None:
        max_boundary = min(max_boundary, local_tokens + max_num_new_tokens)
    if req_status.max_load_tokens is not None:
        max_boundary = min(
            max_boundary,
            local_tokens + req_status.max_load_tokens,
        )
    max_boundary = round_down(max_boundary, tokens_per_hash)
    if max_boundary <= complete_boundary:
        return complete_hit

    # A boundary past the block of anchor_boundary also loads that
    # block's complete chunk. Look it up once, and only if it is needed.
    # Partial tails use one block per chunk, so a block index is a chunk
    # index.
    bridge_chunk_idx = anchor_boundary // block_size
    bridge_boundary = (bridge_chunk_idx + 1) * block_size
    # Bridge keys that the complete-chunk lookup did not confirm.
    unconfirmed_bridge_keys: list[tuple[GroupOffloadConfig, OffloadKey]] = []
    pending = anchor_pending
    if max_boundary >= bridge_boundary:
        for group_config, group_state in zip(
            self.config.kv_group_configs, req_status.group_states
        ):
            if group_config.requires_cow_source:
                continue
            key = group_state.offload_keys[bridge_chunk_idx]
            pending |= key in (self._chunks_being_loaded or ())
            # An EAGLE group always confirmed the chunk that it popped.
            if group_state.lookup_hit_end_chunk_idx <= bridge_chunk_idx:
                unconfirmed_bridge_keys.append((group_config, key))
        for _, key in unconfirmed_bridge_keys:
            result = self.manager.lookup(key, req_status.req_context)
            if result is LookupResult.MISS:
                max_boundary = bridge_boundary - tokens_per_hash
                break
            pending |= result is not LookupResult.HIT
    bridge_pending = pending
    # A tail of this request may lie below a sibling's longer full-attention
    # prefix. A block-aligned boundary there is a complete chunk, which the
    # complete-chunk lookup already missed.
    for boundary in range(max_boundary, complete_boundary, -tokens_per_hash):
        if boundary <= anchor_boundary and boundary % block_size == 0:
            continue
        loads_bridge = boundary >= bridge_boundary
        hit, boundary_pending = self._lookup_partial_tail(
            req_status, boundary, loads_bridge and bridge_pending
        )
        if hit:
            # The complete-chunk lookup recorded the confirmed bridge keys.
            if loads_bridge:
                for group_config, key in unconfirmed_bridge_keys:
                    self._events_tracker.record_lookup(
                        req_status.req, group_config, bridge_chunk_idx, key
                    )
            req_status.partial_tail_boundary = boundary
            return boundary - local_tokens
        pending |= boundary_pending

    if pending and complete_hit == 0:
        return None
    return complete_hit

_manager_locked()

Hold the manager for the duration of one hook outside schedule().

None of these hooks can legitimately run inside the step-scoped region: on_new_request comes from add_request, has_pending_push_work from the engine's liveness check before schedule(), the rest from output processing or a client RPC. So finding the region open here means a previous schedule() raised between _acquire_step() and the release in build_connector_meta. Recover instead of deadlocking.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
@contextmanager
def _manager_locked(self) -> Generator[None]:
    """Hold the manager for the duration of one hook outside schedule().

    None of these hooks can legitimately run inside the step-scoped region:
    on_new_request comes from add_request, has_pending_push_work from the
    engine's liveness check before schedule(), the rest from output
    processing or a client RPC. So finding the region open here means a
    previous schedule() raised between _acquire_step() and the release in
    build_connector_meta. Recover instead of deadlocking.
    """
    if self._step_region is not None:
        logger.error(
            "Offloading manager lock was left held by a previous "
            "schedule(); releasing it before continuing."
        )
        self._release_step()
    with self.manager.lock:
        yield

_maximal_prefix_lookup(keys, req_context, req, group_config, start_chunk_idx)

Return the number of consecutive offloaded chunks from the start, or None if the backend deferred a lookup.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _maximal_prefix_lookup(
    self,
    keys: Iterable[OffloadKey],
    req_context: ReqContext,
    req: Request,
    group_config: GroupOffloadConfig,
    start_chunk_idx: int,
) -> int | None:
    """Return the number of consecutive offloaded chunks from the start,
    or None if the backend deferred a lookup."""
    hit_count = 0
    defer_lookup = False
    for local_idx, key in enumerate(keys):
        result = self.manager.lookup(key, req_context)
        match result:
            case LookupResult.HIT | LookupResult.HIT_PENDING:
                self._events_tracker.record_lookup(
                    req,
                    group_config,
                    start_chunk_idx + local_idx,
                    key,
                )
                hit_count += 1
                defer_lookup |= result is LookupResult.HIT_PENDING
            case LookupResult.RETRY:
                # Don't break: keep scanning to let manager kick off
                # async lookups (until a miss is detected).
                defer_lookup = True
            case LookupResult.MISS:
                break
    return hit_count if not defer_lookup else None

_maybe_register_shared_prefix_junction(req_status, boundary, num_hit_tokens)

Pin a shared-prefix junction where a sparse group cut the hit, as get_computed_blocks does for a local hit, so the recompute retains the sliding-window/Mamba state at the full-attention hit boundary.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _maybe_register_shared_prefix_junction(
    self, req_status: RequestOffloadState, boundary: int, num_hit_tokens: int
) -> None:
    """Pin a shared-prefix junction where a sparse group cut the hit, as
    `get_computed_blocks` does for a local hit, so the recompute retains
    the sliding-window/Mamba state at the full-attention hit boundary."""
    if self.config.retention_interval is None:
        return
    if boundary <= req_status.num_locally_computed_tokens + num_hit_tokens:
        return
    request = req_status.req
    # Never lower a junction the local prefix cache already set.
    if boundary > request.shared_prefix_boundary:
        request.shared_prefix_boundary = boundary

_num_prefix_blocks(group_idx, local_prefix_tokens)

Return the number of shared prefix blocks in a full-attention group.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _num_prefix_blocks(self, group_idx: int, local_prefix_tokens: int) -> int:
    """Return the number of shared prefix blocks in a full-attention group."""
    group_config = self._group_configs_by_id.get(group_idx)
    if (
        group_config is None
        or group_config.sliding_window_size_in_chunks is not None
    ):
        return 0
    return local_prefix_tokens // group_config.tokens_per_block

_release_step()

Close the step-scoped region, letting other threads in.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _release_step(self) -> None:
    """Close the step-scoped region, letting other threads in."""
    region = self._step_region
    if region is None:
        return
    self._step_region = None
    region.close()

_remove_request(req_id)

Remove the state of a request, and tell each store handler.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _remove_request(self, req_id: ReqId) -> None:
    """Remove the state of a request, and tell each store handler."""
    del self._req_status[req_id]
    for handler in self._store_handlers.values():
        handler.on_request_removed(req_id)

_reserve_sync_load(req_status, num_computed_tokens, num_hit_tokens)

Reserve the step's sync-load budget for a hit, if it fits.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _reserve_sync_load(
    self,
    req_status: RequestOffloadState,
    num_computed_tokens: int,
    num_hit_tokens: int | None,
) -> bool:
    """Reserve the step's sync-load budget for a hit, if it fits."""
    req_id = req_status.req.request_id
    self._current_batch_sync_loads.discard(req_id)
    # Keep at least one token to compute, or the request can't be scheduled.
    if (
        not self.config.sync_load_budget_bytes
        or not num_hit_tokens
        or num_computed_tokens + num_hit_tokens >= req_status.req.num_tokens
    ):
        return False
    # Overestimates sliding-window and recurrent groups, which load less.
    num_bytes = num_hit_tokens * self.config.kv_bytes_per_token
    budget = self.config.sync_load_budget_bytes
    if self._current_batch_sync_load_bytes + num_bytes > budget:
        return False
    self._current_batch_sync_load_bytes += num_bytes
    self._current_batch_sync_loads.add(req_id)
    return True

_sliding_window_lookup(keys, sliding_window_size, req_context, initial_window_size=None)

Return the end index (in keys) of the last run of sliding_window_size consecutive hits, scanning from the end. The first run may need a larger window for a partial rightmost chunk. Returns 0 on miss, None if the backend deferred a lookup.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _sliding_window_lookup(
    self,
    keys: Sequence[OffloadKey],
    sliding_window_size: int,
    req_context: ReqContext,
    initial_window_size: int | None = None,
) -> int | None:
    """Return the end index (in `keys`) of the last run of
    `sliding_window_size` consecutive hits, scanning from the end.
    The first run may need a larger window for a partial rightmost chunk.
    Returns 0 on miss, None if the backend deferred a lookup."""
    defer_lookup = False
    pending_in_window = False
    consecutive_hits = 0
    required_window = initial_window_size or sliding_window_size
    for idx in range(len(keys) - 1, -1, -1):
        match self.manager.lookup(keys[idx], req_context):
            case LookupResult.HIT:
                consecutive_hits += 1
            case LookupResult.HIT_PENDING:
                # Block is in cache, just not readable yet — counts
                # as hit for the consecutive streak. Don't break:
                # keep scanning to let manager kick off async lookups.
                pending_in_window = True
                consecutive_hits += 1
            case LookupResult.RETRY:
                # Block location uncertain — does not count as hit.
                # Don't break: keep scanning to let manager kick off
                # async lookups.
                defer_lookup = True
                consecutive_hits = 0
                pending_in_window = False
                required_window = sliding_window_size
            case LookupResult.MISS:
                consecutive_hits = 0
                # This gap rules out the incomplete window to its right.
                pending_in_window = False
                required_window = sliding_window_size
        if consecutive_hits == required_window:
            return (
                None if defer_lookup or pending_in_window else idx + required_window
            )
    return None if defer_lookup or pending_in_window else consecutive_hits

_track_store_job(req_id, keys, *, trigger, fenced_block_ids, deferred_fence_block_ids)

Give a store job a job ID, track it, and return the ID.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _track_store_job(
    self,
    req_id: ReqId,
    keys: Collection[OffloadKey],
    *,
    trigger: StoreTrigger,
    fenced_block_ids: list[int] | None,
    deferred_fence_block_ids: list[int] | None,
) -> int:
    """Give a store job a job ID, track it, and return the ID."""
    job_status = TransferJobStatus(
        req_id=req_id,
        pending_count=self.config.num_workers,
        keys=tuple(dict.fromkeys(keys)),
        is_store=True,
        deferred_fence_block_ids=deferred_fence_block_ids,
        fenced_block_ids=fenced_block_ids,
        trigger=trigger,
    )
    job_id = self._generate_job_id()
    req_status = self._req_status[req_id]
    # a store can only be issued when no load is pending.
    if req_status.transfer_jobs:
        any_jid = next(iter(req_status.transfer_jobs))
        any_job = self._job_statuses[any_jid]
        assert any_job.is_store or any_job.is_sync_load
    req_status.transfer_jobs.add(job_id)

    # Watch sliding window blocks as they may get evicted
    # before the request finishes
    for bid in job_status.fenced_block_ids or ():
        self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)

    self._job_statuses[job_id] = job_status
    logger.debug(
        "Request %s offloading %d chunks (job %d)",
        job_status.req_id,
        len(job_status.keys),
        job_id,
    )

    if req_status.req.is_finished():
        # Register non-sliding-window blocks for flush detection.
        for bid in job_status.deferred_fence_block_ids or ():
            self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
            if bid in self._current_batch_allocated_block_ids:
                self._current_batch_jobs_to_flush.add(job_id)
    return job_id

_update_req_states(scheduler_output)

Update request states from the Scheduler's output.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _update_req_states(self, scheduler_output: SchedulerOutput) -> None:
    """Update request states from the Scheduler's output."""
    # new_block_ids_end[req_id][i] = end of pre-existing block_ids for
    # the i-th sliding window group (before this step's extend).
    # Used to detect sliding window blocks that got re-allocated.
    new_block_ids_end: dict[str, tuple[int, ...]] = {}

    for req_id, block_id_groups, is_full_table in yield_req_data(scheduler_output):
        req_status = self._req_status[req_id]
        req_status.update_offload_keys()

        if is_full_table:
            for group_state in req_status.group_states:
                group_state.block_ids.clear()

        if block_id_groups:
            if self._sliding_window_groups:
                new_block_ids_end[req_id] = tuple(
                    len(req_status.group_states[config_group_idx].block_ids)
                    for config_group_idx in self._sliding_window_groups
                )
            req_status.update_block_id_groups(block_id_groups)
            local_prefix_tokens = 0
            if is_full_table:
                local_prefix_tokens = req_status.num_locally_computed_tokens
                # Core can shorten or discard the hit after connector lookup.
                # Replay instead moves the execution start without dropping FA.
                if not req_status.req.replay_start:
                    local_prefix_tokens = min(
                        local_prefix_tokens, req_status.req.num_computed_tokens
                    )
            for group_idx, group_blocks in enumerate(block_id_groups):
                num_prefix_blocks = self._num_prefix_blocks(
                    group_idx, local_prefix_tokens
                )
                self._current_batch_allocated_block_ids.update(
                    bid for bid in group_blocks[num_prefix_blocks:] if bid != 0
                )

    for copy in scheduler_output.kv_cache_block_copies or ():
        self._current_batch_allocated_block_ids.add(copy.dst_block_id)

    # Zero out stale block_ids in sliding window groups' pending-store
    # positions. Only sliding window groups can have stale entries (blocks
    # freed by remove_skipped_blocks then reallocated). Only positions in
    # [first pending chunk * bsf, end) need checking, where the first
    # pending chunk is the lowest over the handlers and end is the
    # pre-extend length: earlier positions were already offloaded, later
    # ones are fresh allocations from this step.
    if self._sliding_window_groups and self._current_batch_allocated_block_ids:
        blocks_per_chunk = self.config.blocks_per_chunk
        for req_id, req_status in self._req_status.items():
            ends = new_block_ids_end.get(req_id)
            for i, config_group_idx in enumerate(self._sliding_window_groups):
                group_state = req_status.group_states[config_group_idx]
                first_live_idxs = (
                    handler.first_live_chunk_idx(req_status, config_group_idx)
                    for handler in self._store_handlers.values()
                )
                start = (
                    min(
                        (idx for idx in first_live_idxs if idx is not None),
                        default=0,
                    )
                    * blocks_per_chunk
                )
                end = ends[i] if ends is not None else len(group_state.block_ids)
                for j in range(start, end):
                    if (
                        group_state.block_ids[j]
                        in self._current_batch_allocated_block_ids
                    ):
                        group_state.block_ids[j] = 0

get_external_cache_hit_sources(request, num_external_tokens)

Split the accepted external hit by the tier each loaded key came from.

Attention keys cover only their loaded token ranges; recurrent state covers the whole reused prefix. Overlaps report the outermost tier. Uncovered tokens, including omitted sliding-window history, are external_unspecified.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def get_external_cache_hit_sources(
    self,
    request: Request,
    num_external_tokens: int,
) -> dict[CacheHitSource, int]:
    """Split the accepted external hit by the tier each loaded key came from.

    Attention keys cover only their loaded token ranges; recurrent state
    covers the whole reused prefix. Overlaps report the outermost tier.
    Uncovered tokens, including omitted sliding-window history, are
    `external_unspecified`.
    """
    sources: dict[CacheHitSource, int] = {}
    if num_external_tokens == 0:
        return sources
    req_status = self._req_status[request.request_id]
    start = req_status.num_locally_computed_tokens
    end = start + num_external_tokens

    events: list[tuple[int, int, CacheHitSource]] = [
        (end, 0, CacheHitSource.EXTERNAL_UNSPECIFIED)
    ]
    for lo, hi, key in req_status.load_key_ranges:
        lo, hi = max(start, lo), min(end, hi)
        if lo >= hi:
            continue
        source = CacheHitSource(
            self.manager.get_load_source(key, req_status.req_context)
        )
        events.extend(((lo, 1, source), (hi, -1, source)))

    # Sweep endpoints instead of rescanning every range for each interval.
    active: Counter[CacheHitSource] = Counter()
    position = start
    for boundary, delta, source in sorted(events):
        if boundary > position:
            tiers = [tier for tier, count in active.items() if count > 0]
            tier = (
                CacheHitSource.outermost(tiers)
                if tiers
                else CacheHitSource.EXTERNAL_UNSPECIFIED
            )
            sources[tier] = sources.get(tier, 0) + boundary - position
        active[source] += delta
        position = boundary
    return sources

get_num_new_matched_tokens(request, num_computed_tokens, max_num_new_tokens=None)

Get number of new tokens that can be loaded beyond the num_computed_tokens.

Parameters:

  • request

    (Request) –

    the request object.

  • num_computed_tokens

    (int) –

    the number of locally computed tokens for this request

  • max_num_new_tokens

    (int | None, default: None ) –

    cap on the number of tokens that may be loaded beyond num_computed_tokens, if any.

Returns:

  • tuple[int | None, bool] –

    A tuple with the following elements: - The number of tokens that can be loaded beyond what is already computed. If None, it means that the connector needs more time to determine the number of matched tokens, and the scheduler should query for this request again later. - True if tokens will be loaded asynchronously (between scheduler steps).

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def get_num_new_matched_tokens(
    self,
    request: Request,
    num_computed_tokens: int,
    max_num_new_tokens: int | None = None,
) -> tuple[int | None, bool]:
    """Get number of new tokens that can be loaded beyond the
    num_computed_tokens.

    Args:
        request (Request): the request object.
        num_computed_tokens (int): the number of locally
            computed tokens for this request
        max_num_new_tokens (int | None): cap on the number of tokens that
            may be loaded beyond `num_computed_tokens`, if any.

    Returns:
        A tuple with the following elements:
            - The number of tokens that can be loaded beyond what is
              already computed.
              If None, it means that the connector needs more time to
              determine the number of matched tokens, and the scheduler
              should query for this request again later.
            - `True` if tokens will be loaded asynchronously
              (between scheduler steps).

    """
    # First manager-touching hook of a step: opens the step-scoped region,
    # closed at the end of build_connector_meta.
    self._acquire_step()

    req_status = self._req_status[request.request_id]
    for group_state in req_status.group_states:
        group_state.block_ids.clear()
    for handler in self._store_handlers.values():
        handler.on_request_restarted(request.request_id)

    if req_status.transfer_jobs:
        logger.debug(
            "Delaying request %s since it still has in-flight transfers",
            request.request_id,
        )
        return None, False

    if request.resumable and self.use_lookahead_block_hashes:
        # Core may have recomputed block hashes after preemption.
        for group_state in req_status.group_states:
            group_state.offload_keys.clear()
    req_status.update_offload_keys()
    req_status.num_locally_computed_tokens = num_computed_tokens

    num_hit_tokens: int | None
    if (
        request.skip_reading_external_kv
        or req_status.load_failed
        or not self._offloading_enabled
    ):
        num_hit_tokens = 0
    else:
        lookup_start = time.monotonic()
        num_hit_tokens = self._lookup(req_status, max_num_new_tokens)
        self._connector_stats.observe_histogram(
            _ConnectorMetricName.LOOKUP_SYNC_DELAY,
            time.monotonic() - lookup_start,
        )
        if num_hit_tokens is None:
            if req_status.deferred_lookup_start_time is None:
                req_status.deferred_lookup_start_time = lookup_start
        else:
            self._maybe_observe_lookup_async_delay(req_status)
    req_status.update_num_hit_chunks(num_computed_tokens + (num_hit_tokens or 0))

    sync_load = self._reserve_sync_load(
        req_status, num_computed_tokens, num_hit_tokens
    )
    return num_hit_tokens, bool(num_hit_tokens) and not sync_load

has_pending_push_work()

Whether the engine must keep stepping.

While True, build_connector_meta() and update_connector_output() continue to be called even when no requests are scheduled.

Must use a short region, never the step-scoped one. The engine asks this before schedule() and returns early when the answer is False, so a step-scoped acquire here would never reach build_connector_meta's release and would hold the manager for good -- which is precisely the state an idle engine settles into.

A cache reset fence also keeps it stepping: the flush must reach the workers and their reports come back before on_reset_flushed().

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def has_pending_push_work(self) -> bool:
    """Whether the engine must keep stepping.

    While True, build_connector_meta() and update_connector_output()
    continue to be called even when no requests are scheduled.

    Must use a short region, never the step-scoped one. The engine asks this
    before schedule() and returns early when the answer is False, so a
    step-scoped acquire here would never reach build_connector_meta's
    release and would hold the manager for good -- which is precisely the
    state an idle engine settles into.

    A cache reset fence also keeps it stepping: the flush must reach the
    workers and their reports come back before on_reset_flushed().
    """
    if self._job_statuses or self._reset_fence:
        return True
    with self._manager_locked():
        return self.manager.has_pending_work()

on_new_request(request)

Called when a new request is added to the scheduler.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def on_new_request(self, request: Request) -> None:
    """Called when a new request is added to the scheduler."""
    req_context = _create_req_context(request, self)
    req_status = RequestOffloadState(
        config=self.config,
        req=request,
        req_context=req_context,
        offloading_context=RequestOffloadingContext(),
    )
    self._req_status[request.request_id] = req_status
    try:
        with self._manager_locked():
            req_status.offloading_context = self.manager.on_new_request(req_context)
    except Exception:
        del self._req_status[request.request_id]
        raise
    self._events_tracker.on_new_request(
        req_context, request, self.config.kv_group_configs
    )

request_finished(request)

Called when a request has finished, before its blocks are freed.

Returns:

  • bool –

    True if the request is being saved/sent asynchronously and blocks

  • dict[str, Any] | None –

    should not be freed until the request_id is returned from

  • tuple[bool, dict[str, Any] | None] –

    get_finished().

  • tuple[bool, dict[str, Any] | None] –

    Optional KVTransferParams to be included in the request outputs

  • tuple[bool, dict[str, Any] | None] –

    returned by the engine.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def request_finished(
    self,
    request: Request,
) -> tuple[bool, dict[str, Any] | None]:
    """Called when a request has finished, before its blocks are freed.

    Returns:
        True if the request is being saved/sent asynchronously and blocks
        should not be freed until the request_id is returned from
        get_finished().
        Optional KVTransferParams to be included in the request outputs
        returned by the engine.

    """
    req_status = self._req_status.get(request.request_id)
    req_context = (
        req_status.req_context
        if req_status is not None
        else _create_req_context(request, self)
    )
    # Freeze at logical completion, independent of deferred tier callbacks.
    req_context.set_state(
        _RequestTokens(
            request.num_computed_tokens,
            request.num_in_flight_tokens,
            request.num_prompt_tokens,
            request.num_tokens,
        )
    )

    if req_status is None:
        # Untracked request (offloading never started): no in-flight jobs,
        # nothing was deferred, so finalize immediately.
        with self._manager_locked():
            self.manager.on_new_request(req_context)
            self.manager.on_request_finished(req_context)
        return False, None

    self._maybe_observe_lookup_async_delay(req_status)

    # Update offload keys with final block hash so the store handlers can
    # create store jobs for the last block(s) on the next schedule step.
    req_status.update_offload_keys()

    # Keep req_status alive: the store handlers process finished_req_ids
    # on the next step, and the cleanup follows their store jobs.
    # Register deferred fences so future block reuse triggers a flush via
    # _block_id_to_pending_jobs.
    for job_id in req_status.transfer_jobs:
        job_status = self._job_statuses[job_id]
        for bid in job_status.deferred_fence_block_ids or ():
            self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)

    return False, None

reset_cache()

Reset the offloading manager cache, evicting all stored chunks.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def reset_cache(self) -> None:
    """Reset the offloading manager cache, evicting all stored chunks."""
    # reset_cache cannot be called in the middle of a schedule step.
    # _current_batch_jobs_to_flush may still hold a previous reset's flush
    # set if no step ran since; the new ids are merged into it.
    assert not self._current_batch_load_jobs
    assert not self._current_batch_sync_loads
    assert not self._current_batch_allocated_block_ids

    # Held for the whole reset. manager.reset_cache() drains tiers by
    # polling them itself, so a tier-poll thread locked out here loses
    # nothing; it simply skips rounds until the reset completes.
    with self._manager_locked():
        self._reset_cache()

take_events()

Drain pending KV cache events.

Complete metadata is available only when self-describing KV events are enabled, and only for full-attention groups. Other shapes retain the previous placeholder payload so consumers can ignore them.

Yields:

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def take_events(self) -> Iterable[KVCacheEvent]:
    """Drain pending KV cache events.

    Complete metadata is available only when self-describing KV events
    are enabled, and only for full-attention groups. Other shapes retain
    the previous placeholder payload so consumers can ignore them.

    Yields:
        `BlockStored` or `BlockRemoved` events corresponding to
        the underlying `OffloadingEvent` stream.

    """
    # Drain the manager eagerly, inside the region. manager.take_events() is
    # itself a generator whose backing list is cleared once exhausted, so
    # iterating it lazily would both hold the manager for as long as the
    # consumer takes and let a concurrent writer append events that the
    # clear then discards unpublished.
    with self._manager_locked():
        manager_events = list(self.manager.take_events())
    # Still a generator, so callers testing the result for truthiness keep
    # seeing a non-empty object the way they do today.
    yield from self._events_tracker.take_events(manager_events)

update_connector_output(connector_output)

Update KVConnector state from worker-side connectors output.

Parameters:

  • connector_output

    (KVConnectorOutput) –

    the worker-side connectors output.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def update_connector_output(self, connector_output: KVConnectorOutput):
    """Update KVConnector state from worker-side connectors output.

    Args:
        connector_output (KVConnectorOutput): the worker-side
            connectors output.

    """
    with self._manager_locked():
        self._update_connector_output(connector_output)

_StoreJobSink

The store-job intake of one handler in one scheduler step.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
class _StoreJobSink:
    """The store-job intake of one handler in one scheduler step."""

    def __init__(
        self, scheduler: "OffloadingConnectorScheduler", trigger: StoreTrigger
    ):
        self._scheduler = scheduler
        self._trigger = trigger
        self.jobs: dict[int, TransferJob] = {}

    def prepare_store(
        self, keys: Collection[OffloadKey], req_context: ReqContext
    ) -> PrepareStoreOutput | None:
        output = self._scheduler.manager.prepare_store(
            keys, req_context, trigger=self._trigger
        )
        if output is None:
            self._scheduler._connector_stats.increase_counter(
                _ConnectorMetricName.ALLOCATION_FAILURE
            )
        return output

    def add_job(
        self,
        req_id: ReqId,
        keys: Collection[OffloadKey],
        src_spec: GPULoadStoreSpec,
        dst_spec: LoadStoreSpec,
        *,
        fenced_block_ids: list[int] | None = None,
        deferred_fence_block_ids: list[int] | None = None,
    ) -> None:
        job_id = self._scheduler._track_store_job(
            req_id,
            keys,
            trigger=self._trigger,
            fenced_block_ids=fenced_block_ids,
            deferred_fence_block_ids=deferred_fence_block_ids,
        )
        self.jobs[job_id] = TransferJob(
            req_id=req_id, src_spec=src_spec, dst_spec=dst_spec
        )

_parse_tier_filter(raw)

Parse raw kv_transfer_params tier matchers into a TierFilter.

Source code in vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
def _parse_tier_filter(raw: Any) -> TierFilter:
    """Parse raw kv_transfer_params tier matchers into a TierFilter."""
    if not isinstance(raw, list):
        logger.warning(
            "_parse_tier_filter: expected list, got %s; ignoring",
            type(raw).__name__,
        )
        return TierFilter.ALL
    matchers: list[TierMatcher] = []
    for entry in raw:
        if not isinstance(entry, dict):
            logger.warning("_parse_tier_filter: entry is not a dict; skipping")
            continue
        medium: Medium | None = None
        locality: Locality | None = None
        raw_medium = entry.get(MATCHER_MEDIUM_KEY)
        if raw_medium is not None:
            try:
                medium = Medium(raw_medium.upper())
            except (ValueError, AttributeError):
                logger.warning(
                    "_parse_tier_filter: unknown medium %r; skipping entry",
                    raw_medium,
                )
                continue
        raw_locality = entry.get(MATCHER_LOCALITY_KEY)
        if raw_locality is not None:
            try:
                locality = Locality(raw_locality.upper())
            except (ValueError, AttributeError):
                logger.warning(
                    "_parse_tier_filter: unknown locality %r; skipping entry",
                    raw_locality,
                )
                continue
        matchers.append(TierMatcher(medium=medium, locality=locality))
    if not matchers:
        if not raw:  # input was [] — user explicitly wants nothing
            return TierFilter(matchers=())
        # all entries were invalid — fall back to ALL
        return TierFilter.ALL
    return TierFilter(matchers=tuple(matchers))