# Initialize a range with three replicas, none of which have send tokens.
init regular_init=0 elastic_init=0
range_id=1 tenant_id=1 local_replica_id=1 next_raft_index=1
  store_id=1 replica_id=1 type=VOTER_FULL state=StateReplicate next=1
  store_id=2 replica_id=2 type=VOTER_FULL state=StateReplicate next=1
  store_id=3 replica_id=3 type=VOTER_FULL state=StateReplicate next=1
----
r1: [(n1,s1):1*,(n2,s2):2,(n3,s3):3]
t1/s1: eval reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB
       send reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB
t1/s2: eval reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB
       send reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB
t1/s3: eval reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB
       send reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB

# Append three entries. Replica 2 has a send-queue.
raft_event pull-mode
range_id=1
  entries
    term=1 index=1 pri=NormalPri size=1MiB
    term=1 index=2 pri=NormalPri size=1MiB
    term=1 index=3 pri=NormalPri size=1MiB
----
t1/s1: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
t1/s2: eval reg=+0 B/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB
t1/s3: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB

stream_state range_id=1
----
(n1,s1):1: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n2,s2):2: state=replicate closed=false inflight=[1,1) send_queue=[1,4) precise_q_size=+3.0 MiB watching-for-tokens
eval deducted: reg=+0 B ela=+3.0 MiB
eval original in send-q: reg=+3.0 MiB ela=+0 B
++++
(n3,s3):3: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
MsgApps sent in pull mode:
 to: 3, lowPri: false entries: [1 2 3]
++++

# Give s2 some send tokens. The watcher will subtract the available tokens, so
# elastic tokens will continue to be 0.
adjust_tokens send
  store_id=2 pri=HighPri tokens=512KiB
----
t1/s1: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
t1/s2: eval reg=+0 B/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=+512 KiB/+16 MiB ela=+0 B/+8.0 MiB
t1/s3: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB

# Replica 2 has deducted some tokens and waiting for a scheduler event to send
# using these tokens.
stream_state range_id=1
----
(n1,s1):1: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n2,s2):2: state=replicate closed=false inflight=[1,1) send_queue=[1,4) precise_q_size=+3.0 MiB deducted=+512 KiB
eval deducted: reg=+0 B ela=+3.0 MiB
eval original in send-q: reg=+3.0 MiB ela=+0 B
++++
(n3,s3):3: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
schedule-controller-event-count: 1
scheduled-replicas: 2

# Scheduler event causes one entry to be sent. It was 1MiB, but only 512KiB of
# tokens were deducted, to we will deduct an additional 512KiB.
handle_scheduler_event range_id=1
----
(n1,s1):1: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n2,s2):2: state=replicate closed=false inflight=[1,2) (1.0 MiB) send_queue=[2,4) precise_q_size=+2.0 MiB watching-for-tokens
eval deducted: reg=+0 B ela=+3.0 MiB
eval original in send-q: reg=+2.0 MiB ela=+0 B
LowPri:
  term=1 index=1  tokens=1048576
++++
(n3,s3):3: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
MsgApps sent in pull mode:
 to: 2, lowPri: true entries: [1]
++++
schedule-controller-event-count: 1

# Noop adjustment, just to see the current token counts. s2 has -512KiB of
# elastic tokens.
adjust_tokens send
  store_id=2 pri=HighPri tokens=0
----
t1/s1: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
t1/s2: eval reg=+0 B/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=+512 KiB/+16 MiB ela=-512 KiB/+8.0 MiB
t1/s3: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB

# Bump up s2 to -512KiB + 2MiB = 1.5MiB of elastic tokens. These are
# immediately deducted.
adjust_tokens send
  store_id=2 pri=HighPri tokens=2MiB
----
t1/s1: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
t1/s2: eval reg=+0 B/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=+2.5 MiB/+16 MiB ela=+0 B/+8.0 MiB
t1/s3: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB

# Replica 2 has deducted 1.5MiB of tokens and is waiting for a scheduler
# event.
stream_state range_id=1
----
(n1,s1):1: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n2,s2):2: state=replicate closed=false inflight=[1,2) (1.0 MiB) send_queue=[2,4) precise_q_size=+2.0 MiB deducted=+1.5 MiB
eval deducted: reg=+0 B ela=+3.0 MiB
eval original in send-q: reg=+2.0 MiB ela=+0 B
LowPri:
  term=1 index=1  tokens=1048576
++++
(n3,s3):3: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
schedule-controller-event-count: 2
scheduled-replicas: 2

# Scheduler event. Only the first entry in the send-queue is sent, because the
# second entry would exceed the max size limit.
# Replica 2 still has 512KiB of deducted tokens that are not used, and is
# waiting for a scheduler event.
handle_scheduler_event range_id=1
----
(n1,s1):1: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n2,s2):2: state=replicate closed=false inflight=[1,3) (2.0 MiB) send_queue=[3,4) precise_q_size=+1.0 MiB deducted=+512 KiB
eval deducted: reg=+0 B ela=+3.0 MiB
eval original in send-q: reg=+1.0 MiB ela=+0 B
LowPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
++++
(n3,s3):3: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
MsgApps sent in pull mode:
 to: 2, lowPri: true entries: [2]
++++
schedule-controller-event-count: 3
scheduled-replicas: 2

# The second entry is sent, and empties the send-queue. Extra 512KiB needed to
# be deducted.
handle_scheduler_event range_id=1
----
(n1,s1):1: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n2,s2):2: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+0 B ela=+3.0 MiB
eval original in send-q: reg=+0 B ela=+0 B
LowPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n3,s3):3: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
MsgApps sent in pull mode:
 to: 2, lowPri: true entries: [3]
++++
schedule-controller-event-count: 3

# s2 has -512KiB of elastic send tokens.
adjust_tokens send
  store_id=2 pri=HighPri tokens=0
----
t1/s1: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
t1/s2: eval reg=+0 B/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=+2.5 MiB/+16 MiB ela=-512 KiB/+8.0 MiB
t1/s3: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB

# Make replica 3 fall behind, so it has a send-queue.
set_replicas pull-mode
range_id=1 tenant_id=1 local_replica_id=1 next_raft_index=4
  store_id=1 replica_id=1 type=VOTER_FULL state=StateReplicate next=4
  store_id=2 replica_id=2 type=VOTER_FULL state=StateReplicate next=4
  store_id=3 replica_id=3 type=VOTER_FULL state=StateReplicate next=2
----
r1: [(n1,s1):1*,(n2,s2):2,(n3,s3):3]

stream_state range_id=1
----
(n1,s1):1: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n2,s2):2: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+0 B ela=+3.0 MiB
eval original in send-q: reg=+0 B ela=+0 B
LowPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n3,s3):3: state=replicate closed=false inflight=[1,2) (1.0 MiB) send_queue=[2,4) precise_q_size=+0 B watching-for-tokens
eval deducted: reg=+0 B ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
++++
schedule-controller-event-count: 3

# Noop. Note that s3 has 0 send tokens.
adjust_tokens send
  store_id=2 pri=HighPri tokens=0
----
t1/s1: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
t1/s2: eval reg=+0 B/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=+2.5 MiB/+16 MiB ela=-512 KiB/+8.0 MiB
t1/s3: eval reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB
       send reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB

# Provide 10KiB of elastic send tokens to s3. Only 4KiB are deducted, leaving
# 6KiB, since we deduct a minimum of 4KiB of send tokens when trying to empty
# the send-queue.
adjust_tokens send
  store_id=3 pri=HighPri tokens=10KiB
----
t1/s1: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
t1/s2: eval reg=+0 B/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=+2.5 MiB/+16 MiB ela=-512 KiB/+8.0 MiB
t1/s3: eval reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB
       send reg=+10 KiB/+16 MiB ela=+6.0 KiB/+8.0 MiB

# Note the deducted value of 4KiB. Replica 3 is waiting for a scheduler event.
stream_state range_id=1
----
(n1,s1):1: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n2,s2):2: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+0 B ela=+3.0 MiB
eval original in send-q: reg=+0 B ela=+0 B
LowPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n3,s3):3: state=replicate closed=false inflight=[1,2) (1.0 MiB) send_queue=[2,4) precise_q_size=+0 B deducted=+4.0 KiB
eval deducted: reg=+0 B ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
++++
schedule-controller-event-count: 4
scheduled-replicas: 3

# Scheduler event. Replica 3 deducts 1MiB-4KiB without waiting since the entry
# was actually 1MiB.
handle_scheduler_event range_id=1
----
(n1,s1):1: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n2,s2):2: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+0 B ela=+3.0 MiB
eval original in send-q: reg=+0 B ela=+0 B
LowPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n3,s3):3: state=replicate closed=false inflight=[1,3) (2.0 MiB) send_queue=[3,4) precise_q_size=+0 B watching-for-tokens
eval deducted: reg=+0 B ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
LowPri:
  term=1 index=2  tokens=1048576
++++
MsgApps sent in pull mode:
 to: 3, lowPri: true entries: [2]
++++
schedule-controller-event-count: 4

# Add 3MiB of elastic send tokens, to return to 2MiB+10KiB of tokens. The
# send-stream now has an estimate of 1MiB needed per entry, and deduct 1.1 x
# queue size = 1.1MiB.
adjust_tokens send
  store_id=3 pri=HighPri tokens=3MiB
----
t1/s1: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
t1/s2: eval reg=+0 B/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=+2.5 MiB/+16 MiB ela=-512 KiB/+8.0 MiB
t1/s3: eval reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB
       send reg=+3.0 MiB/+16 MiB ela=+932 KiB/+8.0 MiB

# Note the deducted value of 1.1MiB. Replica 3 is waiting for a scheduler
# event.
stream_state range_id=1
----
(n1,s1):1: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n2,s2):2: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+0 B ela=+3.0 MiB
eval original in send-q: reg=+0 B ela=+0 B
LowPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n3,s3):3: state=replicate closed=false inflight=[1,3) (2.0 MiB) send_queue=[3,4) precise_q_size=+0 B deducted=+1.1 MiB
eval deducted: reg=+0 B ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
LowPri:
  term=1 index=2  tokens=1048576
++++
schedule-controller-event-count: 5
scheduled-replicas: 3

# Scheduler event. Replica 3 returns 0.1MiB since the entry was actually 1MiB.
# It no longer has a send-queue.
handle_scheduler_event range_id=1
----
(n1,s1):1: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+3.0 MiB ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
NormalPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n2,s2):2: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+0 B ela=+3.0 MiB
eval original in send-q: reg=+0 B ela=+0 B
LowPri:
  term=1 index=1  tokens=1048576
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
(n3,s3):3: state=replicate closed=false inflight=[1,4) (3.0 MiB) send_queue=[4,4) precise_q_size=+0 B
eval deducted: reg=+0 B ela=+0 B
eval original in send-q: reg=+0 B ela=+0 B
LowPri:
  term=1 index=2  tokens=1048576
  term=1 index=3  tokens=1048576
++++
MsgApps sent in pull mode:
 to: 3, lowPri: true entries: [3]
++++
schedule-controller-event-count: 5

# Noop, to see current token values.
adjust_tokens send
  store_id=3 pri=HighPri tokens=0
----
t1/s1: eval reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=-3.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB
t1/s2: eval reg=+0 B/+16 MiB ela=-3.0 MiB/+8.0 MiB
       send reg=+2.5 MiB/+16 MiB ela=-512 KiB/+8.0 MiB
t1/s3: eval reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB
       send reg=+3.0 MiB/+16 MiB ela=+1.0 MiB/+8.0 MiB
