echo
----
----
-- We will exhaust the tokens across all streams while admission is blocked on
-- n3, using a single 4 MiB (deduction, the write itself is small) write. Then,
-- we will write a 1 MiB put to the range, split it, write a 1 MiB put to the
-- LHS range, and a 1MiB put to the RHS range, merge the ranges, and write a 1
-- MiB put to the merged range. We expect that at each stage where a send
-- queue develops n1->s3, the send queue will be flushed by the range merge
-- and range split range operations.


-- Start by exhausting the tokens from n1->s3 and blocking admission on s3.
-- (Issuing 4x1MiB regular, 3x replicated write that's not admitted on s3.)


(Sending 1 MiB put request to pre-split range)


(Sent 1 MiB put request to pre-split range)


-- Send queue metrics from n1, n3's send queue should have 1 MiB for s3.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 1.0 MiB  
  kvflowcontrol.send_queue.prevent.count                            | 0 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      


-- Observe the total tracked tokens per-stream on n1, s3's entries will still
-- be tracked here.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
    FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 0 B                   
  75       | 2        | 0 B                   
  75       | 3        | 4.0 MiB               


-- Per-store tokens available from n1, these should reflect the lack of tokens 
-- for s3.
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 0 B                    | -3.0 MiB               | 0 B                    | -2.0 MiB                


-- (Splitting range.)


-- Observe the newly split off replica, with its own three streams.
SELECT range_id, count(*) AS streams
    FROM crdb_internal.kv_flow_control_handles_v2
GROUP BY (range_id)
ORDER BY streams DESC;

  range_id | stream_count  
-----------+---------------
  75       | 3             
  76       | 3             


-- Send queue and flow token metrics from n1, post-split.
-- We expect to see a force flush of the send queue for s3.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 0 B      
  kvflowcontrol.send_queue.prevent.count                            | 0 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 0 B                    | -3.0 MiB               | 0 B                    | -3.0 MiB                


(Sending 1 MiB put request to post-split LHS range)


(Sent 1 MiB put request to post-split LHS range)


(Sending 1 MiB put request to post-split RHS range)


(Sent 1 MiB put request to post-split RHS range)


-- Send queue and flow token metrics from n1, post-split and 1 MiB put on
-- each side.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 2.0 MiB  
  kvflowcontrol.send_queue.prevent.count                            | 0 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 0 B                    | -5.0 MiB               | 0 B                    | -3.0 MiB                


-- (Merging ranges.)


-- Send queue and flow token metrics from n1, post-split-merge.
-- We expect to see a force flush of the send queue for s3 again.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 0 B      
  kvflowcontrol.send_queue.prevent.count                            | 0 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 3.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 0 B                    | -4.0 MiB               | 0 B                    | -4.0 MiB                


(Sending 1 MiB put request to post-split-merge range)


(Sent 1 MiB put request to post-split-merge range)


-- Send queue and flow token metrics from n1, post-split-merge and 1 MiB put.
-- We expect to see the send queue develop for s3 again.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 1.0 MiB  
  kvflowcontrol.send_queue.prevent.count                            | 0 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 3.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 0 B                    | -5.0 MiB               | 0 B                    | -4.0 MiB                


-- (Allowing below-raft admission to proceed on [n1,n2,n3].)


-- Send queue and flow token metrics from n1, all tokens should be returned.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 0 B      
  kvflowcontrol.send_queue.prevent.count                            | 0 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 3.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
----
----

# vim:ft=sql
