echo
----
----
-- This test exercises send queue formation, prevention and flushing.
-- The structure roughly follows:
--   Start with three voters on [n1,n2,n3], where n1 is the leader+leaseholder.
--   Large regular write -4 MiB.
--   Allow admission [n1,n2].
--   - Tokens should be returned for n1 and n2.
--   Block admission [n2].
--   Regular write -1 MiB.
--   - Shouldn't be blocked on wait-for-eval because of quorum [n1,n2].
--   - Metrics should reflect send queue formation on n3.
--   Stop n2.
--   Regular write -1 MiB.
--   - Blocks on wait-for-eval, however the test bypasses this instance.
--   - Metrics should reflect n3 being force flushed.
--   Allow admission [n1,n2,n3].
--   Start n2.
--   Add n4, n5, the voters now are [n1,n2,n3,n4,n5].
--   Block admission [n4,n5] (already blocked)
--   Regular write -4 MiB.
--   Regular write -1  MiB.
--   - Shouldn't be blocked on wait-for-eval because of quorum [n1,n2,n3]
--   - Metrics should reflect send queue formation on n4,n5.
--   Unblock admission [n4,n5].
--   - Wait for tokens to be returned.
--   Block admission [n2,n3,n4,n5].
--   Regular write -4 MiB.
--   Regular write -1  MiB.
--   - Blocks on wait-for-eval, however the test bypasses this instance.    
--   - Metrics should reflect 2 streams being prevented from forming a send queue.
--   Allow admission [n1,n2,n3,n4,n5] (all).
--   Assert all tokens returned.
--
-- Start by printing the relevant metrics on n1, first the flow token metrics.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%tokens%'
ORDER BY name ASC;

  kvflowcontrol.tokens.eval.elastic.available                       | 6.0 MiB  
  kvflowcontrol.tokens.eval.elastic.deducted                        | 0 B      
  kvflowcontrol.tokens.eval.elastic.returned                        | 0 B      
  kvflowcontrol.tokens.eval.elastic.returned.disconnect             | 0 B      
  kvflowcontrol.tokens.eval.elastic.unaccounted                     | 0 B      
  kvflowcontrol.tokens.eval.regular.available                       | 12 MiB   
  kvflowcontrol.tokens.eval.regular.deducted                        | 0 B      
  kvflowcontrol.tokens.eval.regular.returned                        | 0 B      
  kvflowcontrol.tokens.eval.regular.returned.disconnect             | 0 B      
  kvflowcontrol.tokens.eval.regular.unaccounted                     | 0 B      
  kvflowcontrol.tokens.send.elastic.available                       | 6.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted                        | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.elastic.returned                        | 0 B      
  kvflowcontrol.tokens.send.elastic.returned.disconnect             | 0 B      
  kvflowcontrol.tokens.send.elastic.unaccounted                     | 0 B      
  kvflowcontrol.tokens.send.regular.available                       | 12 MiB   
  kvflowcontrol.tokens.send.regular.deducted                        | 0 B      
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.returned                        | 0 B      
  kvflowcontrol.tokens.send.regular.returned.disconnect             | 0 B      
  kvflowcontrol.tokens.send.regular.unaccounted                     | 0 B      


-- Send queue metrics from n1.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 0 B  
  kvflowcontrol.send_queue.prevent.count                            | 0 B  
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B  
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B  
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B  
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B  


-- Per-store tokens available from n1.
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 


-- (Issuing 4x1MiB regular, 3x replicated write that's not admitted.)


-- Observe the total tracked tokens per-stream on n1.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
    FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 4.0 MiB               
  75       | 2        | 4.0 MiB               
  75       | 3        | 4.0 MiB               


-- And, the per-store tokens available post-write from n1.
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 0 B                    | -2.0 MiB               | 0 B                    | -2.0 MiB                
  2        | 0 B                    | -2.0 MiB               | 0 B                    | -2.0 MiB                
  3        | 0 B                    | -2.0 MiB               | 0 B                    | -2.0 MiB                


-- (Allowing below-raft admission to proceed on n1 and n2.)
-- [n1(enabled),n2(enabled),n3(blocked)]


-- Observe the total tracked tokens per-stream on n1.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
    FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 0 B                   
  75       | 2        | 0 B                   
  75       | 3        | 4.0 MiB               


-- Per-store tokens available from n1.
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 0 B                    | -2.0 MiB               | 0 B                    | -2.0 MiB                


-- (Blocking below-raft admission on n2.)


-- (Issuing 1x1MiB regular, 3x replicated write that's not admitted.)


-- The send queue metrics from n1 should reflect the 1 MiB write being queued
-- for n3 and 1 MiB tracked for n2 that is yet to be admitted.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 1.0 MiB  
  kvflowcontrol.send_queue.prevent.count                            | 0 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      


-- Per-store tokens available from n1.
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 3.0 MiB                | 1.0 MiB                | 3.0 MiB                | 1.0 MiB                 
  3        | 0 B                    | -3.0 MiB               | 0 B                    | -2.0 MiB                


-- (Stopping n2.)


-- Flow token metrics from n1, the disconnect should be reflected in the metrics.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%tokens%'
ORDER BY name ASC;

  kvflowcontrol.tokens.eval.elastic.available                       | 1.0 MiB  
  kvflowcontrol.tokens.eval.elastic.deducted                        | 15 MiB   
  kvflowcontrol.tokens.eval.elastic.returned                        | 10 MiB   
  kvflowcontrol.tokens.eval.elastic.returned.disconnect             | 1.0 MiB  
  kvflowcontrol.tokens.eval.elastic.unaccounted                     | 0 B      
  kvflowcontrol.tokens.eval.regular.available                       | 8.0 MiB  
  kvflowcontrol.tokens.eval.regular.deducted                        | 14 MiB   
  kvflowcontrol.tokens.eval.regular.returned                        | 10 MiB   
  kvflowcontrol.tokens.eval.regular.returned.disconnect             | 1.0 MiB  
  kvflowcontrol.tokens.eval.regular.unaccounted                     | 0 B      
  kvflowcontrol.tokens.send.elastic.available                       | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted                        | 15 MiB   
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.elastic.returned                        | 10 MiB   
  kvflowcontrol.tokens.send.elastic.returned.disconnect             | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.unaccounted                     | 0 B      
  kvflowcontrol.tokens.send.regular.available                       | 8.0 MiB  
  kvflowcontrol.tokens.send.regular.deducted                        | 14 MiB   
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.returned                        | 10 MiB   
  kvflowcontrol.tokens.send.regular.returned.disconnect             | 1.0 MiB  
  kvflowcontrol.tokens.send.regular.unaccounted                     | 0 B      


-- Send queue metrics from n1, n3's send queue should have been force-flushed.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 0 B      
  kvflowcontrol.send_queue.prevent.count                            | 0 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      


-- Observe the total tracked tokens per-stream on n1, n3's flushed entries 
-- will also be tracked here.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
    FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 0 B                   
  75       | 3        | 5.0 MiB               


-- Per-store tokens available from n1, these should reflect the deducted 
-- tokens from force-flushing n3.
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 0 B                    | -3.0 MiB               | 0 B                    | -3.0 MiB                


-- (Enabling wait-for-eval bypass.)


-- (Issuing 1x1MiB regular, 3x replicated write that's not admitted.)


-- (Disabling wait-for-eval bypass.)


-- Send queue metrics from n1, n3's should not be allowed to form a send queue.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 0 B      
  kvflowcontrol.send_queue.prevent.count                            | 1 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 1.0 MiB  
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 1.0 MiB  


-- Observe the total tracked tokens per-stream on n1, n3's should track the latest write
-- as well.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
    FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 0 B                   
  75       | 3        | 6.0 MiB               


-- (Allowing below-raft admission to proceed on n1, n2, and n3. Note that n2 is still down.)
-- [n1(enabled),n2(enabled),n3(enabled)]


-- Flow token metrics from n1.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%tokens%'
ORDER BY name ASC;

  kvflowcontrol.tokens.eval.elastic.available                       | 6.0 MiB  
  kvflowcontrol.tokens.eval.elastic.deducted                        | 17 MiB   
  kvflowcontrol.tokens.eval.elastic.returned                        | 17 MiB   
  kvflowcontrol.tokens.eval.elastic.returned.disconnect             | 1.0 MiB  
  kvflowcontrol.tokens.eval.elastic.unaccounted                     | 0 B      
  kvflowcontrol.tokens.eval.regular.available                       | 12 MiB   
  kvflowcontrol.tokens.eval.regular.deducted                        | 16 MiB   
  kvflowcontrol.tokens.eval.regular.returned                        | 16 MiB   
  kvflowcontrol.tokens.eval.regular.returned.disconnect             | 1.0 MiB  
  kvflowcontrol.tokens.eval.regular.unaccounted                     | 0 B      
  kvflowcontrol.tokens.send.elastic.available                       | 6.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted                        | 17 MiB   
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.returned                        | 17 MiB   
  kvflowcontrol.tokens.send.elastic.returned.disconnect             | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.unaccounted                     | 0 B      
  kvflowcontrol.tokens.send.regular.available                       | 12 MiB   
  kvflowcontrol.tokens.send.regular.deducted                        | 16 MiB   
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 1.0 MiB  
  kvflowcontrol.tokens.send.regular.returned                        | 16 MiB   
  kvflowcontrol.tokens.send.regular.returned.disconnect             | 1.0 MiB  
  kvflowcontrol.tokens.send.regular.unaccounted                     | 0 B      


-- Per-store tokens available from n1.
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 


-- (Starting n2.)


-- There should now be 3 connected streams again.
SELECT range_id, count(*) AS streams
    FROM crdb_internal.kv_flow_control_handles_v2
GROUP BY (range_id)
ORDER BY streams DESC;

  range_id | stream_count  
-----------+---------------
  75       | 3             


-- (Adding VOTER to n4 and n5.)


-- Now, after adding n4,n5, there should be 5 connected streams.
-- [n1,n2,n3,n4,n5]
SELECT range_id, count(*) AS streams
    FROM crdb_internal.kv_flow_control_handles_v2
GROUP BY (range_id)
ORDER BY streams DESC;

  range_id | stream_count  
-----------+---------------
  75       | 5             


-- Per-store tokens available from n1.
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  4        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  5        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 


-- (Issuing 4x1MiB regular, 5x replicated write that's not admitted.)


-- From n1. We should expect to see the unblocked streams quickly
-- untrack as admission is allowed (so not observed here), while n4,n5 will continue
-- to track as they are blocked from admitting (logically).
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  4        | 0 B                    | -2.0 MiB               | 0 B                    | -2.0 MiB                
  5        | 0 B                    | -2.0 MiB               | 0 B                    | -2.0 MiB                
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
    FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 0 B                   
  75       | 2        | 0 B                   
  75       | 3        | 0 B                   
  75       | 4        | 4.0 MiB               
  75       | 5        | 4.0 MiB               


-- (Issuing 1x1MiB regular, 5x replicated write that's not admitted.)


-- Send queue and flow token metrics from n1. The 1 MiB write should be queued
-- for n4,n5, while the quorum (n1,n2,n3) proceeds.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 2.0 MiB  
  kvflowcontrol.send_queue.prevent.count                            | 1 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 1.0 MiB  
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 1.0 MiB  
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  4        | 0 B                    | -3.0 MiB               | 0 B                    | -2.0 MiB                
  5        | 0 B                    | -3.0 MiB               | 0 B                    | -2.0 MiB                


-- (Allowing below-raft admission to proceed on n4 and n5.)
-- [n1(enabled),n2(enabled),n3(enabled),n4(enabled),n5(enabled)]


-- Per-store tokens available from n1. Expect these to return to the same as 
-- the initial state.
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  4        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  5        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 


-- (Blocking below-raft admission on [n2,n3,n4,n5].)


-- (Issuing 4x1MiB regular, 5x replicated write that's not admitted.)


-- Send queue and flow token metrics from n1. The 4 MiB write should not be
-- queued, but instead exhaust all available regular eval and send tokens across
-- each stream, except s1 (as admission is not blocked).
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 0 B      
  kvflowcontrol.send_queue.prevent.count                            | 1 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 1.0 MiB  
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 1.0 MiB  
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 0 B                    | -2.0 MiB               | 0 B                    | -2.0 MiB                
  3        | 0 B                    | -2.0 MiB               | 0 B                    | -2.0 MiB                
  4        | 0 B                    | -2.0 MiB               | 0 B                    | -2.0 MiB                
  5        | 0 B                    | -2.0 MiB               | 0 B                    | -2.0 MiB                


-- (Enabling wait-for-eval bypass.)


-- (Issuing 1x1MiB regular, 5x replicated write that's not admitted.)


-- (Disabling wait-for-eval bypass.)


-- Observe the total tracked tokens per-stream on n1. We should expect to see the
-- 1 MiB write being tracked across a quorum of streams, while the 4 MiB write
-- is tracked across each stream (except s1). Two(/4 non-leader) replica send 
-- streams should be prevented from forming a send queue and have higher tracked
-- tokens than the other two.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
    FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 0 B                   
  75       | 2        | 4.0 MiB               
  75       | 3        | 4.0 MiB               
  75       | 4        | 5.0 MiB               
  75       | 5        | 5.0 MiB               


-- Send queue and flow token metrics from n1.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 2.0 MiB  
  kvflowcontrol.send_queue.prevent.count                            | 3 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 3.0 MiB  
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 3.0 MiB  
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 0 B                    | -3.0 MiB               | 0 B                    | -2.0 MiB                
  3        | 0 B                    | -3.0 MiB               | 0 B                    | -2.0 MiB                
  4        | -1.0 MiB               | -3.0 MiB               | -1.0 MiB               | -3.0 MiB                
  5        | -1.0 MiB               | -3.0 MiB               | -1.0 MiB               | -3.0 MiB                


-- (Allowing below-raft admission on [n1,n2,n3,n4,n5].)


-- Send queue and flow token metrics from n1. All tokens should be returned.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 0 B      
  kvflowcontrol.send_queue.prevent.count                            | 3 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 1.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 3.0 MiB  
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 3.0 MiB  
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  4        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  5        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
----
----

# vim:ft=sql
