echo
----
----
-- We will exhaust the tokens across all streams while admission is blocked,
-- using a single 16 MiB (deduction, the write itself is small) write. Then,
-- we will write a thousand or so entries (@ 4KiB deduction) which should be
-- queued towards one of the replica send streams, while the other has a send
-- queue prevented from forming. Lastly, we will unblock admission and stress
-- the raft in-flights tracker as the queue is drained.


-- Initial per-store tokens available from n1.
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 16 MiB                 | 8.0 MiB                | 16 MiB                 | 8.0 MiB                 
  2        | 16 MiB                 | 8.0 MiB                | 16 MiB                 | 8.0 MiB                 
  3        | 16 MiB                 | 8.0 MiB                | 16 MiB                 | 8.0 MiB                 


-- (Blocking below-raft admission on [n1,n2,n3].)


-- (Issuing 16MiB regular write that's not admitted.)


-- Per-store tokens available from n1, these should reflect the prior
-- large write.
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 0 B                    | -8.0 MiB               | 0 B                    | -8.0 MiB                
  2        | 0 B                    | -8.0 MiB               | 0 B                    | -8.0 MiB                
  3        | 0 B                    | -8.0 MiB               | 0 B                    | -8.0 MiB                


-- Observe the total tracked tokens per-stream on n1, these should also reflect the
-- large write.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
    FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 16 MiB                
  75       | 2        | 16 MiB                
  75       | 3        | 16 MiB                


-- (Enabling wait-for-eval bypass.)


-- (Issuing 1024x4KiB(=4MiB) regular writes that are not admitted.)


-- (Disabling wait-for-eval bypass.)


-- Per-store tokens available from n1, these should reflect the deducted
-- tokens from preventing send queue formation.
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | -4.0 MiB               | -12 MiB                | -4.0 MiB               | -12 MiB                 
  2        | 0 B                    | -12 MiB                | 0 B                    | -8.0 MiB                
  3        | -4.0 MiB               | -12 MiB                | -4.0 MiB               | -12 MiB                 


-- Send queue metrics from n1, a send queue should have formed for one of the
-- replica send streams, while the other (non-leader stream) should have been
-- prevented from forming. It should be 1024*4KiB=4MiB in size.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 4.0 MiB  
  kvflowcontrol.send_queue.prevent.count                            | 1.0 KiB  
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 4.0 MiB  
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 4.0 MiB  


-- Observe the total tracked tokens per-stream on n1, one of the three
-- streams will only be tracking the 16 MiB write, while the other two will
-- track the 1024x4KiB writes as well.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
    FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 20 MiB                
  75       | 2        | 16 MiB                
  75       | 3        | 20 MiB                


-- (Allowing below-raft admission to proceed on [n1,n2,n3].)


-- Send queue and flow token metrics from n1. All tokens should be returned.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 0 B      
  kvflowcontrol.send_queue.prevent.count                            | 1.0 KiB  
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 4.0 MiB  
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 4.0 MiB  
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 16 MiB                 | 8.0 MiB                | 16 MiB                 | 8.0 MiB                 
  2        | 16 MiB                 | 8.0 MiB                | 16 MiB                 | 8.0 MiB                 
  3        | 16 MiB                 | 8.0 MiB                | 16 MiB                 | 8.0 MiB                 
----
----


# vim:ft=sql
