# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which
# -- are subject to admission control are encoded as `raftpb.LowPri`,
# -- regardless of their original priority; to avoid the overhead of
# -- deserializing the raft admission metadata. Therefore, as the underlying
# -- test is shared between the v1 and v2 encoding testdata files, the reader
# -- should interpret any comments referring to regular tokens as referring to
# -- elastic token.
echo
----
----
-- Flow token metrics from n1 after issuing 1 1MiB 5x replicated write
-- that's not admitted. Since this test is ignoring crashed nodes for token
-- deduction purposes, we see a deduction of 5MiB tokens.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%tokens%'
ORDER BY name ASC;

  kvflowcontrol.tokens.eval.elastic.available                       | 35 MiB   
  kvflowcontrol.tokens.eval.elastic.deducted                        | 5.0 MiB  
  kvflowcontrol.tokens.eval.elastic.returned                        | 0 B      
  kvflowcontrol.tokens.eval.elastic.returned.disconnect             | 0 B      
  kvflowcontrol.tokens.eval.elastic.unaccounted                     | 0 B      
  kvflowcontrol.tokens.eval.regular.available                       | 80 MiB   
  kvflowcontrol.tokens.eval.regular.deducted                        | 0 B      
  kvflowcontrol.tokens.eval.regular.returned                        | 0 B      
  kvflowcontrol.tokens.eval.regular.returned.disconnect             | 0 B      
  kvflowcontrol.tokens.eval.regular.unaccounted                     | 0 B      
  kvflowcontrol.tokens.send.elastic.available                       | 35 MiB   
  kvflowcontrol.tokens.send.elastic.deducted                        | 5.0 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.elastic.returned                        | 0 B      
  kvflowcontrol.tokens.send.elastic.returned.disconnect             | 0 B      
  kvflowcontrol.tokens.send.elastic.unaccounted                     | 0 B      
  kvflowcontrol.tokens.send.regular.available                       | 80 MiB   
  kvflowcontrol.tokens.send.regular.deducted                        | 0 B      
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.returned                        | 0 B      
  kvflowcontrol.tokens.send.regular.returned.disconnect             | 0 B      
  kvflowcontrol.tokens.send.regular.unaccounted                     | 0 B      


-- Observe the total tracked tokens per-stream on n1. 1MiB is tracked for n1-n5.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
	   FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 1.0 MiB               
  75       | 2        | 1.0 MiB               
  75       | 3        | 1.0 MiB               
  75       | 4        | 1.0 MiB               
  75       | 5        | 1.0 MiB               


-- (Killing n2 and n3, but preventing their tokens from being returned +
-- artificially allowing tokens to get deducted.)


-- Observe the total tracked tokens per-stream on n1. 1MiB is (still) tracked
-- for n1-n5, because they are not in StateSnapshot yet and have likely been
-- in StateProbe for less than the close timer.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
	   FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 1.0 MiB               
  75       | 2        | 1.0 MiB               
  75       | 3        | 1.0 MiB               
  75       | 4        | 1.0 MiB               
  75       | 5        | 1.0 MiB               


-- (Issuing another 1MiB of 5x replicated writes while n2 and n3 are down and
-- below-raft admission is paused.)


-- Flow token metrics from n1 after issuing 1 1MiB 5x replicated write
-- that's not admitted. We'll have deducted another 5*1MiB=5MiB worth of tokens.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%tokens%'
ORDER BY name ASC;

  kvflowcontrol.tokens.eval.elastic.available                       | 30 MiB  
  kvflowcontrol.tokens.eval.elastic.deducted                        | 10 MiB  
  kvflowcontrol.tokens.eval.elastic.returned                        | 0 B     
  kvflowcontrol.tokens.eval.elastic.returned.disconnect             | 0 B     
  kvflowcontrol.tokens.eval.elastic.unaccounted                     | 0 B     
  kvflowcontrol.tokens.eval.regular.available                       | 80 MiB  
  kvflowcontrol.tokens.eval.regular.deducted                        | 0 B     
  kvflowcontrol.tokens.eval.regular.returned                        | 0 B     
  kvflowcontrol.tokens.eval.regular.returned.disconnect             | 0 B     
  kvflowcontrol.tokens.eval.regular.unaccounted                     | 0 B     
  kvflowcontrol.tokens.send.elastic.available                       | 30 MiB  
  kvflowcontrol.tokens.send.elastic.deducted                        | 10 MiB  
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B     
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B     
  kvflowcontrol.tokens.send.elastic.returned                        | 0 B     
  kvflowcontrol.tokens.send.elastic.returned.disconnect             | 0 B     
  kvflowcontrol.tokens.send.elastic.unaccounted                     | 0 B     
  kvflowcontrol.tokens.send.regular.available                       | 80 MiB  
  kvflowcontrol.tokens.send.regular.deducted                        | 0 B     
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B     
  kvflowcontrol.tokens.send.regular.returned                        | 0 B     
  kvflowcontrol.tokens.send.regular.returned.disconnect             | 0 B     
  kvflowcontrol.tokens.send.regular.unaccounted                     | 0 B     


-- Observe the total tracked tokens per-stream on n1. 2MiB is tracked for n1-n5;
-- see last comment for an explanation why we're still deducting for n2, n3.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
	   FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 2.0 MiB               
  75       | 2        | 2.0 MiB               
  75       | 3        | 2.0 MiB               
  75       | 4        | 2.0 MiB               
  75       | 5        | 2.0 MiB               


-- (Truncating raft log.)


-- (Restarting n2 and n3.)


-- Flow token metrics from n1 after restarting n2 and n3. We've returned the
-- 2MiB previously held by those nodes (2MiB each). We're reacting to it's raft
-- progress state, noting that since we've truncated our log, we need to catch
-- it up via snapshot. So we release all held tokens.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%tokens%'
ORDER BY name ASC;

  kvflowcontrol.tokens.eval.elastic.available                       | 34 MiB   
  kvflowcontrol.tokens.eval.elastic.deducted                        | 10 MiB   
  kvflowcontrol.tokens.eval.elastic.returned                        | 4.0 MiB  
  kvflowcontrol.tokens.eval.elastic.returned.disconnect             | 4.0 MiB  
  kvflowcontrol.tokens.eval.elastic.unaccounted                     | 0 B      
  kvflowcontrol.tokens.eval.regular.available                       | 80 MiB   
  kvflowcontrol.tokens.eval.regular.deducted                        | 0 B      
  kvflowcontrol.tokens.eval.regular.returned                        | 0 B      
  kvflowcontrol.tokens.eval.regular.returned.disconnect             | 0 B      
  kvflowcontrol.tokens.eval.regular.unaccounted                     | 0 B      
  kvflowcontrol.tokens.send.elastic.available                       | 34 MiB   
  kvflowcontrol.tokens.send.elastic.deducted                        | 10 MiB   
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.elastic.returned                        | 4.0 MiB  
  kvflowcontrol.tokens.send.elastic.returned.disconnect             | 4.0 MiB  
  kvflowcontrol.tokens.send.elastic.unaccounted                     | 0 B      
  kvflowcontrol.tokens.send.regular.available                       | 80 MiB   
  kvflowcontrol.tokens.send.regular.deducted                        | 0 B      
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.returned                        | 0 B      
  kvflowcontrol.tokens.send.regular.returned.disconnect             | 0 B      
  kvflowcontrol.tokens.send.regular.unaccounted                     | 0 B      


-- Observe the total tracked tokens per-stream on n1. There's nothing tracked
-- for n2 and n3 anymore.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
   FROM crdb_internal.kv_flow_control_handles_v2
   WHERE total_tracked_tokens > 0

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 2.0 MiB               
  75       | 4        | 2.0 MiB               
  75       | 5        | 2.0 MiB               


-- (Allow below-raft admission to proceed.)


-- Flow token metrics from n1 after work gets admitted. We see the remaining
-- 6MiB of tokens returned.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%tokens%'
ORDER BY name ASC;

  kvflowcontrol.tokens.eval.elastic.available                       | 40 MiB   
  kvflowcontrol.tokens.eval.elastic.deducted                        | 10 MiB   
  kvflowcontrol.tokens.eval.elastic.returned                        | 10 MiB   
  kvflowcontrol.tokens.eval.elastic.returned.disconnect             | 4.0 MiB  
  kvflowcontrol.tokens.eval.elastic.unaccounted                     | 0 B      
  kvflowcontrol.tokens.eval.regular.available                       | 80 MiB   
  kvflowcontrol.tokens.eval.regular.deducted                        | 0 B      
  kvflowcontrol.tokens.eval.regular.returned                        | 0 B      
  kvflowcontrol.tokens.eval.regular.returned.disconnect             | 0 B      
  kvflowcontrol.tokens.eval.regular.unaccounted                     | 0 B      
  kvflowcontrol.tokens.send.elastic.available                       | 40 MiB   
  kvflowcontrol.tokens.send.elastic.deducted                        | 10 MiB   
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.elastic.returned                        | 10 MiB   
  kvflowcontrol.tokens.send.elastic.returned.disconnect             | 4.0 MiB  
  kvflowcontrol.tokens.send.elastic.unaccounted                     | 0 B      
  kvflowcontrol.tokens.send.regular.available                       | 80 MiB   
  kvflowcontrol.tokens.send.regular.deducted                        | 0 B      
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.returned                        | 0 B      
  kvflowcontrol.tokens.send.regular.returned.disconnect             | 0 B      
  kvflowcontrol.tokens.send.regular.unaccounted                     | 0 B      


-- Observe the total tracked tokens per-stream on n1; there should be nothing.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
   FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 0 B                   
  75       | 2        | 0 B                   
  75       | 3        | 0 B                   
  75       | 4        | 0 B                   
  75       | 5        | 0 B                   


-- Another view of tokens, using /inspectz-backed vtables.
SELECT store_id,
	   crdb_internal.humanize_bytes(available_eval_regular_tokens),
	   crdb_internal.humanize_bytes(available_eval_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
 ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available  
-----------+------------------------+-------------------------
  1        | 16 MiB                 | 8.0 MiB                 
  2        | 16 MiB                 | 8.0 MiB                 
  3        | 16 MiB                 | 8.0 MiB                 
  4        | 16 MiB                 | 8.0 MiB                 
  5        | 16 MiB                 | 8.0 MiB                 
----
----

# vim:ft=sql
