echo
----
----
(Rangefeed on n3)


-- We will exhaust the tokens across all streams while admission is blocked on
-- n3, using 4x1 MiB (deduction, the write itself is small) writes. Then,
-- we will write 1 MiB to the range and wait for the closedTS to fall
-- behind on n3. We expect that the closedTS falling behind will trigger
-- an error that is returned to the mux rangefeed client, which will in turn 
-- allows the rangefeed  request to be re-routed to another replica.


-- Observe the server-side rangefeed cancellation metric on n3, before a send
-- queue develops, it should be zero:
SELECT
    name,
    value
  FROM crdb_internal.node_metrics
  WHERE name LIKE 'kv.rangefeed.closed_timestamp.slow_ranges.cancelled'
  ORDER BY name ASC;

  kv.rangefeed.closed_timestamp.slow_ranges.cancelled | 0  


(Sending 1 MiB put request to develop a send queue)


(Sent 1 MiB put request)


-- Send queue metrics from n1, n3's send queue should have 1 MiB for s3.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 1.0 MiB  
  kvflowcontrol.send_queue.prevent.count                            | 0 B      
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B      
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B      
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B      
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B      


-- Observe the total tracked tokens per-stream on n1, s3's entries will still
-- be tracked here.
SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8)
    FROM crdb_internal.kv_flow_control_handles_v2

  range_id | store_id | total_tracked_tokens  
-----------+----------+-----------------------
  75       | 1        | 0 B                   
  75       | 2        | 0 B                   
  75       | 3        | 4.0 MiB               


-- Per-store tokens available from n1, these should reflect the lack of tokens 
-- for s3.
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 0 B                    | -3.0 MiB               | 0 B                    | -2.0 MiB                


(Rangefeed moved to n1)


-- Observe the server-side rangefeed cancellation metric increased on n3:
SELECT
    name,
    value
  FROM crdb_internal.node_metrics
  WHERE name LIKE 'kv.rangefeed.closed_timestamp.slow_ranges.cancelled'
  ORDER BY name ASC;

  kv.rangefeed.closed_timestamp.slow_ranges.cancelled | 1  


-- (Allowing below-raft admission to proceed on n3.)


-- Send queue and flow token metrics from n1. All tokens should be returned.
SELECT name, crdb_internal.humanize_bytes(value::INT8)
    FROM crdb_internal.node_metrics
   WHERE name LIKE '%kvflowcontrol%send_queue%'
     AND name != 'kvflowcontrol.send_queue.count'
ORDER BY name ASC;

  kvflowcontrol.send_queue.bytes                                    | 0 B  
  kvflowcontrol.send_queue.prevent.count                            | 0 B  
  kvflowcontrol.send_queue.scheduled.deducted_bytes                 | 0 B  
  kvflowcontrol.send_queue.scheduled.force_flush                    | 0 B  
  kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B  
  kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue     | 0 B  
  kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue     | 0 B  
SELECT store_id,
     crdb_internal.humanize_bytes(available_eval_regular_tokens),
     crdb_internal.humanize_bytes(available_eval_elastic_tokens),
     crdb_internal.humanize_bytes(available_send_regular_tokens),
     crdb_internal.humanize_bytes(available_send_elastic_tokens)
  FROM crdb_internal.kv_flow_controller_v2
  ORDER BY store_id ASC;

  store_id | eval_regular_available | eval_elastic_available | send_regular_available | send_elastic_available  
-----------+------------------------+------------------------+------------------------+-------------------------
  1        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  2        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
  3        | 4.0 MiB                | 2.0 MiB                | 4.0 MiB                | 2.0 MiB                 
----
----

# vim:ft=sql
