# This test ensures that MsgApp stream to a follower is paused when the
# in-flight state exceeds the configured limits. This is a regression test for
# the issue fixed by https://github.com/etcd-io/etcd/pull/14633.

# Turn off output during the setup of the test.
log-level none
----
ok

# Start with 3 nodes, with a limited in-flight capacity.
add-nodes 3 voters=(1,2,3) index=10 inflight=3
----
ok

campaign 1
----
ok

stabilize
----
ok

# Propose 3 entries.
propose 1 prop_1_12
----
ok

propose 1 prop_1_13
----
ok

propose 1 prop_1_14
----
ok

# Store entries and send proposals.
process-ready 1
----
ok

# Re-enable log messages.
log-level debug
----
ok

# Expect that in-flight tracking to nodes 2 and 3 is saturated.
status 1
----
1: StateReplicate match=14 next=15 sentCommit=11 matchCommit=11
2: StateReplicate match=11 next=15 sentCommit=11 matchCommit=11 paused inflight=3[full]
3: StateReplicate match=11 next=15 sentCommit=11 matchCommit=11 paused inflight=3[full]

log-level none
----
ok

# Commit entries between nodes 1 and 2.
stabilize 1 2
----
ok

log-level debug
----
ok

# Expect that the entries are committed and stored on nodes 1 and 2.
status 1
----
1: StateReplicate match=14 next=15 sentCommit=11 matchCommit=11
2: StateReplicate match=14 next=15 sentCommit=14 matchCommit=14
3: StateReplicate match=11 next=15 sentCommit=14 matchCommit=11 paused inflight=3[full]

# Drop append messages to node 3.
deliver-msgs drop=3
----
dropped: 1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1_12"]
dropped: 1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_1_13"]
dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:11 Entries:[1/14 EntryNormal "prop_1_14"]
dropped: 1->3 MsgApp Term:1 Log:1/14 Commit:12
dropped: 1->3 MsgApp Term:1 Log:1/14 Commit:13
dropped: 1->3 MsgApp Term:1 Log:1/14 Commit:14


# Repeat committing 3 entries.
propose 1 prop_1_15
----
ok

propose 1 prop_1_16
----
ok

propose 1 prop_1_17
----
ok

# In-flight tracking to nodes 2 and 3 is saturated, but node 3 is behind.
status 1
----
1: StateReplicate match=14 next=18 sentCommit=11 matchCommit=11
2: StateReplicate match=14 next=18 sentCommit=14 matchCommit=14 paused inflight=3[full]
3: StateReplicate match=11 next=15 sentCommit=14 matchCommit=11 paused inflight=3[full]

log-level none
----
ok

# Commit entries between nodes 1 and 2 again.
stabilize 1 2
----
ok

log-level debug
----
ok

# Expect that the entries are committed and stored only on nodes 1 and 2.
status 1
----
1: StateReplicate match=17 next=18 sentCommit=14 matchCommit=14
2: StateReplicate match=17 next=18 sentCommit=17 matchCommit=17
3: StateReplicate match=11 next=15 sentCommit=14 matchCommit=11 paused inflight=3[full]

# On the next heartbeat timeout, node 1 sends an empty MsgApp to a throttled
# node 3 because it hasn't yet replied to a single MsgApp, and the in-flight
# tracker is still saturated.
tick-heartbeat 1
----
ok

stabilize 1
----
> 1 handling Ready
  Ready MustSync=false:
  Messages:
  1->3 MsgApp Term:1 Log:1/14 Commit:17

# Node 3 finally receives a MsgApp, but there was a gap, so it rejects it.
stabilize 2 3
----
> 3 receiving messages
  1->3 MsgApp Term:1 Log:1/14 Commit:17
  DEBUG 3 [logterm: 0, index: 14] rejected MsgApp [logterm: 1, index: 14] from 1
> 3 handling Ready
  Ready MustSync=false:
  Messages:
  3->1 MsgAppResp Term:1 Log:1/14 Rejected (Hint: 11) Commit:11

# Node 1 receives the rejection and adjusts the MsgApp sent to node 3.
stabilize 1
----
> 1 receiving messages
  3->1 MsgAppResp Term:1 Log:1/14 Rejected (Hint: 11) Commit:11
  DEBUG 1 received MsgAppResp(rejected, hint: (index 11, term 1)) from 3 for index 14
  DEBUG 1 decreased progress of 3 to [StateReplicate match=11 next=12 sentCommit=11 matchCommit=11 paused inflight=3[full]]
> 1 handling Ready
  Ready MustSync=false:
  Messages:
  1->3 MsgApp Term:1 Log:1/11 Commit:17 Entries:[
    1/12 EntryNormal "prop_1_12"
    1/13 EntryNormal "prop_1_13"
    1/14 EntryNormal "prop_1_14"
    1/15 EntryNormal "prop_1_15"
    1/16 EntryNormal "prop_1_16"
    1/17 EntryNormal "prop_1_17"
  ]

stabilize 1 3
----
> 3 receiving messages
  1->3 MsgApp Term:1 Log:1/11 Commit:17 Entries:[
    1/12 EntryNormal "prop_1_12"
    1/13 EntryNormal "prop_1_13"
    1/14 EntryNormal "prop_1_14"
    1/15 EntryNormal "prop_1_15"
    1/16 EntryNormal "prop_1_16"
    1/17 EntryNormal "prop_1_17"
  ]
> 3 handling Ready
  Ready MustSync=true:
  HardState Term:1 Vote:1 Commit:17 Lead:1 LeadEpoch:1
  Entries:
  1/12 EntryNormal "prop_1_12"
  1/13 EntryNormal "prop_1_13"
  1/14 EntryNormal "prop_1_14"
  1/15 EntryNormal "prop_1_15"
  1/16 EntryNormal "prop_1_16"
  1/17 EntryNormal "prop_1_17"
  CommittedEntries:
  1/12 EntryNormal "prop_1_12"
  1/13 EntryNormal "prop_1_13"
  1/14 EntryNormal "prop_1_14"
  1/15 EntryNormal "prop_1_15"
  1/16 EntryNormal "prop_1_16"
  1/17 EntryNormal "prop_1_17"
  Messages:
  3->1 MsgAppResp Term:1 Log:0/17 Commit:17
> 1 receiving messages
  3->1 MsgAppResp Term:1 Log:0/17 Commit:17

# Eventually all nodes catch up on the committed state.
status 1
----
1: StateReplicate match=17 next=18 sentCommit=14 matchCommit=14
2: StateReplicate match=17 next=18 sentCommit=17 matchCommit=17
3: StateReplicate match=17 next=18 sentCommit=17 matchCommit=17
