# This test demonstrates the effect of delayed commit on a follower node after a
# network hiccup between the leader and this follower that causes withdrawn
# support.

# Skip logging the boilerplate. Set up a raft group of 3 nodes, and elect node 1
# as the leader. Nodes 2 and 3 are the followers.
log-level none
----
ok

add-nodes 3 voters=(1,2,3) index=10
----
ok

campaign 1
----
ok

stabilize
----
ok

# Propose a couple of entries.
propose 1 data1
----
ok

propose 1 data2
----
ok

process-ready 1
----
ok

# The interesting part starts below.
log-level debug
----
ok

deliver-msgs 2 3
----
1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "data1"]
1->2 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "data2"]
1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "data1"]
1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "data2"]

process-ready 3
----
Ready MustSync=true:
Entries:
1/12 EntryNormal "data1"
1/13 EntryNormal "data2"
Messages:
3->1 MsgAppResp Term:1 Log:0/12 Commit:11
3->1 MsgAppResp Term:1 Log:0/13 Commit:11

# Suppose there is a network blip which prevents the leader learning that the
# follower 3 has appended the proposed entries to the log.
deliver-msgs drop=(1)
----
dropped: 3->1 MsgAppResp Term:1 Log:0/12 Commit:11
dropped: 3->1 MsgAppResp Term:1 Log:0/13 Commit:11

# In the meantime, the entries are committed, and the leader sends the commit
# index to all the followers.
stabilize 1 2
----
> 2 handling Ready
  Ready MustSync=true:
  Entries:
  1/12 EntryNormal "data1"
  1/13 EntryNormal "data2"
  Messages:
  2->1 MsgAppResp Term:1 Log:0/12 Commit:11
  2->1 MsgAppResp Term:1 Log:0/13 Commit:11
> 1 receiving messages
  2->1 MsgAppResp Term:1 Log:0/12 Commit:11
  2->1 MsgAppResp Term:1 Log:0/13 Commit:11
> 1 handling Ready
  Ready MustSync=true:
  HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1
  CommittedEntries:
  1/12 EntryNormal "data1"
  1/13 EntryNormal "data2"
  Messages:
  1->2 MsgApp Term:1 Log:1/13 Commit:12
  1->3 MsgApp Term:1 Log:1/13 Commit:12
  1->2 MsgApp Term:1 Log:1/13 Commit:13
  1->3 MsgApp Term:1 Log:1/13 Commit:13
> 2 receiving messages
  1->2 MsgApp Term:1 Log:1/13 Commit:12
  1->2 MsgApp Term:1 Log:1/13 Commit:13
> 2 handling Ready
  Ready MustSync=true:
  HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1
  CommittedEntries:
  1/12 EntryNormal "data1"
  1/13 EntryNormal "data2"
  Messages:
  2->1 MsgAppResp Term:1 Log:0/13 Commit:12
  2->1 MsgAppResp Term:1 Log:0/13 Commit:13
> 1 receiving messages
  2->1 MsgAppResp Term:1 Log:0/13 Commit:12
  2->1 MsgAppResp Term:1 Log:0/13 Commit:13

# The network blip prevents the follower 3 from learning that the previously
# appended entries are now committed.
deliver-msgs drop=(3)
----
dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:12
dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:13

withdraw-support 3 1
----
  1 2 3
1 1 1 1
2 1 1 1
3 x 1 1

status 1
----
1: StateReplicate match=13 next=14 sentCommit=11 matchCommit=11
2: StateReplicate match=13 next=14 sentCommit=13 matchCommit=13
3: StateReplicate match=11 next=14 sentCommit=13 matchCommit=11 inflight=2

tick-heartbeat 1
----
ok

# Although the leader knows that the follower 3 is lagging behind it doesn't
# send a MsgApp because follower 3 doesn't support the leader.
process-ready 1
----
<empty Ready>

# Now that follower 3's store supports the leader's store, will send a MsgApp on
# the next heartbeat timeout.
grant-support 3 1
----
  1 2 3
1 2 1 1
2 1 1 1
3 2 1 1

tick-heartbeat 1
----
ok

process-ready 1
----
Ready MustSync=true:
HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:2
Messages:
1->3 MsgFortifyLeader Term:1 Log:0/0
1->3 MsgApp Term:1 Log:1/13 Commit:13

stabilize
----
> 3 receiving messages
  1->3 MsgFortifyLeader Term:1 Log:0/0
  1->3 MsgApp Term:1 Log:1/13 Commit:13
> 3 handling Ready
  Ready MustSync=true:
  HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:2
  CommittedEntries:
  1/12 EntryNormal "data1"
  1/13 EntryNormal "data2"
  Messages:
  3->1 MsgFortifyLeaderResp Term:1 Log:0/0 LeadEpoch:2
  3->1 MsgAppResp Term:1 Log:0/13 Commit:13
> 1 receiving messages
  3->1 MsgFortifyLeaderResp Term:1 Log:0/0 LeadEpoch:2
  3->1 MsgAppResp Term:1 Log:0/13 Commit:13
> 1 handling Ready
  Ready MustSync=false:
  Messages:
  1->3 MsgApp Term:1 Log:1/13 Commit:13
> 3 receiving messages
  1->3 MsgApp Term:1 Log:1/13 Commit:13
> 3 handling Ready
  Ready MustSync=false:
  Messages:
  3->1 MsgAppResp Term:1 Log:0/13 Commit:13
> 1 receiving messages
  3->1 MsgAppResp Term:1 Log:0/13 Commit:13
