# This test demonstrates that the leader tracks the followers' commit index
# and tries to advance it if it's stale.

# Skip logging the boilerplate. Set up a raft group of 3 nodes, and elect node 1
# as the leader. Nodes 2 and 3 are the followers.
log-level none
----
ok

add-nodes 3 voters=(1, 2, 3) index=10
----
ok

campaign 1
----
ok

stabilize
----
ok

# Propose a couple of entries.
propose 1 data1
----
ok

propose 1 data2
----
ok

process-ready 1
----
ok

# The interesting part starts below.
log-level debug
----
ok

deliver-msgs 2 3
----
1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "data1"]
1->2 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "data2"]
1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "data1"]
1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "data2"]

process-ready 3
----
Ready MustSync=true:
Entries:
1/12 EntryNormal "data1"
1/13 EntryNormal "data2"
Messages:
3->1 MsgAppResp Term:1 Log:0/12 Commit:11
3->1 MsgAppResp Term:1 Log:0/13 Commit:11

# In the meantime, the entries are committed, and the leader sends the commit
# index to all the followers.
stabilize 1 2
----
> 2 handling Ready
  Ready MustSync=true:
  Entries:
  1/12 EntryNormal "data1"
  1/13 EntryNormal "data2"
  Messages:
  2->1 MsgAppResp Term:1 Log:0/12 Commit:11
  2->1 MsgAppResp Term:1 Log:0/13 Commit:11
> 1 receiving messages
  3->1 MsgAppResp Term:1 Log:0/12 Commit:11
  3->1 MsgAppResp Term:1 Log:0/13 Commit:11
  2->1 MsgAppResp Term:1 Log:0/12 Commit:11
  2->1 MsgAppResp Term:1 Log:0/13 Commit:11
> 1 handling Ready
  Ready MustSync=true:
  HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1
  CommittedEntries:
  1/12 EntryNormal "data1"
  1/13 EntryNormal "data2"
  Messages:
  1->2 MsgApp Term:1 Log:1/13 Commit:12
  1->3 MsgApp Term:1 Log:1/13 Commit:12
  1->2 MsgApp Term:1 Log:1/13 Commit:13
  1->3 MsgApp Term:1 Log:1/13 Commit:13
> 2 receiving messages
  1->2 MsgApp Term:1 Log:1/13 Commit:12
  1->2 MsgApp Term:1 Log:1/13 Commit:13
> 2 handling Ready
  Ready MustSync=true:
  HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1
  CommittedEntries:
  1/12 EntryNormal "data1"
  1/13 EntryNormal "data2"
  Messages:
  2->1 MsgAppResp Term:1 Log:0/13 Commit:12
  2->1 MsgAppResp Term:1 Log:0/13 Commit:13
> 1 receiving messages
  2->1 MsgAppResp Term:1 Log:0/13 Commit:12
  2->1 MsgAppResp Term:1 Log:0/13 Commit:13

# The network blip prevents the follower 3 from learning that the previously
# appended entries are now committed.
deliver-msgs drop=(3)
----
dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:12
dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:13

# The network blip ends here.

status 1
----
1: StateReplicate match=13 next=14 sentCommit=11 matchCommit=11
2: StateReplicate match=13 next=14 sentCommit=13 matchCommit=13
3: StateReplicate match=13 next=14 sentCommit=13 matchCommit=11

# Wait for the next heartbeat response.
tick-heartbeat 1
----
ok

process-ready 1
----
Ready MustSync=false:
Messages:
1->3 MsgApp Term:1 Log:1/13 Commit:13

# On the next MsgApp sent to follower 3, the leader will include that the
# commit index is 13. Notice that the leader doesn't send MsgApp to follower 2
# because it knows that it has the latest commit index.
stabilize 1 2 3
----
> 3 receiving messages
  1->3 MsgApp Term:1 Log:1/13 Commit:13
> 3 handling Ready
  Ready MustSync=true:
  HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1
  CommittedEntries:
  1/12 EntryNormal "data1"
  1/13 EntryNormal "data2"
  Messages:
  3->1 MsgAppResp Term:1 Log:0/13 Commit:13
> 1 receiving messages
  3->1 MsgAppResp Term:1 Log:0/13 Commit:13

# The leader's sentCommit and the matchCommit remain stale even after stabilize.
# The reason is that the leader send itself a MsgAppResp when it receives a
# MsgProp. However, the leader doesn't send itself a MsgAppResp when it
# broadcasts a MsgApp to the followers. That's fine because the leader doesn't
# use its sentCommit/matchCommit for anything.
status 1
----
1: StateReplicate match=13 next=14 sentCommit=11 matchCommit=11
2: StateReplicate match=13 next=14 sentCommit=13 matchCommit=13
3: StateReplicate match=13 next=14 sentCommit=13 matchCommit=13

# If the commit index is up-to-date, no MsgApp will be sent.
tick-heartbeat 1
----
ok

stabilize 1 2 3
----
ok
