# Snapshot catch-up: a follower that falls behind the leader's COMPACTED log is caught up via
# InstallSnapshot. With a low snapshot threshold (3), isolate node 3, then propose enough commands
# that node 1 snapshots and compacts its log past node 3's match index. On recovery the leader can no
# longer send node 3 the missing entries (they are compacted away), so it sends an InstallSnapshot;
# node 3 installs it (a snapshot-installed event) and resumes from the snapshot's index.
add-nodes 3 voters=(1,2,3) snap=3
----
n1: created voters={1,2,3} term=0 commit=0 last=0
n2: created voters={1,2,3} term=0 commit=0 last=0
n3: created voters={1,2,3} term=0 commit=0 last=0

campaign 1
----
n1 candidate term=1
RequestVote term=1 last=0/0 prevote=false
RequestVote term=1 last=0/0 prevote=false

stabilize
----
> n2 recv RequestVote term=1 last=0/0 prevote=false
> n3 recv RequestVote term=1 last=0/0 prevote=false
> n2 ready
  VoteResponse term=1 prevote=false reject=false
> n3 ready
  VoteResponse term=1 prevote=false reject=false
> n1 recv VoteResponse term=1 prevote=false reject=false
> n1 recv VoteResponse term=1 prevote=false reject=false
> n1 ready
  Heartbeat term=1 commit=0
  Heartbeat term=1 commit=0
  AppendEntries term=1 prev=0/0 commit=0 entries=[1/1 empty]
  AppendEntries term=1 prev=0/0 commit=0 entries=[1/1 empty]
  n1 leader-changed term=1 lead=1
> n2 recv Heartbeat term=1 commit=0
> n3 recv Heartbeat term=1 commit=0
> n2 recv AppendEntries term=1 prev=0/0 commit=0 entries=[1/1 empty]
> n3 recv AppendEntries term=1 prev=0/0 commit=0 entries=[1/1 empty]
> n2 ready
  HeartbeatResponse term=1
  AppendResponse term=1 match=1
  n2 leader-changed term=1 lead=1
> n3 ready
  HeartbeatResponse term=1
  AppendResponse term=1 match=1
  n3 leader-changed term=1 lead=1
> n1 recv HeartbeatResponse term=1
> n1 recv AppendResponse term=1 match=1
> n1 recv HeartbeatResponse term=1
> n1 recv AppendResponse term=1 match=1
> n1 ready
  AppendEntries term=1 prev=0/0 commit=0 entries=[1/1 empty]
  AppendEntries term=1 prev=1/1 commit=1 entries=[]
  AppendEntries term=1 prev=0/0 commit=1 entries=[1/1 empty]
  AppendEntries term=1 prev=1/1 commit=1 entries=[]
> n2 recv AppendEntries term=1 prev=0/0 commit=0 entries=[1/1 empty]
> n2 recv AppendEntries term=1 prev=1/1 commit=1 entries=[]
> n3 recv AppendEntries term=1 prev=0/0 commit=1 entries=[1/1 empty]
> n3 recv AppendEntries term=1 prev=1/1 commit=1 entries=[]
> n2 ready
  AppendResponse term=1 match=1
  AppendResponse term=1 match=1
> n3 ready
  AppendResponse term=1 match=1
  AppendResponse term=1 match=1
> n1 recv AppendResponse term=1 match=1
> n1 recv AppendResponse term=1 match=1
> n1 recv AppendResponse term=1 match=1
> n1 recv AppendResponse term=1 match=1

isolate 3
----
n3 isolated

propose 1 a
----
AppendEntries term=1 prev=1/1 commit=1 entries=[1/2 normal 0x010000000000000061]

propose 1 b
----
AppendEntries term=1 prev=1/2 commit=1 entries=[1/3 normal 0x010000000000000062]

propose 1 c
----
AppendEntries term=1 prev=1/3 commit=1 entries=[1/4 normal 0x010000000000000063]

propose 1 d
----
AppendEntries term=1 prev=1/4 commit=1 entries=[1/5 normal 0x010000000000000064]

stabilize
----
> n2 recv AppendEntries term=1 prev=1/1 commit=1 entries=[1/2 normal 0x010000000000000061]
> n2 recv AppendEntries term=1 prev=1/2 commit=1 entries=[1/3 normal 0x010000000000000062]
> n2 recv AppendEntries term=1 prev=1/3 commit=1 entries=[1/4 normal 0x010000000000000063]
> n2 recv AppendEntries term=1 prev=1/4 commit=1 entries=[1/5 normal 0x010000000000000064]
> n2 ready
  AppendResponse term=1 match=2
  AppendResponse term=1 match=3
  AppendResponse term=1 match=4
  AppendResponse term=1 match=5
> n1 recv AppendResponse term=1 match=2
> n1 recv AppendResponse term=1 match=3
> n1 recv AppendResponse term=1 match=4
> n1 recv AppendResponse term=1 match=5
> n1 ready
  AppendEntries term=1 prev=1/5 commit=2 entries=[]
  AppendEntries term=1 prev=1/5 commit=3 entries=[]
  AppendEntries term=1 prev=1/5 commit=4 entries=[]
  AppendEntries term=1 prev=1/5 commit=5 entries=[]
  n1 applied index=2
  n1 applied index=3
  n1 applied index=4
  n1 applied index=5
> n2 recv AppendEntries term=1 prev=1/5 commit=2 entries=[]
> n2 recv AppendEntries term=1 prev=1/5 commit=3 entries=[]
> n2 recv AppendEntries term=1 prev=1/5 commit=4 entries=[]
> n2 recv AppendEntries term=1 prev=1/5 commit=5 entries=[]
> n2 ready
  AppendResponse term=1 match=5
  AppendResponse term=1 match=5
  AppendResponse term=1 match=5
  AppendResponse term=1 match=5
  n2 applied index=2
  n2 applied index=3
  n2 applied index=4
  n2 applied index=5
> n1 recv AppendResponse term=1 match=5
> n1 recv AppendResponse term=1 match=5
> n1 recv AppendResponse term=1 match=5
> n1 recv AppendResponse term=1 match=5

raft-log 1
----
n1: commit=5 applied=5 last=5
  (snapshot covers <= index 5)

recover 3
----
n3 recovered

tick-heartbeat 1
----
Heartbeat term=1 commit=5
Heartbeat term=1 commit=1

stabilize
----
> n2 recv Heartbeat term=1 commit=5
> n3 recv Heartbeat term=1 commit=1
> n2 ready
  HeartbeatResponse term=1
> n3 ready
  HeartbeatResponse term=1
> n1 recv HeartbeatResponse term=1
> n1 recv HeartbeatResponse term=1
> n1 ready
  AppendEntries term=1 prev=1/5 commit=5 entries=[]
  AppendEntries term=1 prev=1/5 commit=5 entries=[]
> n2 recv AppendEntries term=1 prev=1/5 commit=5 entries=[]
> n3 recv AppendEntries term=1 prev=1/5 commit=5 entries=[]
> n2 ready
  AppendResponse term=1 match=5
> n3 ready
  AppendResponse term=1 reject hint=1/1
> n1 recv AppendResponse term=1 match=5
> n1 recv AppendResponse term=1 reject hint=1/1
> n1 ready
  InstallSnapshot term=1 snap=1/5 offset=0 len=84 last=true
> n3 recv InstallSnapshot term=1 snap=1/5 offset=0 len=84 last=true
> n3 ready
  SnapshotResponse term=1 reject=false match=5 acked=0
  n3 snapshot-installed snap=1/5
> n1 recv SnapshotResponse term=1 reject=false match=5 acked=0
> n1 ready
  AppendEntries term=1 prev=1/5 commit=5 entries=[]
> n3 recv AppendEntries term=1 prev=1/5 commit=5 entries=[]
> n3 ready
  AppendResponse term=1 match=5
> n1 recv AppendResponse term=1 match=5

raft-log 3
----
n3: commit=5 applied=5 last=5
  (snapshot covers <= index 5)

