(etcd)raft算法的学习笔记 (二)
Raft 一致性算法
section 5几个子章节结合etcd源码讨论
一、section 5.1
- 服务器始终处于三种状态：leader、follower、candidate（etcd新增了一个learner，在后面补充）
- follower: 该角色处于被动位置从不会主动发起请求，仅仅是简单的回应leader和candidate的请求
- leader:该角色处理所有客户端发来的请求。如果follower不小心接收到了客户端请求，该请求将会转发给leader
- candidate：该角色是follower到leader的过度状态
- RAFT将时间划分为任意长度的任期，任期为连续的整数，每一任期从选举开始
- 不同的servers能观察到多次任期的转换。任期在raft中充当逻辑时钟的作用，这可以帮助集群探查一些过期的信息例如旧的leader等。
- 如果一个server的任期号比其他人的小，那么他会更新自己到较新的编号。如果此时处于leader或candidate状态,将会立即变味follower
//来自于 etcd/raft/raft_paper_test.go
func TestFollowerUpdateTermFromMessage(t *testing.T) {
//对于follower更新到较新的编号
testUpdateTermFromMessage(t, StateFollower)
}
raft2020/06/03 10:43:33 raft.go:1521: INFO: 1 switched to configuration voters=(1 2 3)
raft2020/06/03 10:43:33 raft.go:699: INFO: 1 became follower at term 0
raft2020/06/03 10:43:33 raft.go:383: INFO: newRaft 1 [peers: [1,2,3], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
raft2020/06/03 10:43:33 raft.go:699: INFO: 1 became follower at term 1
raft2020/06/03 10:43:33 raft.go:858: INFO: 1 [term: 1] received a MsgApp message with higher term from 0 [term: 2]
raft2020/06/03 10:43:33 raft.go:699: INFO: 1 became follower at term 2
func TestCandidateUpdateTermFromMessage(t *testing.T) {
//对于candidate更新到较新的编号，最后状态会变为follower
testUpdateTermFromMessage(t, StateCandidate)
}
raft2020/06/03 10:43:56 raft.go:1521: INFO: 1 switched to configuration voters=(1 2 3)
raft2020/06/03 10:43:56 raft.go:699: INFO: 1 became follower at term 0
raft2020/06/03 10:43:56 raft.go:383: INFO: newRaft 1 [peers: [1,2,3], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
raft2020/06/03 10:43:56 raft.go:712: INFO: 1 became candidate at term 1
raft2020/06/03 10:43:56 raft.go:858: INFO: 1 [term: 1] received a MsgApp message with higher term from 0 [term: 2]
raft2020/06/03 10:43:56 raft.go:699: INFO: 1 became follower at term 2
func TestLeaderUpdateTermFromMessage(t *testing.T) {
//对于leader更新到较新的编号，最后状态会变为follower
testUpdateTermFromMessage(t, StateLeader)
}
raft2020/06/03 10:44:27 raft.go:1521: INFO: 1 switched to configuration voters=(1 2 3)
raft2020/06/03 10:44:27 raft.go:699: INFO: 1 became follower at term 0
raft2020/06/03 10:44:27 raft.go:383: INFO: newRaft 1 [peers: [1,2,3], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
raft2020/06/03 10:44:27 raft.go:712: INFO: 1 became candidate at term 1
raft2020/06/03 10:44:27 raft.go:764: INFO: 1 became leader at term 1
raft2020/06/03 10:44:27 raft.go:858: INFO: 1 [term: 1] received a MsgApp message with higher term from 0 [term: 2]
raft2020/06/03 10:44:27 raft.go:699: INFO: 1 became follower at term 2
// testUpdateTermFromMessage tests that if one server’s current term is
// smaller than the other’s, then it updates its current term to the larger
// value. If a candidate or leader discovers that its term is out of date,
// it immediately reverts to follower state.
// Reference: section 5.1
func testUpdateTermFromMessage(t *testing.T, state StateType) {
r := newTestRaft(1, []uint64{1, 2, 3}, 10, 1, NewMemoryStorage()) //初始化一个test raft，peers包括1、2、3
switch state {
case StateFollower:
r.becomeFollower(1, 2) //1是给定的任期，2是leader的id
case StateCandidate:
r.becomeCandidate() //将初始化的raft状态变为candidate
case StateLeader:
r.becomeCandidate() //将初始化的raft状态由follower变为candidate，若follower直接转变为leader会出错
r.becomeLeader() //candidate状态变为follower
}
r.Step(pb.Message{Type: pb.MsgApp, Term: 2}) //发送一个
if r.Term != 2 {
t.Errorf("term = %d, want %d", r.Term, 2)
}
if r.state != StateFollower {
t.Errorf("state = %v, want %v", r.state, StateFollower)
}
}
- 如果一个server发现一个请求的任期号是旧的，这个请求将会给拒绝
// TestRejectStaleTermMessage tests that if a server receives a request with
// a stale term number, it rejects the request.
// Our implementation ignores the request instead.
// Reference: section 5.1
func TestRejectStaleTermMessage(t *testing.T) {
called := false
fakeStep := func(r *raft, m pb.Message) error {
called = true
return nil
}
r := newTestRaft(1, []uint64{1, 2, 3}, 10, 1, NewMemoryStorage()) //创建一个新的server
//fmt.Printf("%+v,\n", r)
r.step = fakeStep
r.loadState(pb.HardState{Term: 2}) //将r的任期置为2
//fmt.Println(r.Term)
r.Step(pb.Message{Type: pb.MsgApp, Term: r.Term - 1}) //发送一个旧任期的请求给r
if called {
t.Errorf("stepFunc called = %v, want %v", called, false)
}
}
=== RUN TestRejectStaleTermMessage
raft2020/06/03 19:35:25 raft.go:1521: INFO: 1 switched to configuration voters=(1 2 3)
raft2020/06/03 19:35:25 raft.go:699: INFO: 1 became follower at term 0
raft2020/06/03 19:35:25 raft.go:383: INFO: newRaft 1 [peers: [1,2,3], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
raft2020/06/03 19:35:25 raft.go:900: INFO: 1 [term: 2] ignored a MsgApp message with lower term from 0 [term: 1]
一、section 5.2
- 当服务器启动时，默认为follower状态
// TestStartAsFollower tests that when servers start up, they begin as followers.
// Reference: section 5.2
func TestStartAsFollower(t *testing.T) {
r := newTestRaft(1, []uint64{1, 2, 3}, 10, 1, NewMemoryStorage())
//r.state = StateCandidate
if r.state != StateFollower {
//如果不是follower将会报错
t.Errorf("state = %s, want %s", r.state, StateFollower)
}
}
=== RUN TestStartAsFollower
raft2020/06/04 11:48:18 raft.go:1521: INFO: 1 switched to configuration voters=(1 2 3)
raft2020/06/04 11:48:18 raft.go:699: INFO: 1 became follower at term 0
raft2020/06/04 11:48:18 raft.go:383: INFO: newRaft 1 [peers: [1,2,3], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
--- PASS: TestStartAsFollower (0.00s)
- leader会定时发送心跳检测给其他follower以维护自己的地位
// TestLeaderBcastBeat tests that if the leader receives a heartbeat tick,
// it will send a MsgHeartbeat with m.Index = 0, m.LogTerm=0 and empty entries
// as heartbeat to all followers.
// Reference: section 5.2
func TestLeaderBcastBeat(t *testing.T) {
// heartbeat interval
hi := 1
r := newTestRaft(1, []uint64{1, 2, 3}, 10, hi, NewMemoryStorage()) //10为选举超时时间,hi为心跳超时需要进行一次检测的时间
r.becomeCandidate()
r.becomeLeader() //r.tick方法此时变为了tickHeartbeat;r.step方法变为stepLeader
for i := 0; i < 10; i++ {
mustAppendEntry(r, pb.Entry{Index: uint64(i) + 1})
}
for i := 0; i < hi; i++ {
r.tick() //发送一次心跳检测，运行tickHeartbeat方法；这里hi设为1所以只需滴答一次heartbeatElapsed会等于heartbeatTimeout
}
msgs := r.readMessages() //该方法将r的message读出来，并且将r的message重置
sort.Sort(messageSlice(msgs))
wmsgs := []pb.Message{
{From: 1, To: 2, Term: 1, Type: pb.MsgHeartbeat},
{From: 1, To: 3, Term: 1, Type: pb.MsgHeartbeat},
}
if !reflect.DeepEqual(msgs, wmsgs) {
t.Errorf("msgs = %v, want %v", msgs, wmsgs)
}
}
// tickHeartbeat is run by leaders to send a MsgBeat after r.heartbeatTimeout.
func (r *raft) tickHeartbeat() {
r.heartbeatElapsed++ //每运行一遍该方法heartbeatElapsed增加1，当该值大于或等于heartbeatTimeout就会重置heartbeatElapsed并传递心跳检测
r.electionElapsed++
if r.electionElapsed >= r.electionTimeout {
r.electionElapsed = 0
if r.checkQuorum {
r.Step(pb.Message{From: r.id, Type: pb.MsgCheckQuorum})
}
// If current leader cannot transfer leadership in electionTimeout, it becomes leader again.
if r.state == StateLeader && r.leadTransferee != None {
r.abortLeaderTransfer()
}
}
if r.state != StateLeader {
return
}
if r.heartbeatElapsed >= r.heartbeatTimeout {
r.heartbeatElapsed = 0
r.Step(pb.Message{From: r.id, Type: pb.MsgBeat})
//传递心跳检测;m.Type为MsgBeat所以在Step方法里面跳到r.step即上面说的stepLeader；
//最后经过的方法：stepLeader->bcastHeartbeat->bcastHeartbeatWithCtx->sendHeartbeat->send
//最终在send里面将MsgBeat的信息添加到r.msg切片中去
}
}
- 重新选举的两种情况：1⃣️follower在选举超时时间内没收到心跳检测RPC,将会增加自己的当前任期并且转变为candidate状态，此时该ratf会先给自己投一票并发送vote RPC给其他servers。2⃣️集群中有多个candidate收到了相同选票，将会进入超时并且开始一次新的选举，任期号此生也增加一位
func TestFollowerStartElection(t *testing.T) {
testNonleaderStartElection(t, StateFollower)
}
func TestCandidateStartNewElection(t *testing.T) {
testNonleaderStartElection(t, StateCandidate)
}
// testNonleaderStartElection tests that if a follower receives no communication
// over election timeout, it begins an election to choose a new leader. It
// increments its current term and transitions to candidate state. It then
// votes for itself and issues RequestVote RPCs in parallel to each of the
// other servers in the cluster.
// Reference: section 5.2
// Also if a candidate fails to obtain a majority, it will time out and
// start a new election by incrementing its term and initiating another
// round of RequestVote RPCs.
// Reference: section 5.2
func testNonleaderStartElection(t *testing.T, state StateType) {
// election timeout
et := 10 //选举超时时间10
r := newTestRaft(1, []uint64{1, 2, 3}, et, 1, NewMemoryStorage())
switch state {
case StateFollower:
r.becomeFollower(1, 2) //tick方法是tickElection
case StateCandidate:
r.becomeCandidate() //tick方法是tickElection
}
//r会生成一个randomizedElectionTimeout，当下面的loop达到这个次数时，将进行MsgVote RPC
for i := 1; i < 2*et; i++ {
r.tick()
}
if r.Term != 2 {
t.Errorf("term = %d, want 2", r.Term)
}
if r.state != StateCandidate {
t.Errorf("state = %s, want %s", r.state, StateCandidate)
}
if !r.prs.Votes[r.id] {
t.Errorf("vote for self = false, want true")
}
msgs := r.readMessages()
sort.Sort(messageSlice(msgs))
wmsgs := []pb.Message{
{From: 1, To: 2, Term: 2, Type: pb.MsgVote},
{From: 1, To: 3, Term: 2, Type: pb.MsgVote},
}
if !reflect.DeepEqual(msgs, wmsgs) {
t.Errorf("msgs = %v, want %v", msgs, wmsgs)
}
}
- 当开始一次选举时，follower将会增加它的当前任期并且转变为cadidate状态。该状态将会持续到：1⃣️此节点赢了选举2⃣️别的节点赢的了选举并通知到位3⃣️一段时间后没有节点胜利
// TestLeaderElectionInOneRoundRPC tests all cases that may happen in
// leader election during one round of RequestVote RPC:
// a) it wins the election
// b) it loses the election
// c) it is unclear about the result
// Reference: section 5.2
func TestLeaderElectionInOneRoundRPC(t *testing.T) {
tests := []struct {
size int
votes map[uint64]bool
state StateType
}{
// win the election when receiving votes from a majority of the servers
{1, map[uint64]bool{}, StateLeader},
{3, map[uint64]bool{2: true, 3: true}, StateLeader},
{3, map[uint64]bool{2: true}, StateLeader},
{5, map[uint64]bool{2: true, 3: true, 4: true, 5: true}, StateLeader},
{5, map[uint64]bool{2: true, 3: true, 4: true}, StateLeader},
{5, map[uint64]bool{2: true, 3: true}, StateLeader},
// return to follower state if it receives vote denial from a majority
{3, map[uint64]bool{2: false, 3: false}, StateFollower},
{5, map[uint64]bool{2: false, 3: false, 4: false, 5: false}, StateFollower},
{5, map[uint64]bool{2: true, 3: false, 4: false, 5: false}, StateFollower},
// stay in candidate if it does not obtain the majority
{3, map[uint64]bool{}, StateCandidate},
{5, map[uint64]bool{2: true}, StateCandidate},
{5, map[uint64]bool{2: false, 3: false}, StateCandidate},
{5, map[uint64]bool{}, StateCandidate},
}
for i, tt := range tests {
fmt.Println(i)
fmt.Println(tt)
r := newTestRaft(1, idsBySize(tt.size), 10, 1, NewMemoryStorage())
r.Step(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) //对外发出投票请求
for id, vote := range tt.votes {
fmt.Println(id)
r.Step(pb.Message{From: id, To: 1, Term: r.Term, Type: pb.MsgVoteResp, Reject: !vote}) //模拟接收投票回来的结果，
//当发现投票超过了半数就becomeleader,当继续传回MsgVoteResp将不回有任何操作了
}
if r.state != tt.state {
t.Errorf("#%d: state = %s, want %s", i, r.state, tt.state)
}
if g := r.Term; g != 1 {
t.Errorf("#%d: term = %d, want %d", i, g, 1)
}
}
}
- 如果一个candidate在等待选票的时候，收到了一个追加日志的rpc，并且该rpc发出者的任期大于或等于该candidate，那么就认为这个发出追加日志rpc的leader是合法的。最后，该candidate会转变为follower
// TestCandidateFallback tests that while waiting for votes,
// if a candidate receives an AppendEntries RPC from another server claiming
// to be leader whose term is at least as large as the candidate's current term,
// it recognizes the leader as legitimate and returns to follower state.
// Reference: section 5.2
func TestCandidateFallback(t *testing.T) {
tests := []pb.Message{
{From: 2, To: 1, Term: 1, Type: pb.MsgApp},
{From: 2, To: 1, Term: 2, Type: pb.MsgApp},
}
for i, tt := range tests {
r := newTestRaft(1, []uint64{1, 2, 3}, 10, 1, NewMemoryStorage())
r.Step(pb.Message{From: 1, To: 1, Type: pb.MsgHup}) //新建立的raft选举超时进行新的选举并向2和3发出投票RP，增加了一次任期
if r.state != StateCandidate {
t.Fatalf("unexpected state = %s, want %s", r.state, StateCandidate)
}
r.Step(tt) //当回传的包里的任期参数大于当前raft时，1会变成follower；其他情况只要选票还没超过半数节点1还是处于candidate状态
if g := r.state; g != StateFollower {
t.Errorf("#%d: state = %s, want %s", i, g, StateFollower)
}
if g := r.Term; g != tt.Term {
t.Errorf("#%d: term = %d, want %d", i, g, tt.Term)
}
}
}
待续