T3MP3ST public launch build
This commit is contained in:
t3mp3st contributors committed
1 parent
972247b881
commit
184c7d4e59
1589 files changed
+57150
-506
No files matched your search
@@ -0,0 +1,348 @@
|
||||
// Copyright IBM Corp. 2015, 2025
|
||||
// SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
package nomad
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/raft"
|
||||
autopilot "github.com/hashicorp/raft-autopilot"
|
||||
"github.com/hashicorp/serf/serf"
|
||||
"github.com/shoenig/test/must"
|
||||
|
||||
"github.com/hashicorp/nomad/ci"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
"github.com/hashicorp/nomad/testutil"
|
||||
)
|
||||
|
||||
var _ autopilot.ApplicationIntegration = (*AutopilotDelegate)(nil)
|
||||
|
||||
// wantPeers determines whether the server has the given
|
||||
// number of voting raft peers.
|
||||
func wantPeers(s *Server, peers int) error {
|
||||
future := s.raft.GetConfiguration()
|
||||
if err := future.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var n int
|
||||
for _, server := range future.Configuration().Servers {
|
||||
if server.Suffrage == raft.Voter {
|
||||
n++
|
||||
}
|
||||
}
|
||||
|
||||
if got, want := n, peers; got != want {
|
||||
return fmt.Errorf("server %v: got %d peers want %d\n\tservers: %#+v", s.config.NodeName, got, want, future.Configuration().Servers)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAutopilot_CleanupDeadServer(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
conf := func(c *Config) {
|
||||
c.NumSchedulers = 0 // reduces test log noise
|
||||
c.BootstrapExpect = 3
|
||||
c.RaftConfig.ProtocolVersion = raft.ProtocolVersion(3)
|
||||
}
|
||||
|
||||
s1, cleanupS1 := TestServer(t, conf)
|
||||
defer cleanupS1()
|
||||
|
||||
s2, cleanupS2 := TestServer(t, conf)
|
||||
defer cleanupS2()
|
||||
|
||||
s3, cleanupS3 := TestServer(t, conf)
|
||||
defer cleanupS3()
|
||||
|
||||
servers := []*Server{s1, s2, s3}
|
||||
TestJoin(t, servers...)
|
||||
|
||||
t.Logf("waiting for initial stable cluster")
|
||||
waitForStableLeadership(t, servers)
|
||||
|
||||
s4, cleanupS4 := TestServer(t, conf)
|
||||
defer cleanupS4()
|
||||
|
||||
// Kill a non-leader server
|
||||
killedIdx := 0
|
||||
for i, s := range servers {
|
||||
if !s.IsLeader() {
|
||||
killedIdx = i
|
||||
t.Logf("killing a server (index %d)", killedIdx)
|
||||
s.Shutdown()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("waiting for server loss to be detected")
|
||||
testutil.WaitForResultUntil(10*time.Second, func() (bool, error) {
|
||||
for i, s := range servers {
|
||||
alive := 0
|
||||
if i == killedIdx {
|
||||
// Skip shutdown server
|
||||
continue
|
||||
}
|
||||
for _, m := range s.Members() {
|
||||
if m.Status == serf.StatusAlive {
|
||||
alive++
|
||||
}
|
||||
}
|
||||
|
||||
if alive != 2 {
|
||||
return false, fmt.Errorf("expected 2 alive servers but found %v", alive)
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}, func(err error) { must.NoError(t, err) })
|
||||
|
||||
// Join the new server
|
||||
servers[killedIdx] = s4
|
||||
t.Logf("adding server s4")
|
||||
TestJoin(t, servers...)
|
||||
|
||||
t.Logf("waiting for dead server to be removed")
|
||||
waitForStableLeadership(t, servers)
|
||||
}
|
||||
|
||||
func TestAutopilot_CleanupDeadServerPeriodic(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
conf := func(c *Config) {
|
||||
c.NumSchedulers = 0 // reduces test log noise
|
||||
c.BootstrapExpect = 5
|
||||
}
|
||||
|
||||
s1, cleanupS1 := TestServer(t, conf)
|
||||
defer cleanupS1()
|
||||
|
||||
s2, cleanupS2 := TestServer(t, conf)
|
||||
defer cleanupS2()
|
||||
|
||||
s3, cleanupS3 := TestServer(t, conf)
|
||||
defer cleanupS3()
|
||||
|
||||
s4, cleanupS4 := TestServer(t, conf)
|
||||
defer cleanupS4()
|
||||
|
||||
s5, cleanupS5 := TestServer(t, conf)
|
||||
defer cleanupS5()
|
||||
|
||||
servers := []*Server{s1, s2, s3, s4, s5}
|
||||
TestJoin(t, servers...)
|
||||
|
||||
t.Logf("waiting for initial stable cluster")
|
||||
waitForStableLeadership(t, servers)
|
||||
|
||||
t.Logf("killing a non-leader server")
|
||||
if leader := waitForStableLeadership(t, servers); leader == s4 {
|
||||
s1, s4 = s4, s1
|
||||
}
|
||||
s4.Shutdown()
|
||||
|
||||
t.Logf("waiting for dead peer to be removed")
|
||||
servers = []*Server{s1, s2, s3, s5}
|
||||
waitForStableLeadership(t, servers)
|
||||
}
|
||||
|
||||
func TestAutopilot_RollingUpdate(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
conf := func(c *Config) {
|
||||
c.NumSchedulers = 0 // reduces test log noise
|
||||
c.BootstrapExpect = 3
|
||||
c.RaftConfig.ProtocolVersion = 3
|
||||
}
|
||||
|
||||
s1, cleanupS1 := TestServer(t, conf)
|
||||
defer cleanupS1()
|
||||
|
||||
s2, cleanupS2 := TestServer(t, conf)
|
||||
defer cleanupS2()
|
||||
|
||||
s3, cleanupS3 := TestServer(t, conf)
|
||||
defer cleanupS3()
|
||||
|
||||
servers := []*Server{s1, s2, s3}
|
||||
TestJoin(t, s1, s2, s3)
|
||||
|
||||
t.Logf("waiting for initial stable cluster")
|
||||
waitForStableLeadership(t, servers)
|
||||
|
||||
// Add one more server like we are doing a rolling update.
|
||||
t.Logf("adding server s4")
|
||||
s4, cleanupS4 := TestServer(t, conf)
|
||||
defer cleanupS4()
|
||||
TestJoin(t, s1, s4)
|
||||
|
||||
// Wait for s4 to stabilize and get promoted to a voter
|
||||
t.Logf("waiting for s4 to stabilize and be promoted")
|
||||
servers = append(servers, s4)
|
||||
waitForStableLeadership(t, servers)
|
||||
|
||||
// Now kill one of the "old" nodes like we are doing a rolling update.
|
||||
t.Logf("shutting down server s3")
|
||||
s3.Shutdown()
|
||||
|
||||
// Wait for s3 to be removed and the cluster to stablize.
|
||||
t.Logf("waiting for cluster to stabilize")
|
||||
servers = []*Server{s1, s2, s4}
|
||||
waitForStableLeadership(t, servers)
|
||||
}
|
||||
|
||||
func TestAutopilot_MultiRegion(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
conf := func(c *Config) {
|
||||
c.NumSchedulers = 0 // reduces test log noise
|
||||
c.BootstrapExpect = 3
|
||||
}
|
||||
s1, cleanupS1 := TestServer(t, conf)
|
||||
defer cleanupS1()
|
||||
|
||||
s2, cleanupS2 := TestServer(t, conf)
|
||||
defer cleanupS2()
|
||||
|
||||
s3, cleanupS3 := TestServer(t, conf)
|
||||
defer cleanupS3()
|
||||
|
||||
// federated regions should not be considered raft peers or show up in the
|
||||
// known servers list
|
||||
s4, cleanupS4 := TestServer(t, func(c *Config) {
|
||||
c.BootstrapExpect = 0
|
||||
c.Region = "other"
|
||||
})
|
||||
defer cleanupS4()
|
||||
|
||||
servers := []*Server{s1, s2, s3}
|
||||
TestJoin(t, s1, s2, s3, s4)
|
||||
|
||||
t.Logf("waiting for initial stable cluster")
|
||||
waitForStableLeadership(t, servers)
|
||||
|
||||
apDelegate := &AutopilotDelegate{s3}
|
||||
known := apDelegate.KnownServers()
|
||||
must.Eq(t, 3, len(known))
|
||||
|
||||
}
|
||||
|
||||
func TestAutopilot_CleanupStaleRaftServer(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
conf := func(c *Config) {
|
||||
c.NumSchedulers = 0 // reduces test log noise
|
||||
c.BootstrapExpect = 3
|
||||
}
|
||||
s1, cleanupS1 := TestServer(t, conf)
|
||||
defer cleanupS1()
|
||||
|
||||
s2, cleanupS2 := TestServer(t, conf)
|
||||
defer cleanupS2()
|
||||
|
||||
s3, cleanupS3 := TestServer(t, conf)
|
||||
defer cleanupS3()
|
||||
|
||||
s4, cleanupS4 := TestServer(t, func(c *Config) {
|
||||
c.BootstrapExpect = 0
|
||||
})
|
||||
defer cleanupS4()
|
||||
|
||||
servers := []*Server{s1, s2, s3}
|
||||
TestJoin(t, s1, s2, s3)
|
||||
|
||||
t.Logf("waiting for initial stable cluster")
|
||||
leader := waitForStableLeadership(t, servers)
|
||||
|
||||
t.Logf("adding server s4 to peers directly")
|
||||
addr := fmt.Sprintf("127.0.0.1:%d", s4.config.RPCAddr.Port)
|
||||
future := leader.raft.AddVoter(raft.ServerID(s4.config.NodeID), raft.ServerAddress(addr), 0, 0)
|
||||
if err := future.Error(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("waiting for 4th server to be removed")
|
||||
waitForStableLeadership(t, servers)
|
||||
}
|
||||
|
||||
func TestAutopilot_PromoteNonVoter(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
||||
c.NumSchedulers = 0 // reduces test log noise
|
||||
c.RaftConfig.ProtocolVersion = 3
|
||||
})
|
||||
defer cleanupS1()
|
||||
codec := rpcClient(t, s1)
|
||||
defer codec.Close()
|
||||
testutil.WaitForLeader(t, s1.RPC)
|
||||
|
||||
s2, cleanupS2 := TestServer(t, func(c *Config) {
|
||||
c.NumSchedulers = 0 // reduces test log noise
|
||||
c.BootstrapExpect = 0
|
||||
c.RaftConfig.ProtocolVersion = 3
|
||||
})
|
||||
defer cleanupS2()
|
||||
TestJoin(t, s1, s2)
|
||||
|
||||
// Note: we can't reliably detect that the server is initially a non-voter,
|
||||
// because it can transition too quickly for the test setup to detect,
|
||||
// especially in low-resource environments like CI. We'll assume that
|
||||
// happens correctly here and only test that it transitions to become a
|
||||
// voter.
|
||||
testutil.WaitForResultUntil(10*time.Second, func() (bool, error) {
|
||||
future := s1.raft.GetConfiguration()
|
||||
if err := future.Error(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
servers := future.Configuration().Servers
|
||||
if len(servers) != 2 {
|
||||
return false, fmt.Errorf("expected 2 servers, got: %v", servers)
|
||||
}
|
||||
if servers[1].Suffrage != raft.Voter {
|
||||
return false, fmt.Errorf("expected server to be voter: %v", servers)
|
||||
}
|
||||
return true, nil
|
||||
}, func(err error) { must.NoError(t, err) })
|
||||
}
|
||||
|
||||
func TestAutopilot_ReturnAutopilotHealth(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
s1, cleanupS1 := TestServer(t, func(c *Config) {
|
||||
c.BootstrapExpect = 2
|
||||
c.RaftConfig.ProtocolVersion = 3
|
||||
c.AutopilotConfig.EnableCustomUpgrades = true
|
||||
c.UpgradeVersion = "0.0.1"
|
||||
c.NumSchedulers = 0 // reduce log noise
|
||||
})
|
||||
defer cleanupS1()
|
||||
|
||||
s2, cleanupS2 := TestServer(t, func(c *Config) {
|
||||
c.BootstrapExpect = 2
|
||||
c.RaftConfig.ProtocolVersion = 3
|
||||
c.AutopilotConfig.EnableCustomUpgrades = true
|
||||
c.UpgradeVersion = "0.0.1"
|
||||
c.NumSchedulers = 0 // reduce log noise
|
||||
})
|
||||
defer cleanupS2()
|
||||
|
||||
TestJoin(t, s1, s2)
|
||||
servers := []*Server{s1, s2}
|
||||
leader := waitForStableLeadership(t, servers)
|
||||
|
||||
get := &structs.GenericRequest{
|
||||
QueryOptions: structs.QueryOptions{
|
||||
Region: "global",
|
||||
},
|
||||
}
|
||||
reply := &structs.OperatorHealthReply{}
|
||||
err := s1.RPC("Operator.ServerHealth", get, reply)
|
||||
must.NoError(t, err)
|
||||
|
||||
must.Eq(t, reply.Healthy, true)
|
||||
_, leaderID := leader.raft.LeaderWithID()
|
||||
must.Eq(t, reply.Leader, string(leaderID))
|
||||
}
|
||||
Reference in new issue
Block a user