812 lines
23 KiB
Go
812 lines
23 KiB
Go
// Copyright IBM Corp. 2015, 2025
|
|
// SPDX-License-Identifier: BUSL-1.1
|
|
|
|
package nomad
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hashicorp/nomad/ci"
|
|
"github.com/hashicorp/nomad/helper/testlog"
|
|
"github.com/hashicorp/nomad/nomad/mock"
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
"github.com/shoenig/test/must"
|
|
"github.com/shoenig/test/wait"
|
|
)
|
|
|
|
// Stats is used to query the state of the blocked eval tracker. This is only
|
|
// ever called in tests.
|
|
func (b *BlockedEvals) Stats() *BlockedStats {
|
|
return b.stats.Copy()
|
|
}
|
|
|
|
func testBlockedEvals(t *testing.T) (*BlockedEvals, *EvalBroker) {
|
|
broker := testBroker(t, 0)
|
|
broker.SetEnabled(true)
|
|
blocked := NewBlockedEvals(broker, testlog.HCLogger(t))
|
|
blocked.SetEnabled(true)
|
|
return blocked, broker
|
|
}
|
|
|
|
func TestBlockedEvals_Block_Disabled(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, _ := testBlockedEvals(t)
|
|
blocked.SetEnabled(false)
|
|
|
|
// Create an escaped eval and add it to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.EscapedComputedClass = true
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block did nothing.
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 0, stats.BlockedResources.ByJob)
|
|
}
|
|
|
|
func TestBlockedEvals_Block_SameJob(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, _ := testBlockedEvals(t)
|
|
|
|
// Create two blocked evals and add them to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e2 := mock.BlockedEval()
|
|
e2.JobID = e.JobID
|
|
<-blocked.Block(e)
|
|
<-blocked.Block(e2)
|
|
|
|
// Verify block didn't track duplicate.
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
}
|
|
|
|
func TestBlockedEvals_Block_Quota(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
blocked, _ := testBlockedEvals(t)
|
|
|
|
// Create a blocked eval on quota.
|
|
e := mock.BlockedEval()
|
|
e.QuotaLimitReached = "foo"
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block did track eval.
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.Eq(t, 1, stats.TotalQuotaLimit)
|
|
}
|
|
|
|
func TestBlockedEvals_Block_PriorUnblocks(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, _ := testBlockedEvals(t)
|
|
|
|
// Do unblocks prior to blocking.
|
|
blocked.Unblock("v1:123", 1000)
|
|
blocked.Unblock("v1:123", 1001)
|
|
|
|
// Create blocked eval with two classes ineligible.
|
|
e := mock.BlockedEval()
|
|
e.ClassEligibility = map[string]bool{"v1:123": false, "v1:456": false}
|
|
e.SnapshotIndex = 999
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block did track eval.
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
}
|
|
|
|
func TestBlockedEvals_GetDuplicates(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, _ := testBlockedEvals(t)
|
|
|
|
// Create duplicate blocked evals and add them to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.CreateIndex = 100
|
|
e2 := mock.BlockedEval()
|
|
e2.JobID = e.JobID
|
|
e2.CreateIndex = 101
|
|
e3 := mock.BlockedEval()
|
|
e3.JobID = e.JobID
|
|
e3.CreateIndex = 102
|
|
e4 := mock.BlockedEval()
|
|
e4.JobID = e.JobID
|
|
e4.CreateIndex = 100
|
|
<-blocked.Block(e)
|
|
<-blocked.Block(e2)
|
|
|
|
// Verify stats such that we are only tracking one.
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
// Get the duplicates.
|
|
out := blocked.GetDuplicates(0)
|
|
must.Len(t, 1, out)
|
|
must.Eq(t, e, out[0])
|
|
|
|
// Call block again after a small sleep.
|
|
go func() {
|
|
time.Sleep(500 * time.Millisecond)
|
|
<-blocked.Block(e3)
|
|
}()
|
|
|
|
// Get the duplicates.
|
|
out = blocked.GetDuplicates(1 * time.Second)
|
|
must.Len(t, 1, out)
|
|
must.Eq(t, e2, out[0])
|
|
|
|
// Verify stats such that we are only tracking one.
|
|
stats = blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
// Add an older evaluation and assert it gets cancelled.
|
|
<-blocked.Block(e4)
|
|
out = blocked.GetDuplicates(0)
|
|
must.Len(t, 1, out)
|
|
must.Eq(t, e4, out[0])
|
|
|
|
// Verify stats such that we are only tracking one.
|
|
stats = blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
}
|
|
|
|
func TestBlockedEvals_UnblockEscaped(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
// Create an escaped eval and add it to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.Status = structs.EvalStatusBlocked
|
|
e.EscapedComputedClass = true
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block caused the eval to be tracked
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 1, stats.TotalEscaped)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
<-blocked.Unblock("v1:123", 1000)
|
|
requireBlockedEvalsEnqueued(t, blocked, broker, 1)
|
|
}
|
|
|
|
func requireBlockedEvalsEnqueued(t *testing.T, blocked *BlockedEvals, broker *EvalBroker, enqueued int) {
|
|
t.Helper()
|
|
|
|
must.Wait(t, wait.InitialSuccess(
|
|
wait.ErrorFunc(func() error {
|
|
brokerStats := broker.Stats()
|
|
if brokerStats.TotalReady != enqueued {
|
|
return fmt.Errorf("missing enqueued evals: %#v", brokerStats)
|
|
}
|
|
return nil
|
|
}),
|
|
wait.Timeout(500*time.Millisecond),
|
|
wait.Gap(10*time.Millisecond),
|
|
))
|
|
|
|
blocked.pruneStats(time.Now().UTC())
|
|
|
|
// verfiy stats have been cleared
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 0, stats.BlockedResources.ByJob)
|
|
}
|
|
|
|
func TestBlockedEvals_UnblockEligible(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
// Create a blocked eval that is eligible on a specific node class and add
|
|
// it to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.Status = structs.EvalStatusBlocked
|
|
e.ClassEligibility = map[string]bool{"v1:123": true}
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block caused the eval to be tracked
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
|
|
blocked.Unblock("v1:123", 1000)
|
|
requireBlockedEvalsEnqueued(t, blocked, broker, 1)
|
|
}
|
|
|
|
func TestBlockedEvals_UnblockIneligible(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
// Create a blocked eval that is ineligible on a specific node class and add
|
|
// it to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.ClassEligibility = map[string]bool{"v1:123": false}
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block caused the eval to be tracked
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
// Should do nothing
|
|
<-blocked.Unblock("v1:123", 1000)
|
|
|
|
// Verify Unblock didn't cause an enqueue
|
|
brokerStats := broker.Stats()
|
|
must.Eq(t, 0, brokerStats.TotalReady, must.Sprintf("eval unblocked: %#v", brokerStats))
|
|
|
|
// Prune old and empty metrics.
|
|
blocked.pruneStats(time.Now().UTC())
|
|
|
|
stats = blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
}
|
|
|
|
func TestBlockedEvals_UnblockUnknown(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
// Create a blocked eval that is ineligible on a specific node class and add
|
|
// it to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.ClassEligibility = map[string]bool{"v1:123": true, "v1:456": false}
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block caused the eval to be tracked.
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
// Should unblock because the eval hasn't seen this node class.
|
|
<-blocked.Unblock("v1:789", 1000)
|
|
requireBlockedEvalsEnqueued(t, blocked, broker, 1)
|
|
}
|
|
|
|
func TestBlockedEvals_UnblockEligible_Quota(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
// Create a blocked eval that is eligible for a particular quota.
|
|
e := mock.BlockedEval()
|
|
e.QuotaLimitReached = "foo"
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block caused the eval to be tracked.
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 1, stats.TotalQuotaLimit)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
<-blocked.UnblockQuota("foo", 1000)
|
|
requireBlockedEvalsEnqueued(t, blocked, broker, 1)
|
|
}
|
|
|
|
// The quota here is incidental. The eval is blocked due to something else,
|
|
// e.g. cpu exhausted, but there happens to also be a quota on the namespace.
|
|
func TestBlockedEvals_UnblockEligible_IncidentalQuota(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
e := mock.BlockedEval()
|
|
e.Status = structs.EvalStatusBlocked
|
|
e.QuotaLimitReached = "" // explicitly not blocked due to quota limit
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block caused the eval to be tracked.
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
// but not due to quota.
|
|
must.Eq(t, 0, stats.TotalQuotaLimit)
|
|
|
|
// When unblocking, the quota name from the alloc is passed in,
|
|
// regardless of the cause of the initial blockage.
|
|
// Since the initial block in this test was due to something else,
|
|
// it should be unblocked without regard to quota.
|
|
<-blocked.UnblockQuota("foo", 1000)
|
|
requireBlockedEvalsEnqueued(t, blocked, broker, 1)
|
|
}
|
|
|
|
func TestBlockedEvals_UnblockIneligible_Quota(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
// Create a blocked eval that is eligible on a specific quota.
|
|
e := mock.BlockedEval()
|
|
e.QuotaLimitReached = "foo"
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block caused the eval to be tracked.
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 1, stats.TotalQuotaLimit)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
// Should do nothing and have no evals enqueued
|
|
<-blocked.UnblockQuota("bar", 1000)
|
|
brokerStats := broker.Stats()
|
|
must.Eq(t, 0, brokerStats.TotalReady)
|
|
|
|
// Prune old and empty metrics.
|
|
blocked.pruneStats(time.Now().UTC())
|
|
|
|
stats = blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.Eq(t, 1, stats.TotalQuotaLimit)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
}
|
|
|
|
func TestBlockedEvals_Reblock(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
// Create an evaluation, Enqueue/Dequeue it to get a token
|
|
e := mock.BlockedEval()
|
|
e.SnapshotIndex = 500
|
|
e.ClassEligibility = map[string]bool{"v1:123": true, "v1:456": false}
|
|
broker.Enqueue(e)
|
|
|
|
_, token, err := broker.Dequeue([]string{e.Type}, time.Second)
|
|
must.NoError(t, err)
|
|
|
|
// Reblock the evaluation
|
|
<-blocked.Reblock(e, token)
|
|
|
|
// Verify block caused the eval to be tracked
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
// Should unblock because the eval
|
|
<-blocked.Unblock("v1:123", 1000)
|
|
|
|
brokerStats := broker.Stats()
|
|
must.Eq(t, 0, brokerStats.TotalReady)
|
|
must.Eq(t, 1, brokerStats.TotalUnacked)
|
|
|
|
// Ack the evaluation which should cause the reblocked eval to transition
|
|
// to ready
|
|
err = broker.Ack(e.ID, token)
|
|
must.NoError(t, err)
|
|
|
|
requireBlockedEvalsEnqueued(t, blocked, broker, 1)
|
|
}
|
|
|
|
// Test the block case in which the eval should be immediately unblocked since
|
|
// it is escaped and old
|
|
func TestBlockedEvals_Block_ImmediateUnblock_Escaped(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
// Do an unblock prior to blocking
|
|
<-blocked.Unblock("v1:123", 1000)
|
|
|
|
// Create a blocked eval that is eligible on a specific node class and add
|
|
// it to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.EscapedComputedClass = true
|
|
e.SnapshotIndex = 900
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block caused the eval to be immediately unblocked
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 0, stats.BlockedResources.ByJob)
|
|
|
|
requireBlockedEvalsEnqueued(t, blocked, broker, 1)
|
|
}
|
|
|
|
// Test the block case in which the eval should be immediately unblocked since
|
|
// there is an unblock on an unseen class that occurred while it was in the
|
|
// scheduler
|
|
func TestBlockedEvals_Block_ImmediateUnblock_UnseenClass_After(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
// Do an unblock prior to blocking
|
|
<-blocked.Unblock("v1:123", 1000)
|
|
|
|
// Create a blocked eval that is eligible on a specific node class and add
|
|
// it to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.EscapedComputedClass = false
|
|
e.SnapshotIndex = 900
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block caused the eval to be immediately unblocked
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 0, stats.BlockedResources.ByJob)
|
|
|
|
requireBlockedEvalsEnqueued(t, blocked, broker, 1)
|
|
}
|
|
|
|
// Test the block case in which the eval should not immediately unblock since
|
|
// there is an unblock on an unseen class that occurred before it was in the
|
|
// scheduler
|
|
func TestBlockedEvals_Block_ImmediateUnblock_UnseenClass_Before(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, _ := testBlockedEvals(t)
|
|
|
|
// Do an unblock prior to blocking
|
|
<-blocked.Unblock("v1:123", 500)
|
|
|
|
// Create a blocked eval that is eligible on a specific node class and add
|
|
// it to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.EscapedComputedClass = false
|
|
e.SnapshotIndex = 900
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block caused the eval to be immediately unblocked
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
}
|
|
|
|
// Test the block case in which the eval should be immediately unblocked since
|
|
// it a class it is eligible for has been unblocked
|
|
func TestBlockedEvals_Block_ImmediateUnblock_SeenClass(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
// Do an unblock prior to blocking
|
|
<-blocked.Unblock("v1:123", 1000)
|
|
|
|
// Create a blocked eval that is eligible on a specific node class and add
|
|
// it to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.ClassEligibility = map[string]bool{"v1:123": true, "v1:456": false}
|
|
e.SnapshotIndex = 900
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block caused the eval to be immediately unblocked
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 0, stats.BlockedResources.ByJob)
|
|
|
|
requireBlockedEvalsEnqueued(t, blocked, broker, 1)
|
|
}
|
|
|
|
// Test the block case in which the eval should be immediately unblocked since
|
|
// it a quota has changed that it is using
|
|
func TestBlockedEvals_Block_ImmediateUnblock_Quota(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
// Do an unblock prior to blocking
|
|
<-blocked.UnblockQuota("my-quota", 1000)
|
|
|
|
// Create a blocked eval that is eligible on a specific node class and add
|
|
// it to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.QuotaLimitReached = "my-quota"
|
|
e.SnapshotIndex = 900
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block caused the eval to be immediately unblocked
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.Eq(t, 0, stats.TotalQuotaLimit)
|
|
must.MapLen(t, 0, stats.BlockedResources.ByJob)
|
|
|
|
requireBlockedEvalsEnqueued(t, blocked, broker, 1)
|
|
}
|
|
|
|
func TestBlockedEvals_UnblockFailed(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
// Create blocked evals that are due to failures
|
|
e := mock.BlockedEval()
|
|
e.TriggeredBy = structs.EvalTriggerMaxPlans
|
|
e.EscapedComputedClass = true
|
|
<-blocked.Block(e)
|
|
|
|
e2 := mock.BlockedEval()
|
|
e2.Status = structs.EvalStatusBlocked
|
|
e2.TriggeredBy = structs.EvalTriggerMaxPlans
|
|
e2.ClassEligibility = map[string]bool{"v1:123": true, "v1:456": false}
|
|
<-blocked.Block(e2)
|
|
|
|
e3 := mock.BlockedEval()
|
|
e3.TriggeredBy = structs.EvalTriggerMaxPlans
|
|
e3.QuotaLimitReached = "foo"
|
|
<-blocked.Block(e3)
|
|
|
|
// Trigger an unblock fail
|
|
blocked.UnblockFailed()
|
|
|
|
// Prune old and empty metrics.
|
|
blocked.pruneStats(time.Now().UTC())
|
|
|
|
// Verify UnblockFailed caused the eval to be immediately unblocked
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.Eq(t, 0, stats.TotalQuotaLimit)
|
|
must.MapLen(t, 0, stats.BlockedResources.ByJob)
|
|
|
|
requireBlockedEvalsEnqueued(t, blocked, broker, 3)
|
|
|
|
// Reblock an eval for the same job and check that it gets tracked.
|
|
<-blocked.Block(e)
|
|
stats = blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 1, stats.TotalEscaped)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
}
|
|
|
|
func TestBlockedEvals_Untrack(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, _ := testBlockedEvals(t)
|
|
|
|
// Create blocked eval and add to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.ClassEligibility = map[string]bool{"v1:123": false, "v1:456": false}
|
|
e.SnapshotIndex = 1000
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block did track
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
// Untrack and verify
|
|
<-blocked.Untrack(e.JobID, e.Namespace)
|
|
blocked.pruneStats(time.Now().UTC())
|
|
|
|
stats = blocked.Stats()
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.MapLen(t, 0, stats.BlockedResources.ByJob)
|
|
}
|
|
|
|
func TestBlockedEvals_Untrack_Quota(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, _ := testBlockedEvals(t)
|
|
|
|
// Create a blocked eval and add it to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.QuotaLimitReached = "foo"
|
|
e.SnapshotIndex = 1000
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block did track
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.Eq(t, 1, stats.TotalQuotaLimit)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
// Untrack and verify
|
|
<-blocked.Untrack(e.JobID, e.Namespace)
|
|
stats = blocked.Stats()
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.Eq(t, 0, stats.TotalQuotaLimit)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
now := time.Now().UTC()
|
|
blocked.pruneStats(now)
|
|
stats = blocked.Stats()
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.Eq(t, 0, stats.TotalQuotaLimit)
|
|
must.MapLen(t, 0, stats.BlockedResources.ByJob)
|
|
}
|
|
|
|
func TestBlockedEvals_UnblockNode(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
blocked, broker := testBlockedEvals(t)
|
|
|
|
// Create a blocked evals and add it to the blocked tracker.
|
|
e := mock.BlockedEval()
|
|
e.Type = structs.JobTypeSystem
|
|
e.NodeID = "foo"
|
|
e.SnapshotIndex = 999
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block did track
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
<-blocked.UnblockNode("foo")
|
|
requireBlockedEvalsEnqueued(t, blocked, broker, 1)
|
|
|
|
blocked.pruneStats(time.Now().UTC())
|
|
stats = blocked.Stats()
|
|
must.MapEmpty(t, blocked.system.byNode)
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.MapLen(t, 0, stats.BlockedResources.ByJob)
|
|
}
|
|
|
|
func TestBlockedEvals_SystemUntrack(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, _ := testBlockedEvals(t)
|
|
|
|
// Create a blocked evals and add it to the blocked tracker.
|
|
e := mock.Eval()
|
|
e.Type = structs.JobTypeSystem
|
|
e.NodeID = "foo"
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block did track
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.Eq(t, 0, stats.TotalQuotaLimit)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
// Untrack and verify
|
|
<-blocked.Untrack(e.JobID, e.Namespace)
|
|
blocked.pruneStats(time.Now().UTC())
|
|
|
|
stats = blocked.Stats()
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.Eq(t, 0, stats.TotalQuotaLimit)
|
|
must.MapLen(t, 0, stats.BlockedResources.ByJob)
|
|
}
|
|
|
|
func TestBlockedEvals_SystemDisableFlush(t *testing.T) {
|
|
ci.Parallel(t)
|
|
blocked, _ := testBlockedEvals(t)
|
|
|
|
// Create a blocked evals and add it to the blocked tracker.
|
|
e := mock.Eval()
|
|
e.Type = structs.JobTypeSystem
|
|
e.NodeID = "foo"
|
|
<-blocked.Block(e)
|
|
|
|
// Verify block did track
|
|
stats := blocked.Stats()
|
|
must.Eq(t, 1, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.Eq(t, 0, stats.TotalQuotaLimit)
|
|
must.MapLen(t, 1, stats.BlockedResources.ByJob)
|
|
|
|
// Disable empties
|
|
blocked.SetEnabled(false)
|
|
stats = blocked.Stats()
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
must.Eq(t, 0, stats.TotalQuotaLimit)
|
|
must.MapLen(t, 0, stats.BlockedResources.ByJob)
|
|
must.MapEmpty(t, blocked.system.evals)
|
|
must.MapEmpty(t, blocked.system.byJob)
|
|
must.MapEmpty(t, blocked.system.byNode)
|
|
}
|
|
|
|
// TestBlockedEvals_UnblockBackpressure verifies that calling Unblock methods on
|
|
// BlockedEvals doesn't result in backpressure that could impact the FSM.
|
|
func TestBlockedEvals_UnblockBackpressure(t *testing.T) {
|
|
pctx, cancel := context.WithCancel(context.TODO())
|
|
t.Cleanup(cancel)
|
|
|
|
broker, err := NewEvalBroker(pctx, time.Second, 1, 1, 1)
|
|
must.NoError(t, err)
|
|
broker.SetEnabled(true)
|
|
blockedEvals := NewBlockedEvals(broker, testlog.HCLogger(t))
|
|
blockedEvals.SetEnabled(true)
|
|
|
|
for range 10000 {
|
|
eval := mock.Eval()
|
|
eval.QueuedAllocations = map[string]int{"web": 1}
|
|
eval.ClassEligibility = map[string]bool{"foo": false}
|
|
eval.EscapedComputedClass = true
|
|
eval.Status = structs.EvalStatusBlocked
|
|
|
|
<-blockedEvals.Block(eval)
|
|
}
|
|
stats := blockedEvals.Stats()
|
|
must.Eq(t, 10000, stats.TotalBlocked)
|
|
must.Eq(t, 10000, stats.TotalEscaped)
|
|
|
|
stopCtx, stopCancel := context.WithCancel(pctx)
|
|
t.Cleanup(stopCancel)
|
|
|
|
var wg sync.WaitGroup
|
|
errors := make(chan error, 144)
|
|
|
|
// this loop continuously calls unblock to trigger any contention we might
|
|
// have; these calls should return quickly but be no-ops in watchCapacity
|
|
// once we've unblocked all 10000 blocked evals
|
|
//
|
|
// Note that this test isn't particularly useful in CI where we have low
|
|
// GOMAXPROCS. And with the fix it's intended to exercise, these Unblock
|
|
// calls return too quickly for even highly concurrent continuous Reblock
|
|
// loops to cause meaningful contention.
|
|
for i := range 144 {
|
|
wg.Go(func() {
|
|
index := uint64(1000 * i)
|
|
for {
|
|
ctx, cancel := context.WithTimeout(stopCtx, 100*time.Millisecond)
|
|
defer cancel()
|
|
errCh := make(chan error)
|
|
index++
|
|
go func() {
|
|
blockedEvals.Unblock("foo", index)
|
|
close(errCh)
|
|
}()
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
if ctx.Err() == context.DeadlineExceeded {
|
|
errors <- fmt.Errorf("Unblock took more than 100ms to complete")
|
|
}
|
|
return
|
|
case <-errCh:
|
|
cancel()
|
|
continue
|
|
}
|
|
}
|
|
|
|
})
|
|
}
|
|
|
|
// wait until all 10000 evals are unblocked or 5 seconds pass, whichever is
|
|
// longer, and collect any errors from goroutines that would indicate
|
|
// they've blocked for longer than expected
|
|
timeout := time.After(time.Second * 5)
|
|
poll := time.NewTicker(10 * time.Millisecond)
|
|
errCount := 0
|
|
var lastErr error
|
|
|
|
DONE:
|
|
for {
|
|
select {
|
|
case <-timeout:
|
|
break DONE
|
|
case <-poll.C:
|
|
stats = blockedEvals.Stats()
|
|
if stats.TotalBlocked == 0 && stats.TotalEscaped == 0 {
|
|
break DONE
|
|
}
|
|
case err := <-errors:
|
|
if err != nil {
|
|
lastErr = err
|
|
errCount++
|
|
}
|
|
}
|
|
}
|
|
|
|
must.NoError(t, lastErr, must.Sprintf("got %d errors", errCount))
|
|
stats = blockedEvals.Stats()
|
|
must.Eq(t, 0, stats.TotalBlocked)
|
|
must.Eq(t, 0, stats.TotalEscaped)
|
|
}
|