T3MP3ST public launch build

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/acl.go

@@ -0,0 +1,56 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"github.com/hashicorp/nomad/acl"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+func (s *Server) Authenticate(ctx *RPCContext, args structs.RequestWithIdentity) error {
+	return s.auth.Authenticate(ctx, args)
+}
+
+func (s *Server) AuthenticateServerOnly(ctx *RPCContext, args structs.RequestWithIdentity) (*acl.ACL, error) {
+	return s.auth.AuthenticateServerOnly(ctx, args)
+}
+
+func (s *Server) AuthenticateNodeIdentityGenerator(ctx *RPCContext, args structs.RequestWithIdentity) error {
+	return s.auth.AuthenticateNodeIdentityGenerator(ctx, args)
+}
+
+func (s *Server) AuthenticateClientOnly(ctx *RPCContext, args structs.RequestWithIdentity) (*acl.ACL, error) {
+	return s.auth.AuthenticateClientOnly(ctx, args)
+}
+
+func (s *Server) ResolveAuthorizedClientNodePoolByNodeID(aclObj *acl.ACL, nodeID string) (string, error) {
+	return s.auth.ResolveAuthorizedClientNodePoolByNodeID(aclObj, nodeID)
+}
+
+func (s *Server) AuthorizeClientAllocation(
+	aclObj *acl.ACL,
+	alloc *structs.Allocation,
+	allowNsOp func(*acl.ACL, string) bool,
+) error {
+	return s.auth.AuthorizeClientAllocation(aclObj, alloc, allowNsOp)
+}
+
+func (s *Server) ResolveAuthorizedClientNodePoolByServiceRegistrationID(
+	aclObj *acl.ACL,
+	namespace, id string,
+) (string, error) {
+	return s.auth.ResolveAuthorizedClientNodePoolByServiceRegistrationID(aclObj, namespace, id)
+}
+
+func (s *Server) ResolveACL(args structs.RequestWithIdentity) (*acl.ACL, error) {
+	return s.auth.ResolveACL(args)
+}
+
+func (s *Server) VerifyClaim(token string) (*structs.IdentityClaims, error) {
+	return s.auth.VerifyClaim(token)
+}
+
+func (s *Server) ResolvePoliciesForClaims(claims *structs.IdentityClaims) ([]*structs.ACLPolicy, error) {
+	return s.auth.ResolvePoliciesForClaims(claims)
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/acl_test.go

@@ -0,0 +1,329 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"fmt"
+	"path"
+	"testing"
+	"time"
+
+	msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc/v2"
+	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/hashicorp/nomad/nomad/mock"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/nomad/structs/config"
+	"github.com/hashicorp/nomad/testutil"
+	"github.com/shoenig/test"
+	"github.com/shoenig/test/must"
+)
+
+func TestAuthenticate_mTLS(t *testing.T) {
+	ci.Parallel(t)
+
+	// Set up a cluster with mTLS and ACLs
+
+	dir := t.TempDir()
+
+	tlsCfg := &config.TLSConfig{
+		EnableHTTP:           true,
+		EnableRPC:            true,
+		VerifyServerHostname: true,
+		CAFile:               "../helper/tlsutil/testdata/nomad-agent-ca.pem",
+		CertFile:             "../helper/tlsutil/testdata/regionFoo-server-nomad.pem",
+		KeyFile:              "../helper/tlsutil/testdata/regionFoo-server-nomad-key.pem",
+	}
+	clientTLSCfg := tlsCfg.Copy()
+	clientTLSCfg.CertFile = "../helper/tlsutil/testdata/regionFoo-client-nomad.pem"
+	clientTLSCfg.KeyFile = "../helper/tlsutil/testdata/regionFoo-client-nomad-key.pem"
+
+	setCfg := func(name string, bootstrapExpect int) func(*Config) {
+		return func(c *Config) {
+			c.Region = "regionFoo"
+			c.AuthoritativeRegion = "regionFoo"
+			c.ACLEnabled = true
+			c.BootstrapExpect = bootstrapExpect
+			c.NumSchedulers = 0
+			c.DevMode = false
+			c.DataDir = path.Join(dir, name)
+			c.TLSConfig = tlsCfg
+		}
+	}
+
+	leader, cleanupLeader := TestServer(t, setCfg("node1", 1))
+	defer cleanupLeader()
+	testutil.WaitForLeader(t, leader.RPC)
+
+	follower, cleanupFollower := TestServer(t, setCfg("node2", 0))
+	defer cleanupFollower()
+
+	TestJoin(t, leader, follower)
+	testutil.WaitForLeader(t, leader.RPC)
+
+	testutil.Wait(t, func() (bool, error) {
+		cs, err := follower.encrypter.activeCipherSet()
+		return cs != nil, err
+	})
+
+	rootToken := uuid.Generate()
+	var bootstrapResp *structs.ACLTokenUpsertResponse
+
+	codec := rpcClientWithTLS(t, follower, tlsCfg)
+	must.NoError(t, msgpackrpc.CallWithCodec(codec,
+		"ACL.Bootstrap", &structs.ACLTokenBootstrapRequest{
+			BootstrapSecret: rootToken,
+			WriteRequest:    structs.WriteRequest{Region: "regionFoo"},
+		}, &bootstrapResp))
+	must.NotNil(t, bootstrapResp)
+	must.Len(t, 1, bootstrapResp.Tokens)
+	rootAccessor := bootstrapResp.Tokens[0].AccessorID
+
+	// create some ACL tokens directly into raft so we can bypass RPC validation
+	// around expiration times
+
+	token1 := mock.ACLToken()
+	token2 := mock.ACLToken()
+	expireTime := time.Now().Add(time.Second * -10)
+	token2.ExpirationTime = &expireTime
+
+	_, _, err := leader.raftApply(structs.ACLTokenUpsertRequestType,
+		&structs.ACLTokenUpsertRequest{Tokens: []*structs.ACLToken{token1, token2}})
+	must.NoError(t, err)
+
+	// create a node so we can test client RPCs
+
+	node := mock.Node()
+	nodeRegisterReq := &structs.NodeRegisterRequest{
+		Node:         node,
+		WriteRequest: structs.WriteRequest{Region: "regionFoo"},
+	}
+	var nodeRegisterResp structs.NodeUpdateResponse
+
+	must.NoError(t, msgpackrpc.CallWithCodec(codec,
+		"Node.Register", nodeRegisterReq, &nodeRegisterResp))
+	must.NotNil(t, bootstrapResp)
+
+	// create some allocations so we can test WorkloadIdentity claims. we'll
+	// create directly into raft so we can bypass RPC validation and the whole
+	// eval, plan, etc. workflow.
+	job := mock.Job()
+
+	_, _, err = leader.raftApply(structs.JobRegisterRequestType,
+		&structs.JobRegisterRequest{Job: job})
+	must.NoError(t, err)
+
+	alloc1 := mock.Alloc()
+	alloc1.NodeID = node.ID
+	alloc1.ClientStatus = structs.AllocClientStatusFailed
+	alloc1.Job = job
+	alloc1.JobID = job.ID
+
+	alloc2 := mock.Alloc()
+	alloc2.NodeID = node.ID
+	alloc2.Job = job
+	alloc2.JobID = job.ID
+	alloc2.ClientStatus = structs.AllocClientStatusRunning
+
+	wiHandle := &structs.WIHandle{
+		WorkloadIdentifier: "web",
+		WorkloadType:       structs.WorkloadTypeTask,
+	}
+
+	task1 := alloc1.LookupTask("web")
+	claims1 := structs.NewIdentityClaimsBuilder(job, alloc1,
+		wiHandle,
+		task1.Identity).
+		WithTask(task1).
+		Build(time.Now())
+
+	claims1Token, _, err := leader.encrypter.SignClaims(claims1)
+	must.NoError(t, err, must.Sprint("could not sign claims"))
+
+	task2 := alloc2.LookupTask("web")
+	claims2 := structs.NewIdentityClaimsBuilder(job, alloc2,
+		wiHandle,
+		task2.Identity).
+		WithTask(task1).
+		Build(time.Now())
+
+	claims2Token, _, err := leader.encrypter.SignClaims(claims2)
+	must.NoError(t, err, must.Sprint("could not sign claims"))
+
+	planReq := &structs.ApplyPlanResultsRequest{
+		AllocsUpdated: []*structs.Allocation{alloc1, alloc2},
+		Job:           job,
+	}
+	_, _, err = leader.raftApply(structs.ApplyPlanResultsRequestType, planReq)
+	must.NoError(t, err)
+
+	testutil.WaitForResult(func() (bool, error) {
+		store := follower.fsm.State()
+		alloc, err := store.AllocByID(nil, alloc1.ID)
+		return alloc != nil, err
+	}, func(err error) {
+		t.Fatalf("alloc was not replicated via raft: %v", err) // should never happen
+	})
+
+	testCases := []struct {
+		name           string
+		tlsCfg         *config.TLSConfig
+		stale          bool
+		testToken      string
+		expectAccessor string
+		expectClientID string
+		expectAllocID  string
+		expectTLSName  string
+		expectIP       string
+		expectErr      string
+		expectIDKey    string
+		sendFromPeer   *Server
+	}{
+		{
+			name:           "root token",
+			tlsCfg:         clientTLSCfg, // TODO: this is a mixed use cert
+			testToken:      rootToken,
+			expectAccessor: rootAccessor,
+			expectIDKey:    fmt.Sprintf("token:%s", rootAccessor),
+		},
+		{
+			name:           "from peer to leader without token", // ex. Eval.Dequeue
+			tlsCfg:         tlsCfg,
+			expectTLSName:  "server.regionFoo.nomad",
+			expectAccessor: "anonymous",
+			expectIP:       follower.GetConfig().RPCAddr.IP.String(),
+			sendFromPeer:   follower,
+			expectIDKey:    "token:anonymous",
+		},
+		{
+			// note: this test is somewhat bogus because under test all the
+			// servers share the same IP address with the RPC client
+			name:           "anonymous forwarded from peer to leader",
+			tlsCfg:         tlsCfg,
+			expectAccessor: "anonymous",
+			expectTLSName:  "server.regionFoo.nomad",
+			expectIP:       "127.0.0.1",
+			expectIDKey:    "token:anonymous",
+		},
+		{
+			name:          "invalid token",
+			tlsCfg:        clientTLSCfg,
+			testToken:     uuid.Generate(),
+			expectTLSName: "server.regionFoo.nomad",
+			expectIP:      follower.GetConfig().RPCAddr.IP.String(),
+			expectIDKey:   "server.regionFoo.nomad:127.0.0.1",
+			expectErr:     "rpc error: Permission denied",
+		},
+		{
+			name:           "from peer to leader with leader ACL", // ex. core job GC
+			tlsCfg:         tlsCfg,
+			testToken:      leader.getLeaderAcl(),
+			expectTLSName:  "server.regionFoo.nomad",
+			expectAccessor: "leader",
+			expectIP:       follower.GetConfig().RPCAddr.IP.String(),
+			sendFromPeer:   follower,
+			expectIDKey:    "token:leader",
+		},
+		{
+			name:           "from client", // ex. Node.GetAllocs
+			tlsCfg:         clientTLSCfg,
+			testToken:      node.SecretID,
+			expectClientID: node.ID,
+			expectIDKey:    fmt.Sprintf("client:%s", node.ID),
+		},
+		{
+			name:           "from client missing secret", // ex. Node.Register
+			tlsCfg:         clientTLSCfg,
+			expectAccessor: "anonymous",
+			expectTLSName:  "server.regionFoo.nomad",
+			expectIP:       follower.GetConfig().RPCAddr.IP.String(),
+		},
+		{
+			name:      "from failed workload", // ex. Variables.List
+			tlsCfg:    clientTLSCfg,
+			testToken: claims1Token,
+			expectErr: "rpc error: allocation is terminal",
+		},
+		{
+			name:          "from running workload", // ex. Variables.List
+			tlsCfg:        clientTLSCfg,
+			testToken:     claims2Token,
+			expectAllocID: alloc2.ID,
+			expectIDKey:   fmt.Sprintf("alloc:%s", alloc2.ID),
+		},
+		{
+			name:           "valid user token",
+			tlsCfg:         clientTLSCfg,
+			testToken:      token1.SecretID,
+			expectAccessor: token1.AccessorID,
+			expectIDKey:    fmt.Sprintf("token:%s", token1.AccessorID),
+		},
+		{
+			name:      "expired user token",
+			tlsCfg:    clientTLSCfg,
+			testToken: token2.SecretID,
+			expectErr: "rpc error: ACL token expired",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+
+			req := &structs.GenericRequest{
+				QueryOptions: structs.QueryOptions{
+					Region:     "regionFoo",
+					AllowStale: tc.stale,
+					AuthToken:  tc.testToken,
+				},
+			}
+			var resp structs.ACLWhoAmIResponse
+			var err error
+
+			if tc.sendFromPeer != nil {
+				aclEndpoint := NewACLEndpoint(tc.sendFromPeer, nil)
+				err = aclEndpoint.WhoAmI(req, &resp)
+			} else {
+				err = msgpackrpc.CallWithCodec(codec, "ACL.WhoAmI", req, &resp)
+			}
+
+			if tc.expectErr != "" {
+				must.EqError(t, err, tc.expectErr)
+				return
+			}
+
+			must.NoError(t, err)
+			must.NotNil(t, resp)
+			must.NotNil(t, resp.Identity)
+
+			if tc.expectIDKey != "" {
+				must.Eq(t, tc.expectIDKey, resp.Identity.String(),
+					must.Sprintf("expected identity key for metrics to match"))
+			}
+
+			if tc.expectAccessor != "" {
+				must.NotNil(t, resp.Identity.ACLToken, must.Sprint("expected ACL token"))
+				test.Eq(t, tc.expectAccessor, resp.Identity.ACLToken.AccessorID,
+					test.Sprint("expected ACL token accessor ID"))
+			}
+
+			test.Eq(t, tc.expectClientID, resp.Identity.ClientID,
+				test.Sprint("expected client ID"))
+
+			if tc.expectAllocID != "" {
+				must.NotNil(t, resp.Identity.Claims, must.Sprint("expected claims"))
+				test.Eq(t, tc.expectAllocID, resp.Identity.Claims.AllocationID,
+					test.Sprint("expected workload identity"))
+			}
+
+			test.Eq(t, tc.expectTLSName, resp.Identity.TLSName, test.Sprint("expected TLS name"))
+
+			if tc.expectIP == "" {
+				test.Nil(t, resp.Identity.RemoteIP, test.Sprint("expected no remote IP"))
+			} else {
+				test.Eq(t, tc.expectIP, resp.Identity.RemoteIP.String())
+			}
+
+		})
+	}
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/alloc_endpoint.go

@@ -0,0 +1,696 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-memdb"
+	metrics "github.com/hashicorp/go-metrics/compat"
+
+	"github.com/hashicorp/nomad/acl"
+	"github.com/hashicorp/nomad/helper/pointer"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/hashicorp/nomad/nomad/state"
+	"github.com/hashicorp/nomad/nomad/state/paginator"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+// Alloc endpoint is used for manipulating allocations
+type Alloc struct {
+	srv    *Server
+	ctx    *RPCContext
+	logger hclog.Logger
+}
+
+func NewAllocEndpoint(srv *Server, ctx *RPCContext) *Alloc {
+	return &Alloc{srv: srv, ctx: ctx, logger: srv.logger.Named("alloc")}
+}
+
+// List is used to list the allocations in the system
+func (a *Alloc) List(args *structs.AllocListRequest, reply *structs.AllocListResponse) error {
+	authErr := a.srv.Authenticate(a.ctx, args)
+	if done, err := a.srv.forward("Alloc.List", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("alloc", structs.RateMetricList, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "alloc", "list"}, time.Now())
+
+	namespace := args.RequestNamespace()
+
+	// Check namespace read-job permissions
+	aclObj, err := a.srv.ResolveACL(args)
+	if err != nil {
+		return err
+	}
+	if !aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityReadJob) {
+		return structs.ErrPermissionDenied
+	}
+	allow := aclObj.AllowNsOpFunc(acl.NamespaceCapabilityReadJob)
+
+	// Setup the blocking query
+	sort := state.SortOption(args.Reverse)
+	opts := blockingOptions{
+		queryOpts: &args.QueryOptions,
+		queryMeta: &reply.QueryMeta,
+		run: func(ws memdb.WatchSet, state *state.StateStore) error {
+			// Scan all the allocations
+			var err error
+			var iter memdb.ResultIterator
+
+			// get list of accessible namespaces
+			allowableNamespaces, err := allowedNSes(aclObj, state, allow)
+			if err == structs.ErrPermissionDenied {
+				// return empty allocation if token is not authorized for any
+				// namespace, matching other endpoints
+				reply.Allocations = make([]*structs.AllocListStub, 0)
+			} else if err != nil {
+				return err
+			} else {
+				var tokenizer paginator.Tokenizer[*structs.Allocation]
+
+				if prefix := args.QueryOptions.Prefix; prefix != "" {
+					iter, err = state.AllocsByIDPrefix(ws, namespace, prefix, sort)
+					tokenizer = paginator.IDTokenizer[*structs.Allocation](args.NextToken)
+				} else if namespace != structs.AllNamespacesSentinel {
+					iter, err = state.AllocsByNamespaceOrdered(ws, namespace, sort)
+					tokenizer = paginator.CreateIndexAndIDTokenizer[*structs.Allocation](args.NextToken)
+				} else {
+					iter, err = state.Allocs(ws, sort)
+					tokenizer = paginator.CreateIndexAndIDTokenizer[*structs.Allocation](args.NextToken)
+				}
+				if err != nil {
+					return err
+				}
+
+				pager, err := paginator.NewPaginator(iter, args.QueryOptions,
+					paginator.NamespaceSelectorFunc[*structs.Allocation](allowableNamespaces),
+					tokenizer,
+					func(a *structs.Allocation) (*structs.AllocListStub, error) {
+						return a.Stub(args.Fields), nil
+					},
+				)
+				if err != nil {
+					return structs.NewErrRPCCodedf(
+						http.StatusBadRequest, "failed to create result paginator: %v", err)
+				}
+
+				stubs, nextToken, err := pager.Page()
+				if err != nil {
+					return structs.NewErrRPCCodedf(
+						http.StatusBadRequest, "failed to read result page: %v", err)
+				}
+
+				reply.QueryMeta.NextToken = nextToken
+				reply.Allocations = stubs
+			}
+
+			// Use the last index that affected the allocs table
+			index, err := state.Index("allocs")
+			if err != nil {
+				return err
+			}
+			reply.Index = index
+
+			// Set the query response
+			a.srv.setQueryMeta(&reply.QueryMeta)
+			return nil
+		}}
+	return a.srv.blockingRPC(&opts)
+}
+
+// GetAlloc is used to lookup a particular allocation
+func (a *Alloc) GetAlloc(args *structs.AllocSpecificRequest,
+	reply *structs.SingleAllocResponse) error {
+	authErr := a.srv.Authenticate(a.ctx, args)
+	if done, err := a.srv.forward("Alloc.GetAlloc", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("alloc", structs.RateMetricRead, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "alloc", "get_alloc"}, time.Now())
+
+	// Check namespace read-job permissions before performing blocking query.
+	allowNsOp := acl.NamespaceValidator(acl.NamespaceCapabilityReadJob)
+	aclObj, err := a.srv.ResolveACL(args)
+	if err != nil {
+		return err
+	}
+
+	// Setup the blocking query
+	opts := blockingOptions{
+		queryOpts: &args.QueryOptions,
+		queryMeta: &reply.QueryMeta,
+		run: func(ws memdb.WatchSet, state *state.StateStore) error {
+			// Lookup the allocation
+			out, err := state.AllocByID(ws, args.AllocID)
+			if err != nil {
+				return err
+			}
+
+			// Setup the output
+			if out != nil {
+				if out.SignedIdentities != nil {
+					out = out.Sanitize()
+				}
+				reply.Alloc = out
+
+				// Re-check namespace in case it differs from request.
+				if err := a.srv.AuthorizeClientAllocation(aclObj, out, allowNsOp); err != nil {
+					return structs.NewErrUnknownAllocation(args.AllocID)
+				}
+
+				reply.Index = out.ModifyIndex
+			} else {
+				// Use the last index that affected the allocs table
+				index, err := state.Index("allocs")
+				if err != nil {
+					return err
+				}
+				reply.Index = index
+			}
+
+			// Set the query response
+			a.srv.setQueryMeta(&reply.QueryMeta)
+			return nil
+		}}
+	return a.srv.blockingRPC(&opts)
+}
+
+// GetAllocs is used to lookup a set of allocations
+func (a *Alloc) GetAllocs(args *structs.AllocsGetRequest,
+	reply *structs.AllocsGetResponse) error {
+
+	aclObj, err := a.srv.AuthenticateClientOnly(a.ctx, args)
+	if done, err := a.srv.forward("Alloc.GetAllocs", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("alloc", structs.RateMetricWrite, args)
+	if err != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "alloc", "get_allocs"}, time.Now())
+
+	allocs := make([]*structs.Allocation, len(args.AllocIDs))
+
+	// Setup the blocking query. We wait for at least one of the requested
+	// allocations to be above the min query index. This guarantees that the
+	// server has received that index.
+	opts := blockingOptions{
+		queryOpts: &args.QueryOptions,
+		queryMeta: &reply.QueryMeta,
+		run: func(ws memdb.WatchSet, state *state.StateStore) error {
+			// Lookup the allocation
+			thresholdMet := false
+			maxIndex := uint64(0)
+			for i, alloc := range args.AllocIDs {
+				out, err := state.AllocByID(ws, alloc)
+				if err != nil {
+					return err
+				}
+				if out == nil {
+					// The alloc may have been GC'd or purged. Continue so we can
+					// potentially satisfy the query with partial results from the
+					// remaining allocs.
+					continue
+				}
+
+				// It shouldn't be possible to have an alloc with a nil job, but
+				// let's not block the node from getting other allocs if that's
+				// the case.
+				if out.Job == nil {
+					continue
+				}
+
+				if err := a.srv.AuthorizeClientAllocation(aclObj, out, nil); err != nil {
+					return err
+				}
+
+				// Store the pointer
+				allocs[i] = out
+
+				// Check if we have passed the minimum index
+				if out.ModifyIndex > args.QueryOptions.MinQueryIndex {
+					thresholdMet = true
+				}
+
+				if maxIndex < out.ModifyIndex {
+					maxIndex = out.ModifyIndex
+				}
+			}
+
+			// Setup the output
+			if thresholdMet {
+				reply.Allocs = allocs
+				reply.Index = maxIndex
+			} else {
+				// Use the last index that affected the allocs table
+				index, err := state.Index("allocs")
+				if err != nil {
+					return err
+				}
+				reply.Index = index
+			}
+
+			// Set the query response
+			a.srv.setQueryMeta(&reply.QueryMeta)
+			return nil
+		},
+	}
+	return a.srv.blockingRPC(&opts)
+}
+
+// Stop is used to stop an allocation and migrate it to another node.
+func (a *Alloc) Stop(args *structs.AllocStopRequest, reply *structs.AllocStopResponse) error {
+
+	authErr := a.srv.Authenticate(a.ctx, args)
+	if done, err := a.srv.forward("Alloc.Stop", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("alloc", structs.RateMetricWrite, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+
+	defer metrics.MeasureSince([]string{"nomad", "alloc", "stop"}, time.Now())
+
+	alloc, err := getAlloc(a.srv.State(), args.AllocID)
+	if err != nil {
+		return err
+	}
+
+	// Check for namespace alloc-lifecycle permissions.
+	allowNsOp := acl.NamespaceValidator(acl.NamespaceCapabilityAllocLifecycle)
+	aclObj, err := a.srv.ResolveACL(args)
+	if err != nil {
+		return err
+	} else if !allowNsOp(aclObj, alloc.Namespace) {
+		return structs.ErrPermissionDenied
+	}
+
+	now := time.Now().UTC().UnixNano()
+	eval := &structs.Evaluation{
+		ID:             uuid.Generate(),
+		Namespace:      alloc.Namespace,
+		Priority:       alloc.Job.Priority,
+		Type:           alloc.Job.Type,
+		TriggeredBy:    structs.EvalTriggerAllocStop,
+		JobID:          alloc.Job.ID,
+		JobModifyIndex: alloc.Job.ModifyIndex,
+		Status:         structs.EvalStatusPending,
+		CreateTime:     now,
+		ModifyTime:     now,
+	}
+
+	transitionReq := &structs.AllocUpdateDesiredTransitionRequest{
+		Evals: []*structs.Evaluation{eval},
+		Allocs: map[string]*structs.DesiredTransition{
+			args.AllocID: {
+				Migrate:         pointer.Of(true),
+				NoShutdownDelay: pointer.Of(args.NoShutdownDelay),
+				Reschedule:      pointer.Of(args.Reschedule),
+			},
+		},
+	}
+
+	// Commit this update via Raft
+	_, index, err := a.srv.raftApply(structs.AllocUpdateDesiredTransitionRequestType, transitionReq)
+	if err != nil {
+		a.logger.Error("AllocUpdateDesiredTransitionRequest failed", "error", err)
+		return err
+	}
+
+	// Setup the response
+	reply.Index = index
+	reply.EvalID = eval.ID
+	return nil
+}
+
+// UpdateDesiredTransition is used to update the desired transitions of an
+// allocation.
+func (a *Alloc) UpdateDesiredTransition(args *structs.AllocUpdateDesiredTransitionRequest, reply *structs.GenericResponse) error {
+	authErr := a.srv.Authenticate(a.ctx, args)
+	if done, err := a.srv.forward("Alloc.UpdateDesiredTransition", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("alloc", structs.RateMetricWrite, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+
+	defer metrics.MeasureSince([]string{"nomad", "alloc", "update_desired_transition"}, time.Now())
+
+	// Check that it is a management token.
+	if aclObj, err := a.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.IsManagement() {
+		return structs.ErrPermissionDenied
+	}
+
+	// Ensure at least a single alloc
+	if len(args.Allocs) == 0 {
+		return fmt.Errorf("must update at least one allocation")
+	}
+
+	// Commit this update via Raft
+	_, index, err := a.srv.raftApply(structs.AllocUpdateDesiredTransitionRequestType, args)
+	if err != nil {
+		a.logger.Error("AllocUpdateDesiredTransitionRequest failed", "error", err)
+		return err
+	}
+
+	// Setup the response
+	reply.Index = index
+	return nil
+}
+
+// GetServiceRegistrations returns a list of service registrations which belong
+// to the passed allocation ID.
+func (a *Alloc) GetServiceRegistrations(
+	args *structs.AllocServiceRegistrationsRequest,
+	reply *structs.AllocServiceRegistrationsResponse) error {
+
+	authErr := a.srv.Authenticate(a.ctx, args)
+	if done, err := a.srv.forward(structs.AllocServiceRegistrationsRPCMethod, args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("alloc", structs.RateMetricList, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+
+	defer metrics.MeasureSince([]string{"nomad", "alloc", "get_service_registrations"}, time.Now())
+
+	// Ensure the caller has the read-job namespace capability.
+	aclObj, err := a.srv.ResolveACL(args)
+	if err != nil {
+		return err
+	}
+	if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) {
+		return structs.ErrPermissionDenied
+	}
+
+	// Set up the blocking query.
+	return a.srv.blockingRPC(&blockingOptions{
+		queryOpts: &args.QueryOptions,
+		queryMeta: &reply.QueryMeta,
+		run: func(ws memdb.WatchSet, stateStore *state.StateStore) error {
+
+			// Read the allocation to ensure its namespace matches the request
+			// args.
+			alloc, err := stateStore.AllocByID(ws, args.AllocID)
+			if err != nil {
+				return err
+			}
+
+			// Guard against the alloc not-existing or that the namespace does
+			// not match the request arguments.
+			if alloc == nil || alloc.Namespace != args.RequestNamespace() {
+				return nil
+			}
+
+			// Perform the state query to get an iterator.
+			iter, err := stateStore.GetServiceRegistrationsByAllocID(ws, args.AllocID)
+			if err != nil {
+				return err
+			}
+
+			// Set up our output after we have checked the error.
+			services := make([]*structs.ServiceRegistration, 0)
+
+			// Iterate the iterator, appending all service registrations
+			// returned to the reply.
+			for raw := iter.Next(); raw != nil; raw = iter.Next() {
+				services = append(services, raw.(*structs.ServiceRegistration))
+			}
+			reply.Services = services
+
+			// Use the index table to populate the query meta as we have no way
+			// of tracking the max index on deletes.
+			return a.srv.setReplyQueryMeta(stateStore, state.TableServiceRegistrations, &reply.QueryMeta)
+		},
+	})
+}
+
+// SignIdentities allows nodes to retrieve workload identities for their
+// allocations.
+//
+// This is an internal-only RPC and not exposed via the HTTP API.
+func (a *Alloc) SignIdentities(args *structs.AllocIdentitiesRequest, reply *structs.AllocIdentitiesResponse) error {
+
+	aclObj, err := a.srv.AuthenticateClientOnly(a.ctx, args)
+	if done, err := a.srv.forward("Alloc.SignIdentities", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("alloc", structs.RateMetricRead, args)
+	if err != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "alloc", "sign_identities"}, time.Now())
+
+	if len(args.Identities) == 0 {
+		// Client bug. Fail loudly instead of letting clients waste time with
+		// noops.
+		return fmt.Errorf("no identities requested")
+	}
+
+	// Tracks whether the min index was satisfied by the blocking query
+	thresholdMet := false
+
+	// Most if not all identity requests will be for the same alloc, so create a
+	// set of alloc IDs to avoid unnecessary looping in the blocking query.
+	allocs := make(map[string]*structs.Allocation, len(args.Identities))
+	for _, idReq := range args.Identities {
+		allocs[idReq.AllocID] = nil // to be set while watching
+	}
+
+	opts := blockingOptions{
+		queryOpts: &args.QueryOptions,
+		queryMeta: &reply.QueryMeta,
+		run: func(ws memdb.WatchSet, state *state.StateStore) error {
+			var maxIndex uint64
+
+			// Lookup the allocations
+			for allocID := range allocs {
+				out, err := state.AllocByID(ws, allocID)
+				if err != nil {
+					return err
+				}
+
+				if out == nil {
+					// Alloc may have been GC'd and therefore should not be able to get
+					// identities signed.
+					continue
+				}
+
+				if err := a.srv.AuthorizeClientAllocation(aclObj, out, nil); err != nil {
+					return err
+				}
+
+				// Keep the alloc around so we can skip the statestore lookup later
+				allocs[allocID] = out
+
+				// If we have found a requested alloc created after the min query index
+				// that means we're observing a new enough version of state to satisfy
+				// the query.
+				if out.CreateIndex > args.QueryOptions.MinQueryIndex {
+					thresholdMet = true
+				}
+
+				if maxIndex < out.CreateIndex {
+					maxIndex = out.CreateIndex
+				}
+			}
+
+			// If we could not find an alloc created after the min index, note when
+			// the index allocs were last updated.
+			if !thresholdMet {
+				index, err := state.Index("allocs")
+				if err != nil {
+					return err
+				}
+				maxIndex = index
+			}
+
+			reply.Index = maxIndex
+			return nil
+		}}
+
+	if err := a.srv.blockingRPC(&opts); err != nil {
+		return err
+	}
+
+	// Index threshold was not met in the blocking query. Set rejections since
+	// allocs could not be found and should be considered invalid.
+	if !thresholdMet {
+		for _, idReq := range args.Identities {
+			reply.Rejections = append(reply.Rejections, &structs.WorkloadIdentityRejection{
+				WorkloadIdentityRequest: *idReq,
+				Reason:                  structs.WIRejectionReasonMissingAlloc,
+			})
+		}
+
+		// Return early so the rest of the func acts as the else (thresholdMet)
+		return nil
+	}
+
+	// Threshold met, so create the response
+	now := time.Now().UTC()
+	for _, idReq := range args.Identities {
+		out := allocs[idReq.AllocID]
+
+		if out == nil {
+			// Alloc may have been GC'd and therefore should not be able to get
+			// new identities.
+			reply.Rejections = append(reply.Rejections, &structs.WorkloadIdentityRejection{
+				WorkloadIdentityRequest: *idReq,
+				Reason:                  structs.WIRejectionReasonMissingAlloc,
+			})
+			continue
+		}
+
+		job := out.Job
+
+		switch idReq.WorkloadType {
+		case structs.WorkloadTypeTask:
+			task := out.LookupTask(idReq.WorkloadIdentifier)
+			if task == nil {
+				// Job has likely been updated to remove this task
+				reply.Rejections = append(reply.Rejections, &structs.WorkloadIdentityRejection{
+					WorkloadIdentityRequest: *idReq,
+					Reason:                  structs.WIRejectionReasonMissingTask,
+				})
+				continue
+			}
+
+			widFound, err := a.signTasks(task, out, idReq, reply, now)
+			if err != nil {
+				return err
+			}
+
+			if !widFound {
+				reply.Rejections = append(reply.Rejections, &structs.WorkloadIdentityRejection{
+					WorkloadIdentityRequest: *idReq,
+					Reason:                  structs.WIRejectionReasonMissingIdentity,
+				})
+				continue
+			}
+
+		case structs.WorkloadTypeService:
+			widFound, err := a.signServices(job, out, idReq, reply, now)
+			if err != nil {
+				return err
+			}
+
+			if !widFound {
+				reply.Rejections = append(reply.Rejections, &structs.WorkloadIdentityRejection{
+					WorkloadIdentityRequest: *idReq,
+					Reason:                  structs.WIRejectionReasonMissingIdentity,
+				})
+				continue
+			}
+		}
+	}
+	return nil
+}
+
+func (a *Alloc) signTasks(
+	task *structs.Task,
+	alloc *structs.Allocation,
+	idReq *structs.WorkloadIdentityRequest,
+	reply *structs.AllocIdentitiesResponse,
+	now time.Time,
+) (widFound bool, err error) {
+	for _, wid := range task.Identities {
+		if wid.Name != idReq.IdentityName {
+			continue
+		}
+
+		widFound = true
+		builder := structs.NewIdentityClaimsBuilder(alloc.Job, alloc, &idReq.WIHandle, wid).
+			WithTask(task).
+			WithConsul()
+
+		var node *structs.Node
+		node, err = a.srv.State().NodeByID(nil, alloc.NodeID)
+		if err != nil {
+			return
+		}
+		builder.WithNode(node)
+
+		vaultCfg := a.srv.GetConfig().GetVaultForIdentity(wid)
+		if vaultCfg != nil && vaultCfg.DefaultIdentity != nil {
+			builder.WithVault(vaultCfg.DefaultIdentity.ExtraClaims)
+		}
+
+		claims := builder.Build(now)
+		err = a.signClaims(claims, idReq, reply)
+		break
+	}
+	return
+}
+
+func (a *Alloc) signServices(
+	job *structs.Job,
+	alloc *structs.Allocation,
+	idReq *structs.WorkloadIdentityRequest,
+	reply *structs.AllocIdentitiesResponse,
+	now time.Time,
+) (widFound bool, err error) {
+	wid := idReq.WIHandle
+
+	// services can be on the level of task groups or tasks
+	for _, tg := range job.TaskGroups {
+		for _, service := range tg.Services {
+			if service.IdentityHandle(nil).Equal(wid) {
+				claims := structs.NewIdentityClaimsBuilder(
+					alloc.Job, alloc, &idReq.WIHandle, service.Identity).
+					WithConsul().
+					WithService(service).
+					Build(now)
+				return true, a.signClaims(claims, idReq, reply)
+			}
+		}
+		for _, task := range tg.Tasks {
+			for _, service := range task.Services {
+				if service.IdentityHandle(nil).Equal(wid) {
+					claims := structs.NewIdentityClaimsBuilder(
+						alloc.Job, alloc, &idReq.WIHandle, service.Identity).
+						WithTask(task).
+						WithConsul().
+						WithService(service).
+						Build(now)
+					return true, a.signClaims(claims, idReq, reply)
+				}
+			}
+		}
+	}
+	return
+}
+
+func (a *Alloc) signClaims(
+	claims *structs.IdentityClaims,
+	idReq *structs.WorkloadIdentityRequest,
+	reply *structs.AllocIdentitiesResponse,
+) error {
+	token, _, err := a.srv.encrypter.SignClaims(claims)
+	if err != nil {
+		return err
+	}
+	reply.SignedIdentities = append(reply.SignedIdentities, &structs.SignedWorkloadIdentity{
+		WorkloadIdentityRequest: *idReq,
+		JWT:                     token,
+		Expiration:              claims.Expiry.Time(),
+	})
+
+	return nil
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/auth/auth.go

@@ -0,0 +1,957 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package auth
+
+import (
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net"
+	"slices"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/hashicorp/go-hclog"
+	metrics "github.com/hashicorp/go-metrics/compat"
+	"github.com/hashicorp/nomad/acl"
+	"github.com/hashicorp/nomad/helper"
+	"github.com/hashicorp/nomad/nomad/state"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+// aclCacheSize is the number of ACL objects to keep cached. ACLs have a parsing
+// and construction cost, so we keep the hot objects cached to reduce the ACL
+// token resolution time.
+const aclCacheSize = 512
+
+type StateGetter func() *state.StateStore
+type LeaderACLGetter func() string
+
+type RPCContext interface {
+	IsTLS() bool
+	IsStatic() bool
+	Certificate() *x509.Certificate
+	GetRemoteIP() (net.IP, error)
+}
+
+type Encrypter interface {
+	VerifyClaim(string) (*structs.IdentityClaims, error)
+}
+
+type Authenticator struct {
+	aclsEnabled bool
+
+	// verifyTLS is used to determine whether the server should verify TLS and
+	// is an atomic bool, so that the server TLS reload can update it at runtime
+	// with a race condition.
+	verifyTLS *atomic.Bool
+
+	logger       hclog.Logger
+	getState     StateGetter
+	getLeaderACL LeaderACLGetter
+	region       string
+
+	validServerCertNames []string
+	validClientCertNames []string
+
+	// aclCache is used to maintain the parsed ACL objects
+	aclCache *structs.ACLCache[*acl.ACL]
+
+	// encrypter is a pointer to the server's Encrypter that can be used to
+	// verify claims
+	encrypter Encrypter
+}
+
+type AuthenticatorConfig struct {
+	StateFn        StateGetter
+	Logger         hclog.Logger
+	GetLeaderACLFn LeaderACLGetter
+	AclsEnabled    bool
+	VerifyTLS      bool
+	Region         string
+	Encrypter      Encrypter
+}
+
+func NewAuthenticator(cfg *AuthenticatorConfig) *Authenticator {
+	a := Authenticator{
+		aclsEnabled:          cfg.AclsEnabled,
+		verifyTLS:            &atomic.Bool{},
+		logger:               cfg.Logger.With("auth"),
+		getState:             cfg.StateFn,
+		getLeaderACL:         cfg.GetLeaderACLFn,
+		region:               cfg.Region,
+		aclCache:             structs.NewACLCache[*acl.ACL](aclCacheSize),
+		encrypter:            cfg.Encrypter,
+		validServerCertNames: []string{"server." + cfg.Region + ".nomad"},
+		validClientCertNames: []string{
+			"client." + cfg.Region + ".nomad",
+			"server." + cfg.Region + ".nomad",
+		},
+	}
+
+	a.verifyTLS.Store(cfg.VerifyTLS)
+	return &a
+}
+
+// SetVerifyTLS is a helper method to set the verifyTLS field. This is used in
+// when the server TLS configuration is updated.
+func (s *Authenticator) SetVerifyTLS(verifyTLS bool) { s.verifyTLS.Store(verifyTLS) }
+
+// Authenticate extracts an AuthenticatedIdentity from the request context or
+// provided token and sets the identity on the request. The caller can extract
+// an acl.ACL, WorkloadIdentity, or other identifying tokens to use for
+// authorization. Keeping these fields independent rather than merging them into
+// an ephemeral ACLToken makes the original of the credential clear to RPC
+// handlers, who may have different behavior for internal vs external origins.
+//
+// Note: when making a server-to-server RPC that authenticates with this method,
+// the RPC *must* include the leader's ACL token. Use AuthenticateServerOnly for
+// requests that don't have access to the leader's ACL token.
+//
+// Note: when called on the follower we'll be making stale queries, so it's
+// possible if the follower is behind that the leader will get a different value
+// if an ACL token or allocation's WI has just been created.
+//
+// This method returns errors that are used for testing diagnostics. RPC callers
+// should always return ErrPermissionDenied after checking forwarding when one
+// of these errors is received.
+func (s *Authenticator) Authenticate(ctx RPCContext, args structs.RequestWithIdentity) error {
+
+	// get the user ACLToken or anonymous token
+	secretID := args.GetAuthToken()
+	aclToken, err := s.resolveSecretToken(secretID)
+
+	switch {
+	case err == nil && (aclToken == structs.AnonymousACLToken ||
+		aclToken == structs.ACLsDisabledToken):
+		// When ACLs are disabled or if we have an anonymous token, we want to
+		// continue on to check mTLS certs, if available, so set the token but
+		// don't return yet
+		args.SetIdentity(&structs.AuthenticatedIdentity{ACLToken: aclToken})
+
+	case err == nil:
+		// ACLs are enabled and we have a non-anonymous token, so set that as
+		// our identity and return
+		args.SetIdentity(&structs.AuthenticatedIdentity{ACLToken: aclToken})
+		return nil
+
+	case errors.Is(err, structs.ErrTokenExpired):
+		return err
+
+	case errors.Is(err, structs.ErrTokenInvalid):
+		// if it's not a UUID it might be an identity claim
+		claims, err := s.VerifyClaim(secretID)
+		if err != nil {
+			// we already know the token wasn't valid for an ACL in the state
+			// store, so if we get an error at this point we have an invalid
+			// token and there are no other options but to bail out
+			return err
+		}
+
+		args.SetIdentity(&structs.AuthenticatedIdentity{Claims: claims})
+		return nil
+
+	case errors.Is(err, structs.ErrTokenNotFound):
+		// Check if the secret ID is the leader's secret ID, in which case treat
+		// it as a management token.
+		leaderAcl := s.getLeaderACL()
+		if leaderAcl != "" && secretID == leaderAcl {
+			aclToken = structs.LeaderACLToken
+			break
+		} else {
+			// Otherwise, see if the secret ID belongs to a node. We should
+			// reach this point only on first connection.
+			node, err := s.getState().NodeBySecretID(nil, secretID)
+			if err != nil {
+				// this is a go-memdb error; shouldn't happen
+				return fmt.Errorf("could not resolve node secret: %w", err)
+			}
+			if node != nil {
+				args.SetIdentity(&structs.AuthenticatedIdentity{ClientID: node.ID})
+				return nil
+			}
+		}
+
+		// we were passed a bogus token so we'll return an error, but we'll also
+		// want to capture the IP for metrics
+		remoteIP, err := ctx.GetRemoteIP()
+		if err != nil {
+			s.logger.Error("could not determine remote address", "error", err)
+		}
+		args.SetIdentity(&structs.AuthenticatedIdentity{RemoteIP: remoteIP})
+		return structs.ErrPermissionDenied
+
+	default: // any other error
+		return fmt.Errorf("could not resolve user: %w", err)
+
+	}
+
+	// If there's no context we're in a "static" handler which only happens for
+	// cases where the leader is making RPCs internally (volumewatcher and
+	// deploymentwatcher)
+	if ctx.IsStatic() {
+		args.SetIdentity(&structs.AuthenticatedIdentity{ACLToken: aclToken})
+		return nil
+	}
+
+	// At this point we either have an anonymous token or an invalid one.
+
+	// Unlike clients that provide their Node ID on first connection, server
+	// RPCs don't include an ID for the server so we identify servers by cert
+	// and IP address.
+	identity := &structs.AuthenticatedIdentity{ACLToken: aclToken}
+	if ctx.IsTLS() {
+		identity.TLSName = ctx.Certificate().Subject.CommonName
+	}
+
+	remoteIP, err := ctx.GetRemoteIP()
+	if err != nil {
+		s.logger.Error(
+			"could not authenticate RPC request or determine remote address", "error", err)
+		return err
+	}
+	identity.RemoteIP = remoteIP
+	args.SetIdentity(identity)
+	return nil
+}
+
+// ResolveACL is an authentication wrapper that handles resolving ACL
+// tokens, Workload Identities, or client secrets into acl.ACL objects.
+// Exclusively server-to-server or client-to-server requests should be using
+// AuthenticateServerOnly or AuthenticateClientOnly unless they use the
+// AuthenticateNodeIdentityGenerator function.
+func (s *Authenticator) ResolveACL(args structs.RequestWithIdentity) (*acl.ACL, error) {
+	identity := args.GetIdentity()
+	if identity == nil {
+		// should never happen
+		return nil, structs.ErrPermissionDenied
+	}
+
+	if !s.aclsEnabled {
+		return acl.ACLsDisabledACL, nil
+	}
+
+	if aclObj, err := s.ResolveClientIdentityACL(identity); aclObj != nil || err != nil {
+		return aclObj, err
+	}
+
+	claims := identity.GetClaims()
+	if claims != nil {
+		return s.resolveClaims(claims)
+	}
+
+	// this will include any anonymous token, so this is the last chance to
+	// avoid an error
+	aclToken := identity.GetACLToken()
+	if aclToken != nil {
+		return s.resolveACLForToken(aclToken)
+	}
+
+	return nil, structs.ErrPermissionDenied
+}
+
+// AuthenticateServerOnly returns an ACL object for use *only* with internal
+// server-to-server RPCs. This should never be used for RPCs that serve HTTP
+// endpoints or accept ACL tokens to avoid confused deputy attacks by making a
+// request to a follower that's forwarded.
+//
+// The returned ACL object is always an acl.ServerACL but in the future this
+// could be extended to allow servers to have jurisdiction over specific pools,
+// etc.
+func (s *Authenticator) AuthenticateServerOnly(ctx RPCContext, args structs.RequestWithIdentity) (*acl.ACL, error) {
+
+	remoteIP, err := ctx.GetRemoteIP() // capture for metrics
+	if err != nil {
+		s.logger.Error("could not determine remote address", "error", err)
+	}
+
+	identity := &structs.AuthenticatedIdentity{RemoteIP: remoteIP}
+	defer args.SetIdentity(identity) // always set the identity, even on errors
+
+	// Note: if servers had auth tokens like clients do, we would be able to
+	// verify them here and only return the server ACL for actual servers even
+	// if mTLS was disabled. Without mTLS, any request can spoof server RPCs.
+	// This is known and documented in the Security Model:
+	// https://developer.hashicorp.com/nomad/docs/secure/acl
+	if err := verifyTLS(s.verifyTLS.Load(), ctx, s.validServerCertNames, identity); err != nil {
+		return nil, err
+	}
+
+	return acl.ServerACL, nil
+}
+
+// AuthenticateNodeIdentityGenerator is used for RPC endpoints (Node.Register
+// and Node.UpdateStatus) that have the potential to generate node identities.
+//
+// While the Authenticate method serves as a complete general purpose
+// authenticator, in some critical cases for identity generation checking, it
+// swallows the information needed.
+func (s *Authenticator) AuthenticateNodeIdentityGenerator(ctx RPCContext, args structs.RequestWithIdentity) error {
+
+	remoteIP, err := ctx.GetRemoteIP() // capture for metrics
+	if err != nil {
+		s.logger.Error("could not determine remote address", "error", err)
+	}
+
+	identity := &structs.AuthenticatedIdentity{RemoteIP: remoteIP}
+	defer args.SetIdentity(identity)
+
+	if err := verifyTLS(s.verifyTLS.Load(), ctx, s.validClientCertNames, identity); err != nil {
+		return err
+	}
+
+	authToken := args.GetAuthToken()
+
+	// If the auth token is empty, we treat it as an anonymous request. In the
+	// event of a node registration, this means the node is not yet registered.
+	if authToken == "" {
+		identity.ACLToken = structs.AnonymousACLToken
+		return nil
+	}
+
+	// If the auth token is a UUID, we check whether it's a node secret ID or
+	// the leader's ACL token. If it's not a UUID, we assume it's a node
+	// identity. Anything outside these cases is not supported and no identity
+	// will be set.
+	if helper.IsUUID(authToken) {
+		if leaderACL := s.getLeaderACL(); leaderACL != "" && authToken == leaderACL {
+			identity.ACLToken = structs.LeaderACLToken
+		} else {
+			node, err := s.getState().NodeBySecretID(nil, authToken)
+			if err != nil {
+				return fmt.Errorf("could not resolve node secret: %w", err)
+			}
+			if node == nil {
+				return structs.ErrPermissionDenied
+			}
+			identity.ClientID = node.ID
+		}
+	} else {
+		// When verifying a node identity claim, we do not want to swallow the
+		// initial error. This is because the caller may want to handle the
+		// error type in the case that the JWT is expired.
+		claims, err := s.VerifyClaim(authToken)
+		if err != nil {
+			return err
+		}
+		if !claims.IsNode() && !claims.IsNodeIntroduction() {
+			return structs.ErrPermissionDenied
+		}
+		identity.Claims = claims
+	}
+	return nil
+}
+
+// AuthenticateClientOnly returns an ACL object for use *only* with internal
+// RPCs originating from clients (including those forwarded). This should never
+// be used for RPCs that serve HTTP endpoints to avoid confused deputy attacks
+// by making a request to a client that's forwarded. It should also not be used
+// with Node.Register or NodeUpdateStatus, which should use
+// AuthenticateNodeIdentityGenerator.
+func (s *Authenticator) AuthenticateClientOnly(ctx RPCContext, args structs.RequestWithIdentity) (*acl.ACL, error) {
+
+	remoteIP, err := ctx.GetRemoteIP() // capture for metrics
+	if err != nil {
+		s.logger.Error("could not determine remote address", "error", err)
+	}
+
+	identity := &structs.AuthenticatedIdentity{RemoteIP: remoteIP}
+	defer args.SetIdentity(identity) // always set the identity, even on errors
+
+	if err := verifyTLS(s.verifyTLS.Load(), ctx, s.validClientCertNames, identity); err != nil {
+		return nil, err
+	}
+
+	authToken := args.GetAuthToken()
+	if authToken == "" {
+		return nil, structs.ErrPermissionDenied
+	}
+
+	// If the auth token is a UUID, we treat it as a node secret ID. Otherwise,
+	// we assume it's a node identity claim. Anything outside these cases is not
+	// permitted when using this method.
+	if helper.IsUUID(authToken) {
+		node, err := s.getState().NodeBySecretID(nil, authToken)
+		if err != nil {
+			return nil, fmt.Errorf("could not resolve node secret: %w", err)
+		}
+		if node == nil {
+			return nil, structs.ErrPermissionDenied
+		}
+		identity.ClientID = node.ID
+	} else {
+		claims, err := s.VerifyClaim(authToken)
+		if err != nil {
+			return nil, err
+		}
+		if !claims.IsNode() || claims.NodeIdentityClaims == nil {
+			return nil, structs.ErrPermissionDenied
+		}
+		identity.ClientID = claims.NodeIdentityClaims.NodeID
+		identity.Claims = claims
+	}
+
+	return s.ResolveClientIdentityACL(identity)
+}
+
+// verifyTLS is a helper function that performs TLS verification, if required,
+// given the passed RPCContext and valid names.
+//
+// It will always set the TLSName on the identity if we are performing
+// verification, so callers don't have to worry about setting it themselves.
+func verifyTLS(verify bool, ctx RPCContext, validNames []string, identity *structs.AuthenticatedIdentity) error {
+
+	if verify && !ctx.IsStatic() {
+
+		tlsCert := ctx.Certificate()
+		if tlsCert == nil {
+			return errors.New("missing certificate information")
+		}
+
+		// Always set on the identity, even before validating the name, so we
+		// can capture it for metrics.
+		identity.TLSName = tlsCert.Subject.CommonName
+
+		// Perform the certificate validation, using the passed valid names.
+		_, err := validateCertificateForNames(tlsCert, validNames)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// validateCertificateForNames returns true if the certificate is valid for any
+// of the given domain names.
+func validateCertificateForNames(cert *x509.Certificate, expectedNames []string) (bool, error) {
+	if cert == nil {
+		return false, nil
+	}
+
+	validNames := []string{cert.Subject.CommonName}
+	validNames = append(validNames, cert.DNSNames...)
+
+	for _, expectedName := range expectedNames {
+		if slices.Contains(validNames, expectedName) {
+			return true, nil
+		}
+	}
+
+	return false, fmt.Errorf("invalid certificate: %s not in expected %s",
+		strings.Join(validNames, ", "),
+		strings.Join(expectedNames, ", "))
+
+}
+
+// IdentityToACLClaim returns an ACLClaim suitable for checking permissions
+func IdentityToACLClaim(ai *structs.AuthenticatedIdentity, store *state.StateStore) *acl.ACLClaim {
+	if ai == nil || ai.Claims == nil {
+		return nil
+	}
+
+	var group string
+	alloc, err := store.AllocByID(nil, ai.Claims.AllocationID)
+	if err != nil {
+		// we should never hit this error, but if we did the caller would get a
+		// nil claim and auth will fail
+		return nil
+	}
+	if alloc != nil {
+		group = alloc.TaskGroup
+	}
+
+	return &acl.ACLClaim{
+		Namespace: ai.Claims.Namespace,
+		Job:       ai.Claims.JobID,
+		Group:     group,
+		Task:      ai.Claims.TaskName,
+	}
+}
+
+// resolveACLForToken resolves an ACL from a token only. It should be used only
+// by Variables endpoints, which have additional implicit policies for their
+// claims so we can't wrap them up in ResolveACL.
+func (s *Authenticator) resolveACLForToken(aclToken *structs.ACLToken) (*acl.ACL, error) {
+	snap, err := s.getState().Snapshot()
+	if err != nil {
+		return nil, err
+	}
+	return resolveACLFromToken(snap, s.aclCache, aclToken)
+}
+
+// ResolveToken is used to translate an ACL Token Secret ID into
+// an ACL object, nil if ACLs are disabled, or an error.
+//
+// TODO(tgross): this is used in lots of places we probably should be calling
+// Authenticate + ResolveACL on, because they support HTTP APIs that may be used
+// by the Task API
+func (s *Authenticator) ResolveToken(secretID string) (*acl.ACL, error) {
+	// Fast-path if ACLs are disabled
+	if !s.aclsEnabled {
+		return acl.ACLsDisabledACL, nil
+	}
+	defer metrics.MeasureSince([]string{"nomad", "acl", "resolveToken"}, time.Now())
+
+	// Check if the secret ID is the leader secret ID, in which case treat it as
+	// a management token.
+	if leaderAcl := s.getLeaderACL(); leaderAcl != "" && secretID == leaderAcl {
+		return acl.ManagementACL, nil
+	}
+
+	// Snapshot the state
+	snap, err := s.getState().Snapshot()
+	if err != nil {
+		return nil, err
+	}
+
+	// Resolve the ACL
+	return resolveTokenFromSnapshotCache(snap, s.aclCache, secretID)
+}
+
+// VerifyClaim asserts that the token is valid. If it is for a workload
+// identity, it will ensure that the resulting allocation ID belongs to a
+// non-terminal allocation. If the token is for a node identity, it will ensure
+// the node ID matches the claim.
+//
+// This should usually not be called by RPC handlers.
+func (s *Authenticator) VerifyClaim(token string) (*structs.IdentityClaims, error) {
+
+	claims, err := s.encrypter.VerifyClaim(token)
+	if err != nil {
+		return nil, err
+	}
+
+	if claims.IsWorkload() {
+		if err := s.verifyWorkloadIdentityClaim(claims); err != nil {
+			return nil, err
+		}
+		return claims, nil
+	}
+
+	// If the claims are for a node identity, we can return them directly once
+	// we have verified the claim. In the happy path, we could read the node out
+	// of state and verify that it is found. However, it is possible the node
+	// has been garbage collected, and if we failed on that check, the node
+	// would not be able to register again without manual intervention.
+	if claims.IsNode() {
+		return claims, nil
+	}
+
+	// Node introduction claims are a case where we don't verify them against
+	// the state store, since they are used to introduce a node that does not
+	// yet exist.
+	if claims.IsNodeIntroduction() {
+		return claims, nil
+	}
+
+	return nil, errors.New("failed to determine claim type")
+}
+
+// ResolveClientIdentityACL resolves a client ACL from an authenticated node
+// identity or client secret based identity. It returns (nil, nil) if the
+// identity does not represent a client.
+func (s *Authenticator) ResolveClientIdentityACL(identity *structs.AuthenticatedIdentity) (*acl.ACL, error) {
+	if identity == nil || identity.ClientID == "" {
+		return nil, nil
+	}
+	if claims := identity.GetClaims(); claims != nil {
+		if !claims.IsNode() || claims.NodeIdentityClaims == nil || claims.NodeIdentityClaims.NodePool == "" {
+			return nil, structs.ErrPermissionDenied
+		}
+		return acl.NewClientACL(claims.NodeIdentityClaims.NodePool), nil
+	}
+
+	snap, err := s.getState().Snapshot()
+	if err != nil {
+		return nil, structs.ErrPermissionDenied
+	}
+	node, err := snap.NodeByID(nil, identity.ClientID)
+	if err != nil {
+		return nil, err
+	}
+	if node == nil {
+		return nil, structs.ErrPermissionDenied
+	}
+	return acl.NewClientACL(node.NodePool), nil
+}
+
+// AuthenticatedNodeID returns the authenticated node ID for client and node
+// identity based requests.
+func AuthenticatedNodeID(identity *structs.AuthenticatedIdentity) string {
+	if identity == nil {
+		return ""
+	}
+
+	if identity.ClientID != "" {
+		return identity.ClientID
+	}
+
+	claims := identity.GetClaims()
+	if claims != nil && claims.IsNode() && claims.NodeIdentityClaims != nil {
+		return claims.NodeIdentityClaims.NodeID
+	}
+
+	return ""
+}
+
+// AuthorizeSameNode returns ErrPermissionDenied unless the authenticated
+// identity represents the same node as targetNodeID.
+func AuthorizeSameNode(identity *structs.AuthenticatedIdentity, targetNodeID string) error {
+	if AuthenticatedNodeID(identity) != targetNodeID {
+		return structs.ErrPermissionDenied
+	}
+
+	return nil
+}
+
+// AuthorizeSameNodeServiceRegistrations returns ErrPermissionDenied unless the
+// authenticated identity represents the same node as every service
+// registration's NodeID.
+func AuthorizeSameNodeServiceRegistrations(
+	identity *structs.AuthenticatedIdentity,
+	services []*structs.ServiceRegistration,
+) error {
+	for _, service := range services {
+		if service == nil {
+			return structs.ErrPermissionDenied
+		}
+		if err := AuthorizeSameNode(identity, service.NodeID); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// ResolveAuthorizedClientNodePoolByNodeID resolves the node pool for nodeID
+func resolveAuthorizedClientNodePoolByNodeID(snap *state.StateSnapshot, aclObj *acl.ACL, nodeID string) (string, error) {
+	if nodeID == "" {
+		return "", nil
+	}
+
+	pool, ok, err := snap.NodePoolByNodeID(nil, nodeID)
+	if err != nil {
+		return "", err
+	}
+	if !ok {
+		return "", nil
+	}
+	if !aclObj.AllowClientOp(pool) {
+		return "", structs.ErrPermissionDenied
+	}
+
+	return pool, nil
+}
+
+// ResolveAuthorizedClientNodePoolByNodeID resolves the node pool for nodeID
+// and returns it if aclObj is authorized for that pool. It returns an empty
+// pool and nil error if the node does not exist yet, which allows callers to
+// continue handling blocking-query flows that may legitimately observe a
+// missing node.
+func (s *Authenticator) ResolveAuthorizedClientNodePoolByNodeID(aclObj *acl.ACL, nodeID string) (string, error) {
+	snap, err := s.getState().Snapshot()
+	if err != nil {
+		return "", err
+	}
+
+	return resolveAuthorizedClientNodePoolByNodeID(snap, aclObj, nodeID)
+}
+
+// AuthorizeClientAllocation returns ErrPermissionDenied unless aclObj is
+// authorized for alloc's node pool. If allowNsOp is provided, callers may
+// fall back to namespace-based authorization when client-scoped authorization
+// does not apply.
+func (s *Authenticator) AuthorizeClientAllocation(
+	aclObj *acl.ACL,
+	alloc *structs.Allocation,
+	allowNsOp func(*acl.ACL, string) bool,
+) error {
+	if alloc == nil || alloc.Job == nil {
+		return structs.ErrPermissionDenied
+	}
+
+	if aclObj.AllowClientOp(alloc.Job.NodePool) {
+		return nil
+	}
+
+	if allowNsOp != nil && allowNsOp(aclObj, alloc.Namespace) {
+		return nil
+	}
+
+	return structs.ErrPermissionDenied
+}
+
+// ResolveAuthorizedClientNodePoolByServiceRegistrationID resolves the node pool
+// for the node backing a service registration and returns it if aclObj is
+// authorized for that pool.
+func (s *Authenticator) ResolveAuthorizedClientNodePoolByServiceRegistrationID(
+	aclObj *acl.ACL,
+	namespace, id string,
+) (string, error) {
+	if aclObj == nil || !(aclObj.AllowAgentRead() || aclObj.AllowServerOp() || aclObj.AllowNodeRead()) {
+		return "", structs.ErrPermissionDenied
+	}
+
+	snap, err := s.getState().Snapshot()
+	if err != nil {
+		return "", err
+	}
+
+	registration, err := snap.GetServiceRegistrationByID(nil, namespace, id)
+	if err != nil {
+		return "", err
+	}
+	if registration == nil {
+		return "", structs.ErrPermissionDenied
+	}
+
+	return resolveAuthorizedClientNodePoolByNodeID(snap, aclObj, registration.NodeID)
+}
+
+func (s *Authenticator) verifyWorkloadIdentityClaim(claims *structs.IdentityClaims) error {
+	snap, err := s.getState().Snapshot()
+	if err != nil {
+		return err
+	}
+	alloc, err := snap.AllocByID(nil, claims.AllocationID)
+	if err != nil {
+		return err
+	}
+	if alloc == nil || alloc.Job == nil {
+		return fmt.Errorf("allocation does not exist")
+	}
+
+	// the claims for terminal allocs are always treated as expired
+	if alloc.ClientTerminalStatus() {
+		return fmt.Errorf("allocation is terminal")
+	}
+
+	return nil
+}
+
+func (s *Authenticator) resolveClaims(claims *structs.IdentityClaims) (*acl.ACL, error) {
+
+	// Nomad node identity claims currently map to a client ACL. If we open this
+	// up in the future, we will want to modify this section to perform similar
+	// work that is done for workload claims.
+	if claims.IsNode() {
+		if claims.NodeIdentityClaims == nil || claims.NodeIdentityClaims.NodePool == "" {
+			return nil, fmt.Errorf("node identity claims missing node pool")
+		}
+		return acl.NewClientACL(claims.NodeIdentityClaims.NodePool), nil
+	}
+
+	policies, err := s.ResolvePoliciesForClaims(claims)
+	if err != nil {
+		return nil, err
+	}
+
+	// Compile and cache the ACL object. For many claims this will result in an
+	// ACL object with no policies, which can be efficiently cached.
+	aclObj, err := structs.CompileACLObject(s.aclCache, policies)
+	if err != nil {
+		return nil, err
+	}
+	return aclObj, nil
+}
+
+// resolveTokenFromSnapshotCache is used to resolve an ACL object from a
+// snapshot of state, using a cache to avoid parsing and ACL construction when
+// possible. It is split from resolveToken to simplify testing.
+func resolveTokenFromSnapshotCache(snap *state.StateSnapshot, cache *structs.ACLCache[*acl.ACL], secretID string) (*acl.ACL, error) {
+	// Lookup the ACL Token
+	var token *structs.ACLToken
+	var err error
+
+	// Handle anonymous requests
+	if secretID == "" {
+		token = structs.AnonymousACLToken
+	} else {
+		token, err = snap.ACLTokenBySecretID(nil, secretID)
+		if err != nil {
+			return nil, err
+		}
+		if token == nil {
+			return nil, structs.ErrTokenNotFound
+		}
+		if token.IsExpired(time.Now().UTC()) {
+			return nil, structs.ErrTokenExpired
+		}
+	}
+
+	return resolveACLFromToken(snap, cache, token)
+
+}
+
+func resolveACLFromToken(snap *state.StateSnapshot, cache *structs.ACLCache[*acl.ACL], token *structs.ACLToken) (*acl.ACL, error) {
+
+	// Check if this is a management token
+	if token.Type == structs.ACLManagementToken {
+		return acl.ManagementACL, nil
+	}
+
+	// Store all policies detailed in the token request, this includes the
+	// named policies and those referenced within the role link.
+	policies := make([]*structs.ACLPolicy, 0, len(token.Policies)+len(token.Roles))
+
+	// Iterate all the token policies and add these to our policy tracking
+	// array.
+	for _, policyName := range token.Policies {
+		policy, err := snap.ACLPolicyByName(nil, policyName)
+		if err != nil {
+			return nil, err
+		}
+		if policy == nil {
+			// Ignore policies that don't exist, since they don't grant any
+			// more privilege.
+			continue
+		}
+
+		// Add the policy to the tracking array.
+		policies = append(policies, policy)
+	}
+
+	// Iterate all the token role links, so we can unpack these and identify
+	// the ACL policies.
+	for _, roleLink := range token.Roles {
+
+		// Any error reading the role means we cannot move forward. We just
+		// ignore any roles that have been detailed but are not within our
+		// state.
+		role, err := snap.GetACLRoleByID(nil, roleLink.ID)
+		if err != nil {
+			return nil, err
+		}
+		if role == nil {
+			continue
+		}
+
+		// Unpack the policies held within the ACL role to form a single list
+		// of ACL policies that this token has available.
+		for _, policyLink := range role.Policies {
+			policy, err := snap.ACLPolicyByName(nil, policyLink.Name)
+			if err != nil {
+				return nil, err
+			}
+
+			// Ignore policies that don't exist, since they don't grant any
+			// more privilege.
+			if policy == nil {
+				continue
+			}
+
+			// Add the policy to the tracking array.
+			policies = append(policies, policy)
+		}
+	}
+
+	// Compile and cache the ACL object
+	aclObj, err := structs.CompileACLObject(cache, policies)
+	if err != nil {
+		return nil, err
+	}
+	return aclObj, nil
+}
+
+// resolveSecretToken is used to translate an ACL Token Secret ID into a
+// Accessor ID, the anonymous accessor, or an error.
+func (s *Authenticator) resolveSecretToken(secretID string) (*structs.ACLToken, error) {
+
+	defer metrics.MeasureSince([]string{"nomad", "acl", "accessorForSecretToken"}, time.Now())
+
+	if !s.aclsEnabled {
+		return structs.ACLsDisabledToken, nil
+	}
+	if secretID == "" {
+		return structs.AnonymousACLToken, nil
+	}
+	if !helper.IsUUID(secretID) {
+		return nil, structs.ErrTokenInvalid
+	}
+
+	snap, err := s.getState().Snapshot()
+	if err != nil {
+		return nil, err
+	}
+
+	// Lookup the ACL Token
+	token, err := snap.ACLTokenBySecretID(nil, secretID)
+	if err != nil {
+		return nil, err
+	}
+	if token == nil {
+		return nil, structs.ErrTokenNotFound
+	}
+	if token.IsExpired(time.Now().UTC()) {
+		return nil, structs.ErrTokenExpired
+	}
+
+	return token, nil
+}
+
+func (s *Authenticator) ResolvePoliciesForClaims(claims *structs.IdentityClaims) ([]*structs.ACLPolicy, error) {
+
+	snap, err := s.getState().Snapshot()
+	if err != nil {
+		return nil, err
+	}
+	alloc, err := snap.AllocByID(nil, claims.AllocationID)
+	if err != nil {
+		return nil, err
+	}
+	if alloc == nil || alloc.Job == nil {
+		return nil, fmt.Errorf("allocation does not exist")
+	}
+
+	// Find any policies attached to the job
+	jobId := alloc.Job.GetIDforWorkloadIdentity()
+	iter, err := snap.ACLPolicyByJob(nil, alloc.Namespace, jobId)
+	if err != nil {
+		return nil, err
+	}
+	policies := []*structs.ACLPolicy{}
+	for {
+		raw := iter.Next()
+		if raw == nil {
+			break
+		}
+		policy := raw.(*structs.ACLPolicy)
+		if policy.JobACL == nil {
+			continue
+		}
+
+		switch {
+		case policy.JobACL.Group == "":
+			policies = append(policies, policy)
+		case policy.JobACL.Group != alloc.TaskGroup:
+			continue // don't bother checking task
+		case policy.JobACL.Task == "":
+			policies = append(policies, policy)
+		case policy.JobACL.Task == claims.TaskName:
+			policies = append(policies, policy)
+		}
+	}
+
+	iter, err = snap.ACLPolicyByNamespace(nil, alloc.Namespace)
+	if err != nil {
+		return nil, err
+	}
+	for {
+		raw := iter.Next()
+		if raw == nil {
+			break
+		}
+
+		policy := raw.(*structs.ACLPolicy)
+		if policy.JobACL == nil {
+			continue
+		}
+
+		policies = append(policies, policy)
+	}
+
+	return policies, nil
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/autopilot.go

@@ -0,0 +1,234 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"context"
+	"fmt"
+
+	metrics "github.com/hashicorp/go-metrics/compat"
+	"github.com/hashicorp/nomad/helper"
+	"github.com/hashicorp/nomad/nomad/peers"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/raft"
+	autopilot "github.com/hashicorp/raft-autopilot"
+	"github.com/hashicorp/serf/serf"
+)
+
+const (
+	// AutopilotRZTag is the Serf tag to use for the redundancy zone value
+	// when passing the server metadata to Autopilot.
+	AutopilotRZTag = "ap_zone"
+
+	// AutopilotRZTag is the Serf tag to use for the custom version value
+	// when passing the server metadata to Autopilot.
+	AutopilotVersionTag = "ap_version"
+)
+
+// AutopilotDelegate is a Nomad delegate for autopilot operations. It implements
+// the autopilot.ApplicationIntegration interface, and the methods required for
+// that interface have been documented as such below.
+type AutopilotDelegate struct {
+	server *Server
+}
+
+// AutopilotConfig is used to retrieve the latest configuration from the Nomad
+// delegate. This method is required to implement the ApplicationIntegration
+// interface.
+func (d *AutopilotDelegate) AutopilotConfig() *autopilot.Config {
+	c := d.server.getOrCreateAutopilotConfig()
+	if c == nil {
+		return nil
+	}
+	conf := &autopilot.Config{
+		CleanupDeadServers:      c.CleanupDeadServers,
+		LastContactThreshold:    c.LastContactThreshold,
+		MaxTrailingLogs:         c.MaxTrailingLogs,
+		MinQuorum:               c.MinQuorum,
+		ServerStabilizationTime: c.ServerStabilizationTime,
+		Ext:                     autopilotConfigExt(c),
+	}
+
+	return conf
+}
+
+// FetchServerStats will be called by autopilot to request Nomad fetch the
+// server stats out of band. This method is required to implement the
+// ApplicationIntegration interface
+func (d *AutopilotDelegate) FetchServerStats(ctx context.Context, servers map[raft.ServerID]*autopilot.Server) map[raft.ServerID]*autopilot.ServerStats {
+	return d.server.statsFetcher.Fetch(ctx, servers)
+}
+
+// KnownServers will be called by autopilot to request the list of servers known
+// to Nomad. This method is required to implement the ApplicationIntegration
+// interface
+func (d *AutopilotDelegate) KnownServers() map[raft.ServerID]*autopilot.Server {
+	return d.server.autopilotServers()
+}
+
+// NotifyState will be called when the autopilot state is updated. The Nomad
+// leader heartbeats a metric for monitoring based on this information. This
+// method is required to implement the ApplicationIntegration interface
+func (d *AutopilotDelegate) NotifyState(state *autopilot.State) {
+	if d.server.raft.State() == raft.Leader {
+		metrics.SetGauge([]string{"nomad", "autopilot", "failure_tolerance"}, float32(state.FailureTolerance))
+		if state.Healthy {
+			metrics.SetGauge([]string{"nomad", "autopilot", "healthy"}, 1)
+		} else {
+			metrics.SetGauge([]string{"nomad", "autopilot", "healthy"}, 0)
+		}
+	}
+}
+
+// RemoveFailedServer will be called by autopilot to notify Nomad to remove the
+// server in a failed state. This method is required to implement the
+// ApplicationIntegration interface. (Note this is expected to return
+// immediately so we'll spawn a goroutine for it.)
+func (d *AutopilotDelegate) RemoveFailedServer(failedSrv *autopilot.Server) {
+	go func() {
+		err := d.server.RemoveFailedNode(failedSrv.Name)
+		if err != nil {
+			d.server.logger.Error("could not remove failed server",
+				"server", string(failedSrv.ID),
+				"error", err,
+			)
+		}
+	}()
+}
+
+// MinRaftProtocol returns the lowest supported Raft protocol among alive
+// servers
+func (s *Server) MinRaftProtocol() (int, error) {
+	return minRaftProtocol(s.peersCache.RegionPeers(s.Region()))
+}
+
+// GetClusterHealth is used to get the current health of the servers, as known
+// by the leader.
+func (s *Server) GetClusterHealth() *structs.OperatorHealthReply {
+
+	state := s.autopilot.GetState()
+	if state == nil {
+		// this behavior seems odd but its functionally equivalent to 1.8.5 where if
+		// autopilot didn't have a health reply yet it would just return no error
+		return nil
+	}
+
+	health := &structs.OperatorHealthReply{
+		Healthy:          state.Healthy,
+		FailureTolerance: state.FailureTolerance,
+		Leader:           string(state.Leader),
+		Voters:           stringIDs(state.Voters),
+		Servers:          make([]structs.ServerHealth, 0, len(state.Servers)),
+	}
+
+	for _, srv := range state.Servers {
+		srvHealth := autopilotToServerHealth(srv)
+
+		health.Servers = append(health.Servers, srvHealth)
+	}
+	err := s.autopilotStateExt(state, health)
+	if err != nil {
+		s.logger.Error("Error parsing autopilot state", "error", err)
+	}
+
+	return health
+}
+
+// -------------------
+// helper functions
+
+func autopilotToServerHealth(srv *autopilot.ServerState) structs.ServerHealth {
+	srvHealth := structs.ServerHealth{
+		ID:          string(srv.Server.ID),
+		Name:        srv.Server.Name,
+		Address:     string(srv.Server.Address),
+		Version:     srv.Server.Version,
+		Leader:      srv.State == autopilot.RaftLeader,
+		Voter:       srv.State == autopilot.RaftLeader || srv.State == autopilot.RaftVoter,
+		LastContact: srv.Stats.LastContact,
+		LastTerm:    srv.Stats.LastTerm,
+		LastIndex:   srv.Stats.LastIndex,
+		Healthy:     srv.Health.Healthy,
+		StableSince: srv.Health.StableSince,
+	}
+
+	switch srv.Server.NodeStatus {
+	case autopilot.NodeAlive:
+		srvHealth.SerfStatus = serf.StatusAlive
+	case autopilot.NodeLeft:
+		srvHealth.SerfStatus = serf.StatusLeft
+	case autopilot.NodeFailed:
+		srvHealth.SerfStatus = serf.StatusFailed
+	default:
+		srvHealth.SerfStatus = serf.StatusNone
+	}
+
+	return srvHealth
+}
+
+func stringIDs(ids []raft.ServerID) []string {
+	return helper.ConvertSlice(ids, func(id raft.ServerID) string { return string(id) })
+}
+
+func minRaftProtocol(members []*peers.Parts) (int, error) {
+	minVersion := -1
+	for _, m := range members {
+		if m.Status != serf.StatusAlive {
+			continue
+		}
+
+		if minVersion == -1 || m.RaftVersion < minVersion {
+			minVersion = m.RaftVersion
+		}
+	}
+
+	if minVersion == -1 {
+		return minVersion, fmt.Errorf("No servers found")
+	}
+
+	return minVersion, nil
+}
+
+func (s *Server) autopilotServers() map[raft.ServerID]*autopilot.Server {
+
+	// Get a list of the regional peers for our local region. We cannot use the
+	// LocalPeers function, as we need peers in all states.
+	localPeers := s.peersCache.RegionPeers(s.Region())
+
+	servers := make(map[raft.ServerID]*autopilot.Server, len(localPeers))
+
+	for _, srv := range localPeers {
+		autopilotServer := s.autopilotServerFromMetadata(srv)
+		servers[autopilotServer.ID] = autopilotServer
+	}
+
+	return servers
+}
+
+func (s *Server) autopilotServerFromMetadata(srv *peers.Parts) *autopilot.Server {
+	server := &autopilot.Server{
+		Name:        srv.Name,
+		ID:          raft.ServerID(srv.ID),
+		Address:     raft.ServerAddress(srv.Addr.String()),
+		Version:     srv.Build.String(),
+		RaftVersion: srv.RaftVersion,
+		Ext:         s.autopilotServerExt(srv),
+		Meta:        srv.Tags,
+	}
+
+	switch srv.Status {
+	case serf.StatusLeft:
+		server.NodeStatus = autopilot.NodeLeft
+	case serf.StatusAlive, serf.StatusLeaving:
+		// we want to treat leaving as alive to prevent autopilot from
+		// prematurely removing the node.
+		server.NodeStatus = autopilot.NodeAlive
+	case serf.StatusFailed:
+		server.NodeStatus = autopilot.NodeFailed
+	default:
+		server.NodeStatus = autopilot.NodeUnknown
+	}
+
+	return server
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/autopilot_ce.go

@@ -0,0 +1,34 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+//go:build !ent
+// +build !ent
+
+package nomad
+
+import (
+	autopilot "github.com/hashicorp/raft-autopilot"
+
+	"github.com/hashicorp/nomad/nomad/peers"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+func (s *Server) autopilotPromoter() autopilot.Promoter {
+	return autopilot.DefaultPromoter()
+}
+
+// autopilotServerExt returns the autopilot-enterprise.Server extensions needed
+// for ENT feature support, but this is the empty OSS implementation.
+func (s *Server) autopilotServerExt(_ *peers.Parts) interface{} {
+	return nil
+}
+
+func (s *Server) autopilotStateExt(_ *autopilot.State, _ *structs.OperatorHealthReply) error {
+	return nil
+}
+
+// autopilotConfigExt returns the autopilot-enterprise.Config extensions needed
+// for ENT feature support, but this is the empty OSS implementation.
+func autopilotConfigExt(_ *structs.AutopilotConfig) interface{} {
+	return nil
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/autopilot_test.go

@@ -0,0 +1,348 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/raft"
+	autopilot "github.com/hashicorp/raft-autopilot"
+	"github.com/hashicorp/serf/serf"
+	"github.com/shoenig/test/must"
+
+	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/testutil"
+)
+
+var _ autopilot.ApplicationIntegration = (*AutopilotDelegate)(nil)
+
+// wantPeers determines whether the server has the given
+// number of voting raft peers.
+func wantPeers(s *Server, peers int) error {
+	future := s.raft.GetConfiguration()
+	if err := future.Error(); err != nil {
+		return err
+	}
+
+	var n int
+	for _, server := range future.Configuration().Servers {
+		if server.Suffrage == raft.Voter {
+			n++
+		}
+	}
+
+	if got, want := n, peers; got != want {
+		return fmt.Errorf("server %v: got %d peers want %d\n\tservers: %#+v", s.config.NodeName, got, want, future.Configuration().Servers)
+	}
+	return nil
+}
+
+func TestAutopilot_CleanupDeadServer(t *testing.T) {
+	ci.Parallel(t)
+
+	conf := func(c *Config) {
+		c.NumSchedulers = 0 // reduces test log noise
+		c.BootstrapExpect = 3
+		c.RaftConfig.ProtocolVersion = raft.ProtocolVersion(3)
+	}
+
+	s1, cleanupS1 := TestServer(t, conf)
+	defer cleanupS1()
+
+	s2, cleanupS2 := TestServer(t, conf)
+	defer cleanupS2()
+
+	s3, cleanupS3 := TestServer(t, conf)
+	defer cleanupS3()
+
+	servers := []*Server{s1, s2, s3}
+	TestJoin(t, servers...)
+
+	t.Logf("waiting for initial stable cluster")
+	waitForStableLeadership(t, servers)
+
+	s4, cleanupS4 := TestServer(t, conf)
+	defer cleanupS4()
+
+	// Kill a non-leader server
+	killedIdx := 0
+	for i, s := range servers {
+		if !s.IsLeader() {
+			killedIdx = i
+			t.Logf("killing a server (index %d)", killedIdx)
+			s.Shutdown()
+			break
+		}
+	}
+
+	t.Logf("waiting for server loss to be detected")
+	testutil.WaitForResultUntil(10*time.Second, func() (bool, error) {
+		for i, s := range servers {
+			alive := 0
+			if i == killedIdx {
+				// Skip shutdown server
+				continue
+			}
+			for _, m := range s.Members() {
+				if m.Status == serf.StatusAlive {
+					alive++
+				}
+			}
+
+			if alive != 2 {
+				return false, fmt.Errorf("expected 2 alive servers but found %v", alive)
+			}
+		}
+		return true, nil
+	}, func(err error) { must.NoError(t, err) })
+
+	// Join the new server
+	servers[killedIdx] = s4
+	t.Logf("adding server s4")
+	TestJoin(t, servers...)
+
+	t.Logf("waiting for dead server to be removed")
+	waitForStableLeadership(t, servers)
+}
+
+func TestAutopilot_CleanupDeadServerPeriodic(t *testing.T) {
+	ci.Parallel(t)
+
+	conf := func(c *Config) {
+		c.NumSchedulers = 0 // reduces test log noise
+		c.BootstrapExpect = 5
+	}
+
+	s1, cleanupS1 := TestServer(t, conf)
+	defer cleanupS1()
+
+	s2, cleanupS2 := TestServer(t, conf)
+	defer cleanupS2()
+
+	s3, cleanupS3 := TestServer(t, conf)
+	defer cleanupS3()
+
+	s4, cleanupS4 := TestServer(t, conf)
+	defer cleanupS4()
+
+	s5, cleanupS5 := TestServer(t, conf)
+	defer cleanupS5()
+
+	servers := []*Server{s1, s2, s3, s4, s5}
+	TestJoin(t, servers...)
+
+	t.Logf("waiting for initial stable cluster")
+	waitForStableLeadership(t, servers)
+
+	t.Logf("killing a non-leader server")
+	if leader := waitForStableLeadership(t, servers); leader == s4 {
+		s1, s4 = s4, s1
+	}
+	s4.Shutdown()
+
+	t.Logf("waiting for dead peer to be removed")
+	servers = []*Server{s1, s2, s3, s5}
+	waitForStableLeadership(t, servers)
+}
+
+func TestAutopilot_RollingUpdate(t *testing.T) {
+	ci.Parallel(t)
+
+	conf := func(c *Config) {
+		c.NumSchedulers = 0 // reduces test log noise
+		c.BootstrapExpect = 3
+		c.RaftConfig.ProtocolVersion = 3
+	}
+
+	s1, cleanupS1 := TestServer(t, conf)
+	defer cleanupS1()
+
+	s2, cleanupS2 := TestServer(t, conf)
+	defer cleanupS2()
+
+	s3, cleanupS3 := TestServer(t, conf)
+	defer cleanupS3()
+
+	servers := []*Server{s1, s2, s3}
+	TestJoin(t, s1, s2, s3)
+
+	t.Logf("waiting for initial stable cluster")
+	waitForStableLeadership(t, servers)
+
+	// Add one more server like we are doing a rolling update.
+	t.Logf("adding server s4")
+	s4, cleanupS4 := TestServer(t, conf)
+	defer cleanupS4()
+	TestJoin(t, s1, s4)
+
+	// Wait for s4 to stabilize and get promoted to a voter
+	t.Logf("waiting for s4 to stabilize and be promoted")
+	servers = append(servers, s4)
+	waitForStableLeadership(t, servers)
+
+	// Now kill one of the "old" nodes like we are doing a rolling update.
+	t.Logf("shutting down server s3")
+	s3.Shutdown()
+
+	// Wait for s3 to be removed and the cluster to stablize.
+	t.Logf("waiting for cluster to stabilize")
+	servers = []*Server{s1, s2, s4}
+	waitForStableLeadership(t, servers)
+}
+
+func TestAutopilot_MultiRegion(t *testing.T) {
+	ci.Parallel(t)
+
+	conf := func(c *Config) {
+		c.NumSchedulers = 0 // reduces test log noise
+		c.BootstrapExpect = 3
+	}
+	s1, cleanupS1 := TestServer(t, conf)
+	defer cleanupS1()
+
+	s2, cleanupS2 := TestServer(t, conf)
+	defer cleanupS2()
+
+	s3, cleanupS3 := TestServer(t, conf)
+	defer cleanupS3()
+
+	// federated regions should not be considered raft peers or show up in the
+	// known servers list
+	s4, cleanupS4 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 0
+		c.Region = "other"
+	})
+	defer cleanupS4()
+
+	servers := []*Server{s1, s2, s3}
+	TestJoin(t, s1, s2, s3, s4)
+
+	t.Logf("waiting for initial stable cluster")
+	waitForStableLeadership(t, servers)
+
+	apDelegate := &AutopilotDelegate{s3}
+	known := apDelegate.KnownServers()
+	must.Eq(t, 3, len(known))
+
+}
+
+func TestAutopilot_CleanupStaleRaftServer(t *testing.T) {
+	ci.Parallel(t)
+
+	conf := func(c *Config) {
+		c.NumSchedulers = 0 // reduces test log noise
+		c.BootstrapExpect = 3
+	}
+	s1, cleanupS1 := TestServer(t, conf)
+	defer cleanupS1()
+
+	s2, cleanupS2 := TestServer(t, conf)
+	defer cleanupS2()
+
+	s3, cleanupS3 := TestServer(t, conf)
+	defer cleanupS3()
+
+	s4, cleanupS4 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 0
+	})
+	defer cleanupS4()
+
+	servers := []*Server{s1, s2, s3}
+	TestJoin(t, s1, s2, s3)
+
+	t.Logf("waiting for initial stable cluster")
+	leader := waitForStableLeadership(t, servers)
+
+	t.Logf("adding server s4 to peers directly")
+	addr := fmt.Sprintf("127.0.0.1:%d", s4.config.RPCAddr.Port)
+	future := leader.raft.AddVoter(raft.ServerID(s4.config.NodeID), raft.ServerAddress(addr), 0, 0)
+	if err := future.Error(); err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("waiting for 4th server to be removed")
+	waitForStableLeadership(t, servers)
+}
+
+func TestAutopilot_PromoteNonVoter(t *testing.T) {
+	ci.Parallel(t)
+
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.NumSchedulers = 0 // reduces test log noise
+		c.RaftConfig.ProtocolVersion = 3
+	})
+	defer cleanupS1()
+	codec := rpcClient(t, s1)
+	defer codec.Close()
+	testutil.WaitForLeader(t, s1.RPC)
+
+	s2, cleanupS2 := TestServer(t, func(c *Config) {
+		c.NumSchedulers = 0 // reduces test log noise
+		c.BootstrapExpect = 0
+		c.RaftConfig.ProtocolVersion = 3
+	})
+	defer cleanupS2()
+	TestJoin(t, s1, s2)
+
+	// Note: we can't reliably detect that the server is initially a non-voter,
+	// because it can transition too quickly for the test setup to detect,
+	// especially in low-resource environments like CI. We'll assume that
+	// happens correctly here and only test that it transitions to become a
+	// voter.
+	testutil.WaitForResultUntil(10*time.Second, func() (bool, error) {
+		future := s1.raft.GetConfiguration()
+		if err := future.Error(); err != nil {
+			return false, err
+		}
+		servers := future.Configuration().Servers
+		if len(servers) != 2 {
+			return false, fmt.Errorf("expected 2 servers, got: %v", servers)
+		}
+		if servers[1].Suffrage != raft.Voter {
+			return false, fmt.Errorf("expected server to be voter: %v", servers)
+		}
+		return true, nil
+	}, func(err error) { must.NoError(t, err) })
+}
+
+func TestAutopilot_ReturnAutopilotHealth(t *testing.T) {
+	ci.Parallel(t)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+		c.RaftConfig.ProtocolVersion = 3
+		c.AutopilotConfig.EnableCustomUpgrades = true
+		c.UpgradeVersion = "0.0.1"
+		c.NumSchedulers = 0 // reduce log noise
+	})
+	defer cleanupS1()
+
+	s2, cleanupS2 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+		c.RaftConfig.ProtocolVersion = 3
+		c.AutopilotConfig.EnableCustomUpgrades = true
+		c.UpgradeVersion = "0.0.1"
+		c.NumSchedulers = 0 // reduce log noise
+	})
+	defer cleanupS2()
+
+	TestJoin(t, s1, s2)
+	servers := []*Server{s1, s2}
+	leader := waitForStableLeadership(t, servers)
+
+	get := &structs.GenericRequest{
+		QueryOptions: structs.QueryOptions{
+			Region: "global",
+		},
+	}
+	reply := &structs.OperatorHealthReply{}
+	err := s1.RPC("Operator.ServerHealth", get, reply)
+	must.NoError(t, err)
+
+	must.Eq(t, reply.Healthy, true)
+	_, leaderID := leader.raft.LeaderWithID()
+	must.Eq(t, reply.Leader, string(leaderID))
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/blocked_evals.go

@@ -0,0 +1,874 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"sync"
+	"time"
+
+	"github.com/hashicorp/go-hclog"
+	metrics "github.com/hashicorp/go-metrics/compat"
+	"github.com/hashicorp/nomad/helper"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+const (
+	// unblockBuffer is the buffer size for the unblock channel. The buffer
+	// should be large to ensure that the FSM doesn't block when calling Unblock
+	// as this would apply back-pressure on Raft.
+	unblockBuffer = 8096
+
+	// pruneInterval is the interval at which we prune objects from the
+	// BlockedEvals tracker
+	pruneInterval = 5 * time.Minute
+
+	// pruneThreshold is the threshold after which objects will be pruned.
+	pruneThreshold = 15 * time.Minute
+)
+
+// BlockedEvals is used to track evaluations that shouldn't be queued until a
+// certain class of nodes becomes available. An evaluation is put into the
+// blocked state when it is run through the scheduler and produced failed
+// allocations. It is unblocked when the capacity of a node that could run the
+// failed allocation becomes available.
+type BlockedEvals struct {
+	// logger is the logger to use by the blocked eval tracker.
+	logger hclog.Logger
+
+	evalBroker *EvalBroker
+	enabled    bool
+
+	// the lock on the enabled flag and the control channels like the stopCh and
+	// duplicateCh. This is read by most top-level methods including those
+	// called from the FSM, but write-locked only in Flush called during
+	// leadership changes
+	flushLock sync.RWMutex
+
+	stats *BlockedStats
+
+	// l is the lock on the data we're tracking
+	l sync.RWMutex
+
+	// captured is the set of evaluations that are captured by computed node
+	// classes.
+	captured map[string]wrappedEval
+
+	// escaped is the set of evaluations that have escaped computed node
+	// classes.
+	escaped map[string]wrappedEval
+
+	// system is the set of system evaluations that failed to start on nodes because of
+	// resource constraints.
+	system *systemEvals
+
+	// capacityChangeCh is used to buffer unblocking of evaluations.
+	capacityChangeCh chan *capacityUpdate
+
+	// jobs is the map of blocked job and is used to ensure that only one
+	// blocked eval exists for each job. The value is the blocked evaluation ID.
+	jobs map[structs.NamespacedID]string
+
+	// unblockIndexes maps computed node classes or quota name to the index and
+	// time at which they were unblocked. This is used to check if an
+	// evaluation could have been unblocked between the time they were in the
+	// scheduler and the time they are being blocked.
+	unblockIndexes map[string]unblockEvent
+
+	// unblockIndexesLock protects unblockIndexes, which has its own lock
+	// because we want to take a write lock in the Unblock* methods called from
+	// the FSM
+	unblockIndexesLock sync.RWMutex
+
+	// duplicates is the set of evaluations for jobs that had pre-existing
+	// blocked evaluations. These should be marked as cancelled since only one
+	// blocked eval is needed per job.
+	duplicates []*structs.Evaluation
+
+	// duplicateCh is used to signal that a duplicate eval was added to the
+	// duplicate set. It can be used to unblock waiting callers looking for
+	// duplicates.
+	duplicateCh chan struct{}
+
+	// stopCh is used to stop any created goroutines.
+	stopCh chan struct{}
+}
+
+// unblockEvent keeps a record of the index and time of the unblock
+type unblockEvent struct {
+	index     uint64
+	timestamp time.Time
+}
+
+// capacityUpdate stores unblock data.
+type capacityUpdate struct {
+	computedClass string
+	quotaChange   string
+	nodeID        string
+
+	blockedEval  *structs.Evaluation
+	blockToken   string
+	untrackJobID structs.NamespacedID
+
+	// future will be closed when the operation is done, which allows callers to
+	// block until then if they need to (this is mostly used for tests). The
+	// callers in the FSM should never wait on this future
+	future chan struct{}
+}
+
+// wrappedEval captures both the evaluation and the optional token
+type wrappedEval struct {
+	eval  *structs.Evaluation
+	token string
+}
+
+// NewBlockedEvals creates a new blocked eval tracker that will enqueue
+// unblocked evals into the passed broker.
+func NewBlockedEvals(evalBroker *EvalBroker, logger hclog.Logger) *BlockedEvals {
+	return &BlockedEvals{
+		logger:           logger.Named("blocked_evals"),
+		evalBroker:       evalBroker,
+		captured:         make(map[string]wrappedEval),
+		escaped:          make(map[string]wrappedEval),
+		system:           newSystemEvals(),
+		jobs:             make(map[structs.NamespacedID]string),
+		unblockIndexes:   make(map[string]unblockEvent),
+		capacityChangeCh: make(chan *capacityUpdate, unblockBuffer),
+		duplicateCh:      make(chan struct{}, 1),
+		stopCh:           make(chan struct{}),
+		stats:            NewBlockedStats(),
+	}
+}
+
+// Enabled is used to check if the broker is enabled.
+func (b *BlockedEvals) Enabled() bool {
+	b.flushLock.RLock()
+	defer b.flushLock.RUnlock()
+	return b.enabled
+}
+
+// SetEnabled is used to control if the blocked eval tracker is enabled. The
+// tracker should only be enabled on the active leader.
+func (b *BlockedEvals) SetEnabled(enabled bool) {
+	b.flushLock.Lock()
+	b.l.Lock()
+	if b.enabled == enabled {
+		// No-op
+		b.l.Unlock()
+		b.flushLock.Unlock()
+		return
+	} else if enabled {
+		go b.watchCapacity(b.stopCh, b.capacityChangeCh)
+		go b.prune(b.stopCh)
+	} else {
+		close(b.stopCh)
+	}
+	b.enabled = enabled
+	b.l.Unlock()
+	b.flushLock.Unlock()
+	if !enabled {
+		b.Flush()
+	}
+}
+
+// Block tracks the passed evaluation and enqueues it into the eval broker when
+// a suitable node calls unblock.
+func (b *BlockedEvals) Block(eval *structs.Evaluation) chan struct{} {
+	fut := make(chan struct{})
+
+	b.flushLock.RLock()
+	defer b.flushLock.RUnlock()
+	if !b.enabled {
+		close(fut)
+		return fut
+	}
+
+	// Capture chan in flushlock as Flush overwrites it
+	ch := b.capacityChangeCh
+	done := b.stopCh
+
+	select {
+	case <-done:
+	case ch <- &capacityUpdate{blockedEval: eval, future: fut}:
+	}
+
+	return fut
+}
+
+// Reblock tracks the passed evaluation and enqueues it into the eval broker when
+// a suitable node calls unblock. Reblock should be used over Block when the
+// blocking is occurring by an outstanding evaluation. The token is the
+// evaluation's token.
+func (b *BlockedEvals) Reblock(eval *structs.Evaluation, token string) chan struct{} {
+	fut := make(chan struct{})
+
+	b.flushLock.RLock()
+	defer b.flushLock.RUnlock()
+	if !b.enabled {
+		close(fut)
+		return fut
+	}
+
+	// Capture chan in flushlock as Flush overwrites it
+	ch := b.capacityChangeCh
+	done := b.stopCh
+
+	select {
+	case <-done:
+	case ch <- &capacityUpdate{blockedEval: eval, blockToken: token, future: fut}:
+	}
+
+	return fut
+}
+
+// processBlock is the implementation of blocking an evaluation. It supports
+// taking an optional evaluation token to use when reblocking an evaluation that
+// may be outstanding.
+func (b *BlockedEvals) processBlock(eval *structs.Evaluation, token string) {
+	b.flushLock.RLock()
+	defer b.flushLock.RUnlock()
+	if !b.enabled {
+		return
+	}
+
+	b.l.Lock()
+	defer b.l.Unlock()
+
+	// Handle the new evaluation being for a job we are already tracking.
+	if b.processBlockJobDuplicate(eval) {
+		// If process block job duplicate returns true, the new evaluation has
+		// been marked as a duplicate and we have nothing to do, so return
+		// early.
+		return
+	}
+
+	// Check if the eval missed an unblock while it was in the scheduler at an
+	// older index. The scheduler could have been invoked with a snapshot of
+	// state that was prior to additional capacity being added or allocations
+	// becoming terminal.
+	if b.missedUnblock(eval) {
+		// Just re-enqueue the eval immediately. We pass the token so that the
+		// eval_broker can properly handle the case in which the evaluation is
+		// still outstanding.
+		b.evalBroker.EnqueueAll(map[*structs.Evaluation]string{eval: token})
+		return
+	}
+
+	// Mark the job as tracked.
+	b.jobs[structs.NewNamespacedID(eval.JobID, eval.Namespace)] = eval.ID
+	b.stats.Block(eval)
+
+	// Wrap the evaluation, capturing its token.
+	wrapped := wrappedEval{
+		eval:  eval,
+		token: token,
+	}
+
+	// If the eval has escaped, meaning computed node classes could not capture
+	// the constraints of the job, we store the eval separately as we have to
+	// unblock it whenever node capacity changes. This is because we don't know
+	// what node class is feasible for the jobs constraints.
+	if eval.EscapedComputedClass {
+		b.escaped[eval.ID] = wrapped
+		return
+	}
+
+	// System evals are indexed by node and re-processed on utilization changes in
+	// existing nodes
+	if eval.Type == structs.JobTypeSystem {
+		b.system.Add(eval, token)
+	}
+
+	// Add the eval to the set of blocked evals whose jobs constraints are
+	// captured by computed node class.
+	b.captured[eval.ID] = wrapped
+}
+
+// processBlockJobDuplicate handles the case where the new eval is for a job
+// that we are already tracking. If the eval is a duplicate, we add the older
+// evaluation by Raft index to the list of duplicates such that it can be
+// cancelled. We only ever want one blocked evaluation per job, otherwise we
+// would create unnecessary work for the scheduler as multiple evals for the
+// same job would be run, all producing the same outcome. It is critical to
+// prefer the newer evaluation, since it will contain the most up to date set of
+// class eligibility. The return value is set to true, if the passed evaluation
+// is cancelled. This should be called with the lock held.
+func (b *BlockedEvals) processBlockJobDuplicate(eval *structs.Evaluation) (newCancelled bool) {
+	existingID, hasExisting := b.jobs[structs.NewNamespacedID(eval.JobID, eval.Namespace)]
+	if !hasExisting {
+		return
+	}
+
+	var dup *structs.Evaluation
+	existingW, ok := b.captured[existingID]
+	if ok {
+		if latestEvalIndex(existingW.eval) <= latestEvalIndex(eval) {
+			delete(b.captured, existingID)
+			dup = existingW.eval
+			b.stats.Unblock(dup, false)
+		} else {
+			dup = eval
+			newCancelled = true
+		}
+	} else {
+		existingW, ok = b.escaped[existingID]
+		if !ok {
+			// This is a programming error
+			b.logger.Error("existing blocked evaluation is neither tracked as captured or escaped", "existing_id", existingID)
+			delete(b.jobs, structs.NewNamespacedID(eval.JobID, eval.Namespace))
+			return
+		}
+
+		if latestEvalIndex(existingW.eval) <= latestEvalIndex(eval) {
+			delete(b.escaped, existingID)
+			b.stats.decrementEscaped()
+			dup = existingW.eval
+		} else {
+			dup = eval
+			newCancelled = true
+		}
+	}
+
+	b.duplicates = append(b.duplicates, dup)
+
+	// Unblock any waiter.
+	select {
+	case b.duplicateCh <- struct{}{}:
+	default:
+	}
+
+	return
+}
+
+// latestEvalIndex returns the max of the evaluations create and snapshot index
+func latestEvalIndex(eval *structs.Evaluation) uint64 {
+	if eval == nil {
+		return 0
+	}
+
+	return max(eval.CreateIndex, eval.SnapshotIndex)
+}
+
+// missedUnblock returns whether an evaluation missed an unblock while it was in
+// the scheduler. Since the scheduler can operate at an index in the past, the
+// evaluation may have been processed missing data that would allow it to
+// complete. This method returns if that is the case and should be called with
+// the lock held.
+func (b *BlockedEvals) missedUnblock(eval *structs.Evaluation) bool {
+	b.unblockIndexesLock.RLock()
+	defer b.unblockIndexesLock.RUnlock()
+
+	var max uint64 = 0
+
+	for id, u := range b.unblockIndexes {
+		// Calculate the max unblock index
+		if max < u.index {
+			max = u.index
+		}
+
+		// The evaluation is blocked because it has hit a quota limit not class
+		// eligibility
+		if eval.QuotaLimitReached != "" {
+			if eval.QuotaLimitReached != id {
+				// Not a match
+				continue
+			} else if eval.SnapshotIndex < u.index {
+				// The evaluation was processed before the quota specification was
+				// updated, so unblock the evaluation.
+				return true
+			}
+
+			// The evaluation was processed having seen all changes to the quota
+			return false
+		}
+
+		elig, ok := eval.ClassEligibility[id]
+		if !ok && eval.SnapshotIndex < u.index {
+			// The evaluation was processed and did not encounter this class
+			// because it was added after it was processed. Thus for correctness
+			// we need to unblock it.
+			return true
+		}
+
+		// The evaluation could use the computed node class and the eval was
+		// processed before the last unblock.
+		if elig && eval.SnapshotIndex < u.index {
+			return true
+		}
+	}
+
+	// If the evaluation has escaped, and the map contains an index older than
+	// the evaluations, it should be unblocked.
+	if eval.EscapedComputedClass && eval.SnapshotIndex < max {
+		return true
+	}
+
+	// The evaluation is ahead of all recent unblocks.
+	return false
+}
+
+// Untrack causes any blocked evaluation for the passed job to be no longer
+// tracked. Untrack is called when there is a successful evaluation for the job
+// and a blocked evaluation is no longer needed.
+func (b *BlockedEvals) Untrack(jobID, namespace string) chan struct{} {
+	fut := make(chan struct{})
+	b.flushLock.RLock()
+	defer b.flushLock.RUnlock()
+	if !b.enabled {
+		close(fut)
+		return fut
+	}
+
+	// Capture chan in flushlock as Flush overwrites it
+	ch := b.capacityChangeCh
+	done := b.stopCh
+
+	nsID := structs.NewNamespacedID(jobID, namespace)
+
+	select {
+	case <-done:
+	case ch <- &capacityUpdate{untrackJobID: nsID, future: fut}:
+	}
+
+	return fut
+}
+
+func (b *BlockedEvals) untrackImpl(nsID structs.NamespacedID) {
+	b.flushLock.RLock()
+	defer b.flushLock.RUnlock()
+	if !b.enabled {
+		return
+	}
+
+	b.l.Lock()
+	defer b.l.Unlock()
+
+	if evals, ok := b.system.JobEvals(nsID); ok {
+		for _, e := range evals {
+			b.system.Remove(e)
+			b.stats.Unblock(e, false)
+		}
+		return
+	}
+
+	// Get the evaluation ID to cancel
+	evalID, ok := b.jobs[nsID]
+	if !ok {
+		// No blocked evaluation so exit
+		return
+	}
+
+	// Attempt to delete the evaluation
+	if w, ok := b.captured[evalID]; ok {
+		delete(b.jobs, nsID)
+		delete(b.captured, evalID)
+		b.stats.Unblock(w.eval, false)
+	}
+
+	if w, ok := b.escaped[evalID]; ok {
+		delete(b.jobs, nsID)
+		delete(b.escaped, evalID)
+		b.stats.Unblock(w.eval, true)
+	}
+}
+
+// Unblock causes any evaluation that could potentially make progress on a
+// capacity change on the passed computed node class to be enqueued into the
+// eval broker.
+func (b *BlockedEvals) Unblock(computedClass string, index uint64) chan struct{} {
+	fut := make(chan struct{})
+
+	b.flushLock.RLock()
+	defer b.flushLock.RUnlock()
+	if !b.enabled {
+		close(fut)
+		return fut
+	}
+	// Capture chan in flushlock as Flush overwrites it
+	ch := b.capacityChangeCh
+	done := b.stopCh
+
+	// Store the index in which the unblock happened. We use this on subsequent
+	// block calls in case the evaluation was in the scheduler when a trigger
+	// occurred.
+	b.unblockIndexesLock.Lock()
+	b.unblockIndexes[computedClass] = unblockEvent{index, time.Now().UTC()}
+	b.unblockIndexesLock.Unlock()
+
+	select {
+	case <-done:
+	case ch <- &capacityUpdate{computedClass: computedClass, future: fut}:
+	}
+
+	return fut
+}
+
+// UnblockQuota causes any evaluation that could potentially make progress on a
+// capacity change on the passed quota to be enqueued into the eval broker.
+func (b *BlockedEvals) UnblockQuota(quota string, index uint64) chan struct{} {
+	fut := make(chan struct{})
+
+	// Nothing to do
+	if quota == "" {
+		return fut
+	}
+
+	b.flushLock.RLock()
+	defer b.flushLock.RUnlock()
+	if !b.enabled {
+		close(fut)
+		return fut
+	}
+	// Capture chan in flushlock as Flush overwrites it
+	ch := b.capacityChangeCh
+	done := b.stopCh
+
+	// Store the index in which the unblock happened. We use this on subsequent
+	// block calls in case the evaluation was in the scheduler when a trigger
+	// occurred.
+	b.unblockIndexesLock.Lock()
+	b.unblockIndexes[quota] = unblockEvent{index, time.Now().UTC()}
+	b.unblockIndexesLock.Unlock()
+
+	select {
+	case <-done:
+	case ch <- &capacityUpdate{quotaChange: quota, future: fut}:
+	}
+
+	return fut
+}
+
+// UnblockClassAndQuota causes any evaluation that could potentially make
+// progress on a capacity change on the passed computed node class or quota to
+// be enqueued into the eval broker.
+func (b *BlockedEvals) UnblockClassAndQuota(class, quota string, index uint64) chan struct{} {
+	fut := make(chan struct{})
+
+	b.flushLock.RLock()
+	defer b.flushLock.RUnlock()
+	if !b.enabled {
+		close(fut)
+		return fut
+	}
+	// Capture chan in flushlock as Flush overwrites it
+	ch := b.capacityChangeCh
+	done := b.stopCh
+
+	// Store the index in which the unblock happened. We use this on subsequent
+	// block calls in case the evaluation was in the scheduler when a trigger
+	// occurred.
+	b.unblockIndexesLock.Lock()
+	now := time.Now().UTC()
+	if quota != "" {
+		b.unblockIndexes[quota] = unblockEvent{index, now}
+	}
+	b.unblockIndexes[class] = unblockEvent{index, now}
+	b.unblockIndexesLock.Unlock()
+
+	select {
+	case <-done:
+	case ch <- &capacityUpdate{computedClass: class, quotaChange: quota, future: fut}:
+	}
+
+	return fut
+}
+
+// UnblockNode finds any blocked evalution that's node specific (system jobs)
+// and enqueues it on the eval broker.
+func (b *BlockedEvals) UnblockNode(nodeID string) chan struct{} {
+	fut := make(chan struct{})
+
+	b.flushLock.RLock()
+	defer b.flushLock.RUnlock()
+	if !b.enabled {
+		close(fut)
+		return fut
+	}
+	// Capture chan in flushlock as Flush overwrites it
+	ch := b.capacityChangeCh
+	done := b.stopCh
+
+	// Note that unlike other unblock methods, we can't usefully track an
+	// unblockIndex because any blocked eval won't have the node ID attached
+	// when we check the index.
+
+	select {
+	case <-done:
+	case ch <- &capacityUpdate{nodeID: nodeID, future: fut}:
+	}
+
+	return fut
+}
+
+// watchCapacity is a long lived function that watches for capacity changes in
+// nodes and unblocks the correct set of evals.
+func (b *BlockedEvals) watchCapacity(
+	stopCh <-chan struct{},
+	changeCh <-chan *capacityUpdate,
+) {
+	for {
+		select {
+		case <-stopCh:
+			return
+		case update := <-changeCh:
+			if update.blockedEval != nil {
+				b.processBlock(update.blockedEval, update.blockToken)
+				close(update.future)
+				continue
+			}
+			if update.untrackJobID.ID != "" {
+				b.untrackImpl(update.untrackJobID)
+				close(update.future)
+				continue
+			}
+
+			b.unblock(update.computedClass, update.quotaChange, update.nodeID)
+			close(update.future)
+		}
+	}
+}
+
+func (b *BlockedEvals) unblock(computedClass, quota, nodeID string) {
+
+	// Protect against the case of a flush.
+	b.flushLock.RLock()
+	defer b.flushLock.RUnlock()
+	if !b.enabled {
+		return
+	}
+
+	b.l.Lock()
+	defer b.l.Unlock()
+
+	// Every eval that has escaped computed node class has to be unblocked
+	// because any node could potentially be feasible.
+	numEscaped := len(b.escaped)
+	unblocked := make(map[*structs.Evaluation]string, max(uint64(numEscaped), 4))
+
+	if numEscaped != 0 && computedClass != "" {
+		for id, wrapped := range b.escaped {
+			unblocked[wrapped.eval] = wrapped.token
+			delete(b.escaped, id)
+			delete(b.jobs, structs.NewNamespacedID(wrapped.eval.JobID, wrapped.eval.Namespace))
+		}
+	}
+
+	if quota == "" && computedClass == "" {
+		goto SKIP_TO_NODE
+	}
+
+	// We unblock any eval that is explicitly eligible for the computed class
+	// and also any eval that is not eligible or uneligible. This signifies that
+	// when the evaluation was originally run through the scheduler, that it
+	// never saw a node with the given computed class and thus needs to be
+	// unblocked for correctness.
+	for id, wrapped := range b.captured {
+		if quota != "" &&
+			wrapped.eval.QuotaLimitReached != "" &&
+			wrapped.eval.QuotaLimitReached != quota {
+			// We are unblocking based on quota and this eval doesn't match
+			continue
+		}
+		if elig, ok := wrapped.eval.ClassEligibility[computedClass]; ok && !elig {
+			// Can skip because the eval has explicitly marked the node class
+			// as ineligible.
+			continue
+		}
+
+		// Unblock the evaluation because it is either for the matching quota,
+		// is eligible based on the computed node class, or never seen the
+		// computed node class.
+		unblocked[wrapped.eval] = wrapped.token
+		delete(b.jobs, structs.NewNamespacedID(wrapped.eval.JobID, wrapped.eval.Namespace))
+		delete(b.captured, id)
+	}
+
+SKIP_TO_NODE:
+	if nodeID != "" {
+		evals, _ := b.system.NodeEvals(nodeID)
+		for eval, token := range evals {
+			b.system.Remove(eval)
+			unblocked[eval] = token
+		}
+	}
+
+	if len(unblocked) != 0 {
+		// Update the counters and reset TotalEscaped
+		b.stats.UnblockAll(unblocked, -1)
+
+		// Enqueue all the unblocked evals into the broker.
+		b.evalBroker.EnqueueAll(unblocked)
+	}
+}
+
+// UnblockFailed unblocks all blocked evaluation that were due to scheduler
+// failure.
+func (b *BlockedEvals) UnblockFailed() {
+	b.flushLock.RLock()
+	defer b.flushLock.RUnlock()
+	if !b.enabled {
+		return
+	}
+
+	b.l.Lock()
+	defer b.l.Unlock()
+
+	escaped := 0
+	unblocked := make(map[*structs.Evaluation]string, 4)
+	for id, wrapped := range b.captured {
+		if wrapped.eval.TriggeredBy == structs.EvalTriggerMaxPlans {
+			unblocked[wrapped.eval] = wrapped.token
+			delete(b.captured, id)
+			delete(b.jobs, structs.NewNamespacedID(wrapped.eval.JobID, wrapped.eval.Namespace))
+		}
+	}
+
+	for id, wrapped := range b.escaped {
+		if wrapped.eval.TriggeredBy == structs.EvalTriggerMaxPlans {
+			unblocked[wrapped.eval] = wrapped.token
+			delete(b.escaped, id)
+			delete(b.jobs, structs.NewNamespacedID(wrapped.eval.JobID, wrapped.eval.Namespace))
+			escaped++
+		}
+	}
+
+	if len(unblocked) > 0 {
+		b.stats.UnblockAll(unblocked, escaped)
+		b.evalBroker.EnqueueAll(unblocked)
+	}
+}
+
+// GetDuplicates returns all the duplicate evaluations and blocks until the
+// passed timeout.
+func (b *BlockedEvals) GetDuplicates(timeout time.Duration) []*structs.Evaluation {
+	var timeoutTimer *time.Timer
+	var timeoutCh <-chan time.Time
+SCAN:
+	b.l.Lock()
+	if len(b.duplicates) != 0 {
+		dups := b.duplicates
+		b.duplicates = nil
+		b.l.Unlock()
+		return dups
+	}
+
+	// Capture chans inside the lock to prevent a race with them getting
+	// reset in Flush
+	dupCh := b.duplicateCh
+	stopCh := b.stopCh
+	b.l.Unlock()
+
+	// Create the timer
+	if timeoutTimer == nil && timeout != 0 {
+		timeoutTimer = time.NewTimer(timeout)
+		timeoutCh = timeoutTimer.C
+		defer timeoutTimer.Stop()
+	}
+
+	select {
+	case <-stopCh:
+		return nil
+	case <-timeoutCh:
+		return nil
+	case <-dupCh:
+		goto SCAN
+	}
+}
+
+// Flush is used to clear the state of blocked evaluations.
+func (b *BlockedEvals) Flush() {
+	b.flushLock.Lock()
+	defer b.flushLock.Unlock()
+
+	b.l.Lock()
+	defer b.l.Unlock()
+
+	// Reset stats
+	b.stats.Reset()
+
+	// Reset the tracker
+	b.captured = make(map[string]wrappedEval)
+	b.escaped = make(map[string]wrappedEval)
+	b.jobs = make(map[structs.NamespacedID]string)
+	b.unblockIndexes = make(map[string]unblockEvent)
+	b.duplicates = nil
+	b.capacityChangeCh = make(chan *capacityUpdate, unblockBuffer)
+	b.stopCh = make(chan struct{})
+	b.duplicateCh = make(chan struct{}, 1)
+	b.system = newSystemEvals()
+}
+
+// EmitStats is used to export metrics about the blocked eval tracker while enabled
+func (b *BlockedEvals) EmitStats(period time.Duration, stopCh <-chan struct{}) {
+	timer, stop := helper.NewSafeTimer(period)
+	defer stop()
+
+	for {
+		timer.Reset(period)
+
+		select {
+
+		case <-timer.C:
+			stats := b.stats.Copy()
+			metrics.SetGauge([]string{"nomad", "blocked_evals", "total_quota_limit"}, float32(stats.TotalQuotaLimit))
+			metrics.SetGauge([]string{"nomad", "blocked_evals", "total_blocked"}, float32(stats.TotalBlocked))
+			metrics.SetGauge([]string{"nomad", "blocked_evals", "total_escaped"}, float32(stats.TotalEscaped))
+
+			for k, v := range stats.BlockedResources.ByJob {
+				labels := []metrics.Label{
+					{Name: "namespace", Value: k.Namespace},
+					{Name: "job", Value: k.ID},
+				}
+				metrics.SetGaugeWithLabels([]string{"nomad", "blocked_evals", "job", "cpu"}, float32(v.CPU), labels)
+				metrics.SetGaugeWithLabels([]string{"nomad", "blocked_evals", "job", "memory"}, float32(v.MemoryMB), labels)
+			}
+
+			for k, v := range stats.BlockedResources.ByClassInDC {
+				labels := []metrics.Label{
+					{Name: "datacenter", Value: k.dc},
+					{Name: "node_class", Value: k.class},
+					{Name: "node_pool", Value: k.nodepool},
+				}
+				metrics.SetGaugeWithLabels([]string{"nomad", "blocked_evals", "cpu"}, float32(v.CPU), labels)
+				metrics.SetGaugeWithLabels([]string{"nomad", "blocked_evals", "memory"}, float32(v.MemoryMB), labels)
+			}
+		case <-stopCh:
+			return
+		}
+	}
+}
+
+// prune is a long lived function that prunes unnecessary objects on a timer.
+func (b *BlockedEvals) prune(stopCh <-chan struct{}) {
+	ticker := time.NewTicker(pruneInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-stopCh:
+			return
+		case t := <-ticker.C:
+			cutoff := t.UTC().Add(-1 * pruneThreshold)
+			b.pruneUnblockIndexes(cutoff)
+			b.pruneStats(cutoff)
+		}
+	}
+}
+
+// pruneUnblockIndexes is used to prune any tracked entry that is excessively
+// old. This protects against unbounded growth of the map.
+func (b *BlockedEvals) pruneUnblockIndexes(cutoff time.Time) {
+	b.unblockIndexesLock.Lock()
+	defer b.unblockIndexesLock.Unlock()
+
+	for key, u := range b.unblockIndexes {
+		if u.timestamp.Before(cutoff) {
+			delete(b.unblockIndexes, key)
+		}
+	}
+}
+
+// pruneStats is used to prune any zero value stats that are excessively old.
+func (b *BlockedEvals) pruneStats(cutoff time.Time) {
+	b.stats.prune(cutoff)
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/blocked_evals_stats.go

@@ -0,0 +1,299 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"sync"
+	"time"
+
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+// BlockedStats returns all the stats about the blocked eval tracker.
+type BlockedStats struct {
+	// TotalEscaped is the total number of blocked evaluations that have escaped
+	// computed node classes.
+	TotalEscaped int
+
+	// TotalBlocked is the total number of blocked evaluations.
+	TotalBlocked int
+
+	// TotalQuotaLimit is the total number of blocked evaluations that are due
+	// to the quota limit being reached.
+	TotalQuotaLimit int
+
+	// BlockedResources stores the amount of resources requested by blocked
+	// evaluations.
+	BlockedResources *BlockedResourcesStats
+
+	lock sync.RWMutex
+}
+
+// classInDC is a coordinate of a specific class in a specific datacenter
+type classInDC struct {
+	dc       string
+	class    string
+	nodepool string
+}
+
+// NewBlockedStats returns a new BlockedStats.
+func NewBlockedStats() *BlockedStats {
+	return &BlockedStats{
+		BlockedResources: NewBlockedResourcesStats(),
+	}
+}
+
+func (b *BlockedStats) Reset() {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+
+	b.TotalEscaped = 0
+	b.TotalBlocked = 0
+	b.TotalQuotaLimit = 0
+	b.BlockedResources = NewBlockedResourcesStats()
+}
+
+// Block updates the stats for the blocked eval tracker with the details of the
+// evaluation being blocked.
+func (b *BlockedStats) Block(eval *structs.Evaluation) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+
+	b.TotalBlocked++
+	resourceStats := generateResourceStats(eval)
+	b.BlockedResources = b.BlockedResources.Add(resourceStats)
+
+	// Track that the evaluation is being added due to reaching the quota limit
+	if eval.QuotaLimitReached != "" {
+		b.TotalQuotaLimit++
+	}
+
+	// If the eval has escaped, meaning computed node classes could not capture
+	// the constraints of the job, we store the eval separately as we have to
+	// unblock it whenever node capacity changes. This is because we don't know
+	// what node class is feasible for the jobs constraints.
+	if eval.EscapedComputedClass {
+		b.TotalEscaped++
+	}
+}
+
+// Copy returns a deep clone of the BlockedStats, holding a read lock
+func (b *BlockedStats) Copy() *BlockedStats {
+	stats := NewBlockedStats()
+
+	b.lock.RLock()
+	defer b.lock.RUnlock()
+
+	stats.TotalEscaped = b.TotalEscaped
+	stats.TotalBlocked = b.TotalBlocked
+	stats.TotalQuotaLimit = b.TotalQuotaLimit
+	stats.BlockedResources = b.BlockedResources.Copy()
+	return stats
+}
+
+// Unblock updates the stats for the blocked eval tracker with the details of the
+// evaluation being unblocked.
+func (b *BlockedStats) Unblock(eval *structs.Evaluation, escaped bool) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	b.unblockImpl(eval, escaped)
+}
+
+// UnblockAll updates the stats for a set of evaluations. It will also decrement
+// TotalEscaped or reset it to zero if negative
+func (b *BlockedStats) UnblockAll(evals map[*structs.Evaluation]string, escaped int) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+
+	for eval := range evals {
+		b.unblockImpl(eval, false)
+	}
+	if escaped >= 0 {
+		b.TotalEscaped -= escaped
+	} else {
+		b.TotalEscaped = 0
+	}
+}
+
+// unblockImpl expects that b.lock is held by the caller
+func (b *BlockedStats) unblockImpl(eval *structs.Evaluation, escaped bool) {
+	b.TotalBlocked--
+
+	resourceStats := generateResourceStats(eval)
+	b.BlockedResources = b.BlockedResources.Subtract(resourceStats)
+
+	if eval.QuotaLimitReached != "" {
+		b.TotalQuotaLimit--
+	}
+	if escaped {
+		b.TotalEscaped--
+	}
+}
+
+func (b *BlockedStats) decrementEscaped() {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	b.TotalEscaped--
+}
+
+// prune deletes any key zero metric values older than the cutoff.
+func (b *BlockedStats) prune(cutoff time.Time) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+
+	shouldPrune := func(s BlockedResourcesSummary) bool {
+		return s.Timestamp.Before(cutoff) && s.IsZero()
+	}
+
+	for k, v := range b.BlockedResources.ByJob {
+		if shouldPrune(v) {
+			delete(b.BlockedResources.ByJob, k)
+		}
+	}
+
+	for k, v := range b.BlockedResources.ByClassInDC {
+		if shouldPrune(v) {
+			delete(b.BlockedResources.ByClassInDC, k)
+		}
+	}
+}
+
+// generateResourceStats returns a summary of the resources requested by the
+// input evaluation.
+func generateResourceStats(eval *structs.Evaluation) *BlockedResourcesStats {
+	dcs := make(map[string]struct{})
+	classes := make(map[string]struct{})
+	nodepools := make(map[string]struct{})
+
+	resources := BlockedResourcesSummary{
+		Timestamp: time.Now().UTC(),
+	}
+
+	for _, allocMetrics := range eval.FailedTGAllocs {
+		for dc := range allocMetrics.NodesAvailable {
+			dcs[dc] = struct{}{}
+		}
+		for class := range allocMetrics.ClassExhausted {
+			classes[class] = struct{}{}
+		}
+
+		nodepools[allocMetrics.NodePool] = struct{}{}
+
+		if len(allocMetrics.ClassExhausted) == 0 {
+			// some evaluations have no class
+			classes[""] = struct{}{}
+		}
+		for _, r := range allocMetrics.ResourcesExhausted {
+			resources.CPU += r.CPU
+			resources.MemoryMB += r.MemoryMB
+		}
+	}
+
+	byJob := make(map[structs.NamespacedID]BlockedResourcesSummary)
+	nsID := structs.NewNamespacedID(eval.JobID, eval.Namespace)
+	byJob[nsID] = resources
+
+	byClassInDC := make(map[classInDC]BlockedResourcesSummary)
+	for nodepool := range nodepools {
+		for dc := range dcs {
+			for class := range classes {
+				k := classInDC{dc: dc, class: class, nodepool: nodepool}
+				byClassInDC[k] = resources
+			}
+		}
+	}
+
+	return &BlockedResourcesStats{
+		ByJob:       byJob,
+		ByClassInDC: byClassInDC,
+	}
+}
+
+// BlockedResourcesStats stores resources requested by blocked evaluations,
+// tracked both by job and by node.
+type BlockedResourcesStats struct {
+	ByJob       map[structs.NamespacedID]BlockedResourcesSummary
+	ByClassInDC map[classInDC]BlockedResourcesSummary
+}
+
+// NewBlockedResourcesStats returns a new BlockedResourcesStats.
+func NewBlockedResourcesStats() *BlockedResourcesStats {
+	return &BlockedResourcesStats{
+		ByJob:       make(map[structs.NamespacedID]BlockedResourcesSummary),
+		ByClassInDC: make(map[classInDC]BlockedResourcesSummary),
+	}
+}
+
+// Copy returns a deep copy of the blocked resource stats.
+func (b *BlockedResourcesStats) Copy() *BlockedResourcesStats {
+	result := NewBlockedResourcesStats()
+
+	for k, v := range b.ByJob {
+		result.ByJob[k] = v // value copy
+	}
+
+	for k, v := range b.ByClassInDC {
+		result.ByClassInDC[k] = v // value copy
+	}
+
+	return result
+}
+
+// Add returns a new BlockedResourcesStats with the values set to the current
+// resource values plus the input.
+func (b *BlockedResourcesStats) Add(a *BlockedResourcesStats) *BlockedResourcesStats {
+
+	for k, v := range a.ByJob {
+		b.ByJob[k] = b.ByJob[k].Add(v)
+	}
+	for k, v := range a.ByClassInDC {
+		b.ByClassInDC[k] = b.ByClassInDC[k].Add(v)
+	}
+	return b
+}
+
+// Subtract returns a new BlockedResourcesStats with the values set to the
+// current resource values minus the input.
+func (b *BlockedResourcesStats) Subtract(a *BlockedResourcesStats) *BlockedResourcesStats {
+
+	for k, v := range a.ByJob {
+		b.ByJob[k] = b.ByJob[k].Subtract(v)
+	}
+	for k, v := range a.ByClassInDC {
+		b.ByClassInDC[k] = b.ByClassInDC[k].Subtract(v)
+	}
+	return b
+}
+
+// BlockedResourcesSummary stores resource values for blocked evals.
+type BlockedResourcesSummary struct {
+	Timestamp time.Time
+	CPU       int
+	MemoryMB  int
+}
+
+// Add returns a new BlockedResourcesSummary with each resource set to the
+// current value plus the input.
+func (b BlockedResourcesSummary) Add(a BlockedResourcesSummary) BlockedResourcesSummary {
+	return BlockedResourcesSummary{
+		Timestamp: a.Timestamp,
+		CPU:       b.CPU + a.CPU,
+		MemoryMB:  b.MemoryMB + a.MemoryMB,
+	}
+}
+
+// Subtract returns a new BlockedResourcesSummary with each resource set to the
+// current value minus the input.
+func (b BlockedResourcesSummary) Subtract(a BlockedResourcesSummary) BlockedResourcesSummary {
+	return BlockedResourcesSummary{
+		Timestamp: a.Timestamp,
+		CPU:       b.CPU - a.CPU,
+		MemoryMB:  b.MemoryMB - a.MemoryMB,
+	}
+}
+
+// IsZero returns true if all resource values are zero.
+func (b BlockedResourcesSummary) IsZero() bool {
+	return b.CPU == 0 && b.MemoryMB == 0
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/blocked_evals_stats_test.go

@@ -0,0 +1,471 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"fmt"
+	"math/rand"
+	"reflect"
+	"testing"
+	"testing/quick"
+	"time"
+
+	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/nomad/mock"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/stretchr/testify/require"
+)
+
+func now(year int) time.Time {
+	return time.Date(2000+year, 1, 2, 3, 4, 5, 6, time.UTC)
+}
+
+func TestBlockedResourceSummary_Add(t *testing.T) {
+	now1 := now(1)
+	now2 := now(2)
+	a := BlockedResourcesSummary{
+		Timestamp: now1,
+		CPU:       600,
+		MemoryMB:  256,
+	}
+
+	b := BlockedResourcesSummary{
+		Timestamp: now2,
+		CPU:       250,
+		MemoryMB:  128,
+	}
+
+	result := a.Add(b)
+
+	// a not modified
+	require.Equal(t, 600, a.CPU)
+	require.Equal(t, 256, a.MemoryMB)
+	require.Equal(t, now1, a.Timestamp)
+
+	// b not modified
+	require.Equal(t, 250, b.CPU)
+	require.Equal(t, 128, b.MemoryMB)
+	require.Equal(t, now2, b.Timestamp)
+
+	// result is a + b, using timestamp from b
+	require.Equal(t, 850, result.CPU)
+	require.Equal(t, 384, result.MemoryMB)
+	require.Equal(t, now2, result.Timestamp)
+}
+
+func TestBlockedResourceSummary_Add_nil(t *testing.T) {
+	now1 := now(1)
+	b := BlockedResourcesSummary{
+		Timestamp: now1,
+		CPU:       250,
+		MemoryMB:  128,
+	}
+
+	// zero + b == b
+	result := (BlockedResourcesSummary{}).Add(b)
+	require.Equal(t, now1, result.Timestamp)
+	require.Equal(t, 250, result.CPU)
+	require.Equal(t, 128, result.MemoryMB)
+}
+
+func TestBlockedResourceSummary_Subtract(t *testing.T) {
+	now1 := now(1)
+	now2 := now(2)
+	a := BlockedResourcesSummary{
+		Timestamp: now1,
+		CPU:       600,
+		MemoryMB:  256,
+	}
+
+	b := BlockedResourcesSummary{
+		Timestamp: now2,
+		CPU:       250,
+		MemoryMB:  120,
+	}
+
+	result := a.Subtract(b)
+
+	// a not modified
+	require.Equal(t, 600, a.CPU)
+	require.Equal(t, 256, a.MemoryMB)
+	require.Equal(t, now1, a.Timestamp)
+
+	// b not modified
+	require.Equal(t, 250, b.CPU)
+	require.Equal(t, 120, b.MemoryMB)
+	require.Equal(t, now2, b.Timestamp)
+
+	// result is a + b, using timestamp from b
+	require.Equal(t, 350, result.CPU)
+	require.Equal(t, 136, result.MemoryMB)
+	require.Equal(t, now2, result.Timestamp)
+}
+
+func TestBlockedResourceSummary_IsZero(t *testing.T) {
+	now1 := now(1)
+
+	// cpu and mem zero, timestamp is ignored
+	require.True(t, (&BlockedResourcesSummary{
+		Timestamp: now1,
+		CPU:       0,
+		MemoryMB:  0,
+	}).IsZero())
+
+	// cpu non-zero
+	require.False(t, (&BlockedResourcesSummary{
+		Timestamp: now1,
+		CPU:       1,
+		MemoryMB:  0,
+	}).IsZero())
+
+	// mem non-zero
+	require.False(t, (&BlockedResourcesSummary{
+		Timestamp: now1,
+		CPU:       0,
+		MemoryMB:  1,
+	}).IsZero())
+}
+
+func TestBlockedResourceStats_New(t *testing.T) {
+	a := NewBlockedResourcesStats()
+	require.NotNil(t, a.ByJob)
+	require.Empty(t, a.ByJob)
+	require.NotNil(t, a.ByClassInDC)
+	require.Empty(t, a.ByClassInDC)
+}
+
+var (
+	id1 = structs.NamespacedID{
+		ID:        "1",
+		Namespace: "one",
+	}
+
+	id2 = structs.NamespacedID{
+		ID:        "2",
+		Namespace: "two",
+	}
+
+	node1 = classInDC{
+		dc:       "dc1",
+		class:    "alpha",
+		nodepool: "default",
+	}
+
+	node2 = classInDC{
+		dc:       "dc1",
+		class:    "beta",
+		nodepool: "dev",
+	}
+
+	node3 = classInDC{
+		dc:    "dc1",
+		class: "", // not set
+	}
+)
+
+func TestBlockedResourceStats_Copy(t *testing.T) {
+	now1 := now(1)
+	now2 := now(2)
+
+	a := NewBlockedResourcesStats()
+	a.ByJob = map[structs.NamespacedID]BlockedResourcesSummary{
+		id1: {
+			Timestamp: now1,
+			CPU:       100,
+			MemoryMB:  256,
+		},
+	}
+	a.ByClassInDC = map[classInDC]BlockedResourcesSummary{
+		node1: {
+			Timestamp: now1,
+			CPU:       300,
+			MemoryMB:  333,
+		},
+	}
+
+	c := a.Copy()
+	c.ByJob[id1] = BlockedResourcesSummary{
+		Timestamp: now2,
+		CPU:       888,
+		MemoryMB:  888,
+	}
+	c.ByClassInDC[node1] = BlockedResourcesSummary{
+		Timestamp: now2,
+		CPU:       999,
+		MemoryMB:  999,
+	}
+
+	// underlying data should have been deep copied
+	require.Equal(t, 100, a.ByJob[id1].CPU)
+	require.Equal(t, 300, a.ByClassInDC[node1].CPU)
+}
+
+func TestBlockedResourcesStats_Add(t *testing.T) {
+	a := NewBlockedResourcesStats()
+	a.ByJob = map[structs.NamespacedID]BlockedResourcesSummary{
+		id1: {Timestamp: now(1), CPU: 111, MemoryMB: 222},
+	}
+	a.ByClassInDC = map[classInDC]BlockedResourcesSummary{
+		node1: {Timestamp: now(2), CPU: 333, MemoryMB: 444},
+	}
+
+	b := NewBlockedResourcesStats()
+	b.ByJob = map[structs.NamespacedID]BlockedResourcesSummary{
+		id1: {Timestamp: now(3), CPU: 200, MemoryMB: 300},
+		id2: {Timestamp: now(4), CPU: 400, MemoryMB: 500},
+	}
+	b.ByClassInDC = map[classInDC]BlockedResourcesSummary{
+		node1: {Timestamp: now(5), CPU: 600, MemoryMB: 700},
+		node2: {Timestamp: now(6), CPU: 800, MemoryMB: 900},
+	}
+
+	t.Run("a add b", func(t *testing.T) {
+		result := a.Copy().Add(b)
+
+		require.Equal(t, map[structs.NamespacedID]BlockedResourcesSummary{
+			id1: {Timestamp: now(3), CPU: 311, MemoryMB: 522},
+			id2: {Timestamp: now(4), CPU: 400, MemoryMB: 500},
+		}, result.ByJob)
+
+		require.Equal(t, map[classInDC]BlockedResourcesSummary{
+			node1: {Timestamp: now(5), CPU: 933, MemoryMB: 1144},
+			node2: {Timestamp: now(6), CPU: 800, MemoryMB: 900},
+		}, result.ByClassInDC)
+	})
+
+	// make sure we handle zeros in both directions
+	// and timestamps originate from rhs
+	t.Run("b add a", func(t *testing.T) {
+		result := b.Copy().Add(a)
+		require.Equal(t, map[structs.NamespacedID]BlockedResourcesSummary{
+			id1: {Timestamp: now(1), CPU: 311, MemoryMB: 522},
+			id2: {Timestamp: now(4), CPU: 400, MemoryMB: 500},
+		}, result.ByJob)
+
+		require.Equal(t, map[classInDC]BlockedResourcesSummary{
+			node1: {Timestamp: now(2), CPU: 933, MemoryMB: 1144},
+			node2: {Timestamp: now(6), CPU: 800, MemoryMB: 900},
+		}, result.ByClassInDC)
+	})
+}
+
+func TestBlockedResourcesStats_Add_NoClass(t *testing.T) {
+	a := NewBlockedResourcesStats()
+	a.ByClassInDC = map[classInDC]BlockedResourcesSummary{
+		node3: {Timestamp: now(1), CPU: 111, MemoryMB: 1111},
+	}
+	result := a.Add(a)
+	require.Equal(t, map[classInDC]BlockedResourcesSummary{
+		node3: {Timestamp: now(1), CPU: 222, MemoryMB: 2222},
+	}, result.ByClassInDC)
+}
+
+func TestBlockedResourcesStats_Subtract(t *testing.T) {
+	a := NewBlockedResourcesStats()
+	a.ByJob = map[structs.NamespacedID]BlockedResourcesSummary{
+		id1: {Timestamp: now(1), CPU: 100, MemoryMB: 100},
+		id2: {Timestamp: now(2), CPU: 200, MemoryMB: 200},
+	}
+	a.ByClassInDC = map[classInDC]BlockedResourcesSummary{
+		node1: {Timestamp: now(3), CPU: 300, MemoryMB: 300},
+		node2: {Timestamp: now(4), CPU: 400, MemoryMB: 400},
+	}
+
+	b := NewBlockedResourcesStats()
+	b.ByJob = map[structs.NamespacedID]BlockedResourcesSummary{
+		id1: {Timestamp: now(5), CPU: 10, MemoryMB: 11},
+		id2: {Timestamp: now(6), CPU: 12, MemoryMB: 13},
+	}
+	b.ByClassInDC = map[classInDC]BlockedResourcesSummary{
+		node1: {Timestamp: now(7), CPU: 14, MemoryMB: 15},
+		node2: {Timestamp: now(8), CPU: 16, MemoryMB: 17},
+	}
+
+	result := a.Subtract(b)
+
+	// id1
+	require.Equal(t, now(5), result.ByJob[id1].Timestamp)
+	require.Equal(t, 90, result.ByJob[id1].CPU)
+	require.Equal(t, 89, result.ByJob[id1].MemoryMB)
+
+	// id2
+	require.Equal(t, now(6), result.ByJob[id2].Timestamp)
+	require.Equal(t, 188, result.ByJob[id2].CPU)
+	require.Equal(t, 187, result.ByJob[id2].MemoryMB)
+
+	// node1
+	require.Equal(t, now(7), result.ByClassInDC[node1].Timestamp)
+	require.Equal(t, 286, result.ByClassInDC[node1].CPU)
+	require.Equal(t, 285, result.ByClassInDC[node1].MemoryMB)
+
+	// node2
+	require.Equal(t, now(8), result.ByClassInDC[node2].Timestamp)
+	require.Equal(t, 384, result.ByClassInDC[node2].CPU)
+	require.Equal(t, 383, result.ByClassInDC[node2].MemoryMB)
+}
+
+// testBlockedEvalsRandomBlockedEval wraps an eval that is randomly generated.
+type testBlockedEvalsRandomBlockedEval struct {
+	eval *structs.Evaluation
+}
+
+// Generate returns a random eval.
+func (t testBlockedEvalsRandomBlockedEval) Generate(rand *rand.Rand, _ int) reflect.Value {
+	resourceTypes := []string{"cpu", "memory"}
+
+	// Start with a mock eval.
+	e := mock.BlockedEval()
+
+	// Get how many task groups, datacenters and node classes to generate.
+	// Add 1 to avoid 0.
+	jobCount := rand.Intn(3) + 1
+	tgCount := rand.Intn(10) + 1
+	dcCount := rand.Intn(3) + 1
+	nodeClassCount := rand.Intn(3) + 1
+	nodePoolName := fmt.Sprintf("node-pool-%d", rand.Intn(3)+1)
+
+	failedTGAllocs := map[string]*structs.AllocMetric{}
+
+	e.JobID = fmt.Sprintf("job-%d", jobCount)
+	for tg := 1; tg <= tgCount; tg++ {
+		tgName := fmt.Sprintf("group-%d", tg)
+
+		// Get which resource type to use for this task group.
+		// Nomad stops at the first dimension that is exhausted, so only 1 is
+		// added per task group.
+		i := rand.Int() % len(resourceTypes)
+		resourceType := resourceTypes[i]
+
+		failedTGAllocs[tgName] = &structs.AllocMetric{
+			DimensionExhausted: map[string]int{
+				resourceType: 1,
+			},
+			NodesAvailable: map[string]int{},
+			ClassExhausted: map[string]int{},
+			NodePool:       nodePoolName,
+		}
+
+		for dc := 1; dc <= dcCount; dc++ {
+			dcName := fmt.Sprintf("dc%d", dc)
+			failedTGAllocs[tgName].NodesAvailable[dcName] = 1
+		}
+
+		for nc := 1; nc <= nodeClassCount; nc++ {
+			nodeClassName := fmt.Sprintf("node-class-%d", nc)
+			failedTGAllocs[tgName].ClassExhausted[nodeClassName] = 1
+		}
+
+		// Generate resources for each task.
+		taskCount := rand.Intn(5) + 1
+		resourcesExhausted := map[string]*structs.Resources{}
+
+		for t := 1; t <= taskCount; t++ {
+			task := fmt.Sprintf("task-%d", t)
+			resourcesExhausted[task] = &structs.Resources{}
+
+			resourceAmount := rand.Intn(1000)
+			switch resourceType {
+			case "cpu":
+				resourcesExhausted[task].CPU = resourceAmount
+			case "memory":
+				resourcesExhausted[task].MemoryMB = resourceAmount
+			}
+		}
+		failedTGAllocs[tgName].ResourcesExhausted = resourcesExhausted
+	}
+	e.FailedTGAllocs = failedTGAllocs
+	t.eval = e
+	return reflect.ValueOf(t)
+}
+
+// clearTimestampFromBlockedResourceStats set timestamp metrics to zero to
+// avoid invalid comparisons.
+func clearTimestampFromBlockedResourceStats(b *BlockedResourcesStats) {
+	for k, v := range b.ByJob {
+		v.Timestamp = time.Time{}
+		b.ByJob[k] = v
+	}
+	for k, v := range b.ByClassInDC {
+		v.Timestamp = time.Time{}
+		b.ByClassInDC[k] = v
+	}
+}
+
+// TestBlockedEvalsStats_BlockedResources generates random evals and processes
+// them using the expected code paths and a manual check of the expeceted result.
+func TestBlockedEvalsStats_BlockedResources(t *testing.T) {
+	ci.Parallel(t)
+	blocked, _ := testBlockedEvals(t)
+
+	// evalHistory stores all evals generated during the test.
+	var evalHistory []*structs.Evaluation
+
+	// blockedEvals keeps track if evals are blocked or unblocked.
+	blockedEvals := map[string]bool{}
+
+	// blockAndUntrack processes the generated evals in order using a
+	// BlockedEvals instance.
+	blockAndUntrack := func(testEval testBlockedEvalsRandomBlockedEval, block bool, unblockIdx uint16) *BlockedResourcesStats {
+		if block || len(evalHistory) == 0 {
+			<-blocked.Block(testEval.eval)
+		} else {
+			i := int(unblockIdx) % len(evalHistory)
+			eval := evalHistory[i]
+			<-blocked.Untrack(eval.JobID, eval.Namespace)
+		}
+		// Remove zero stats from unblocked evals.
+		blocked.pruneStats(time.Now().UTC())
+
+		result := blocked.Stats().BlockedResources
+		clearTimestampFromBlockedResourceStats(result)
+		return result
+	}
+
+	// manualCount processes only the blocked evals and generate a
+	// BlockedResourcesStats result directly from the eval history.
+	manualCount := func(testEval testBlockedEvalsRandomBlockedEval, block bool, unblockIdx uint16) *BlockedResourcesStats {
+
+		if block || len(evalHistory) == 0 {
+
+			evalHistory = append(evalHistory, testEval.eval)
+
+			// Find and unblock evals for the same job.
+			for _, e := range evalHistory {
+				if e.Namespace == testEval.eval.Namespace && e.JobID == testEval.eval.JobID {
+					blockedEvals[e.ID] = false
+				}
+			}
+
+			blockedEvals[testEval.eval.ID] = true
+		} else {
+			i := int(unblockIdx) % len(evalHistory)
+			eval := evalHistory[i]
+			counts := 0
+
+			// Find and unlock all evals for this job.
+			for _, e := range evalHistory {
+				if e.Namespace == eval.Namespace && e.JobID == eval.JobID {
+					counts++
+					blockedEvals[e.ID] = false
+				}
+			}
+
+		}
+
+		result := NewBlockedResourcesStats()
+		for _, e := range evalHistory {
+			if !blockedEvals[e.ID] {
+				continue
+			}
+			result = result.Add(generateResourceStats(e))
+		}
+		clearTimestampFromBlockedResourceStats(result)
+		return result
+	}
+
+	err := quick.CheckEqual(blockAndUntrack, manualCount, nil)
+	if err != nil {
+		t.Error(err)
+	}
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/blocked_evals_system.go

@@ -0,0 +1,97 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import "github.com/hashicorp/nomad/nomad/structs"
+
+// systemEvals are handled specially, each job may have a blocked eval on each node
+type systemEvals struct {
+	// byJob maps a jobID to a nodeID to that job's single blocked evalID on that node
+	byJob map[structs.NamespacedID]map[string]string
+
+	// byNode maps a nodeID to a set of evalIDs
+	byNode map[string]map[string]bool
+
+	// evals maps evalIDs to an eval and token
+	evals map[string]*wrappedEval
+}
+
+func newSystemEvals() *systemEvals {
+	return &systemEvals{
+		evals:  map[string]*wrappedEval{},
+		byJob:  map[structs.NamespacedID]map[string]string{},
+		byNode: map[string]map[string]bool{},
+	}
+}
+
+func (s *systemEvals) Add(eval *structs.Evaluation, token string) {
+	// store the eval by node id
+	if _, ok := s.byNode[eval.NodeID]; !ok {
+		s.byNode[eval.NodeID] = make(map[string]bool)
+	}
+
+	s.byNode[eval.NodeID][eval.ID] = true
+	s.evals[eval.ID] = &wrappedEval{eval: eval, token: token}
+
+	// link the job to the node for cleanup
+	jobID := structs.NewNamespacedID(eval.JobID, eval.Namespace)
+	if _, ok := s.byJob[jobID]; !ok {
+		s.byJob[jobID] = make(map[string]string)
+	}
+
+	// if we're displacing the old blocked id for this job+node, delete it first
+	if prevID, ok := s.byJob[jobID][eval.NodeID]; ok {
+		prev, _ := s.Get(prevID)
+		s.Remove(prev.eval)
+	}
+
+	// set this eval as the new eval for this job on this node
+	s.byJob[jobID][eval.NodeID] = eval.ID
+}
+
+func (s *systemEvals) Get(evalID string) (*wrappedEval, bool) {
+	w, ok := s.evals[evalID]
+	return w, ok
+}
+
+func (s *systemEvals) Remove(eval *structs.Evaluation) {
+	// delete the job index if this eval is the currently listed blocked eval
+	jobID := structs.NewNamespacedID(eval.JobID, eval.Namespace)
+	e, ok := s.byJob[jobID][eval.NodeID]
+	if ok && e == eval.ID {
+		delete(s.byJob[jobID], eval.NodeID)
+	}
+
+	// delete this eval from the node index, and then the map for this node if empty
+	delete(s.byNode[eval.NodeID], eval.ID)
+	if len(s.byNode[eval.NodeID]) == 0 {
+		delete(s.byNode, eval.NodeID)
+	}
+
+	// delete the eval itself
+	delete(s.evals, eval.ID)
+}
+
+func (s *systemEvals) NodeEvals(nodeID string) (map[*structs.Evaluation]string, bool) {
+	out := map[*structs.Evaluation]string{}
+	for eID := range s.byNode[nodeID] {
+		if w, ok := s.Get(eID); ok {
+			out[w.eval] = w.token
+		}
+	}
+
+	ok := len(out) > 0
+	return out, ok
+}
+
+func (s *systemEvals) JobEvals(jobID structs.NamespacedID) ([]*structs.Evaluation, bool) {
+	out := []*structs.Evaluation{}
+	_, ok := s.byJob[jobID]
+	for _, eID := range s.byJob[jobID] {
+		if e, ok := s.Get(eID); ok {
+			out = append(out, e.eval)
+		}
+	}
+	return out, ok
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/blocked_evals_test.go

@@ -0,0 +1,811 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/helper/testlog"
+	"github.com/hashicorp/nomad/nomad/mock"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/shoenig/test/must"
+	"github.com/shoenig/test/wait"
+)
+
+// Stats is used to query the state of the blocked eval tracker. This is only
+// ever called in tests.
+func (b *BlockedEvals) Stats() *BlockedStats {
+	return b.stats.Copy()
+}
+
+func testBlockedEvals(t *testing.T) (*BlockedEvals, *EvalBroker) {
+	broker := testBroker(t, 0)
+	broker.SetEnabled(true)
+	blocked := NewBlockedEvals(broker, testlog.HCLogger(t))
+	blocked.SetEnabled(true)
+	return blocked, broker
+}
+
+func TestBlockedEvals_Block_Disabled(t *testing.T) {
+	ci.Parallel(t)
+	blocked, _ := testBlockedEvals(t)
+	blocked.SetEnabled(false)
+
+	// Create an escaped eval and add it to the blocked tracker.
+	e := mock.BlockedEval()
+	e.EscapedComputedClass = true
+	<-blocked.Block(e)
+
+	// Verify block did nothing.
+	stats := blocked.Stats()
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 0, stats.BlockedResources.ByJob)
+}
+
+func TestBlockedEvals_Block_SameJob(t *testing.T) {
+	ci.Parallel(t)
+	blocked, _ := testBlockedEvals(t)
+
+	// Create two blocked evals and add them to the blocked tracker.
+	e := mock.BlockedEval()
+	e2 := mock.BlockedEval()
+	e2.JobID = e.JobID
+	<-blocked.Block(e)
+	<-blocked.Block(e2)
+
+	// Verify block didn't track duplicate.
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+}
+
+func TestBlockedEvals_Block_Quota(t *testing.T) {
+	ci.Parallel(t)
+
+	blocked, _ := testBlockedEvals(t)
+
+	// Create a blocked eval on quota.
+	e := mock.BlockedEval()
+	e.QuotaLimitReached = "foo"
+	<-blocked.Block(e)
+
+	// Verify block did track eval.
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.Eq(t, 1, stats.TotalQuotaLimit)
+}
+
+func TestBlockedEvals_Block_PriorUnblocks(t *testing.T) {
+	ci.Parallel(t)
+	blocked, _ := testBlockedEvals(t)
+
+	// Do unblocks prior to blocking.
+	blocked.Unblock("v1:123", 1000)
+	blocked.Unblock("v1:123", 1001)
+
+	// Create blocked eval with two classes ineligible.
+	e := mock.BlockedEval()
+	e.ClassEligibility = map[string]bool{"v1:123": false, "v1:456": false}
+	e.SnapshotIndex = 999
+	<-blocked.Block(e)
+
+	// Verify block did track eval.
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+}
+
+func TestBlockedEvals_GetDuplicates(t *testing.T) {
+	ci.Parallel(t)
+	blocked, _ := testBlockedEvals(t)
+
+	// Create duplicate blocked evals and add them to the blocked tracker.
+	e := mock.BlockedEval()
+	e.CreateIndex = 100
+	e2 := mock.BlockedEval()
+	e2.JobID = e.JobID
+	e2.CreateIndex = 101
+	e3 := mock.BlockedEval()
+	e3.JobID = e.JobID
+	e3.CreateIndex = 102
+	e4 := mock.BlockedEval()
+	e4.JobID = e.JobID
+	e4.CreateIndex = 100
+	<-blocked.Block(e)
+	<-blocked.Block(e2)
+
+	// Verify stats such that we are only tracking one.
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	// Get the duplicates.
+	out := blocked.GetDuplicates(0)
+	must.Len(t, 1, out)
+	must.Eq(t, e, out[0])
+
+	// Call block again after a small sleep.
+	go func() {
+		time.Sleep(500 * time.Millisecond)
+		<-blocked.Block(e3)
+	}()
+
+	// Get the duplicates.
+	out = blocked.GetDuplicates(1 * time.Second)
+	must.Len(t, 1, out)
+	must.Eq(t, e2, out[0])
+
+	// Verify stats such that we are only tracking one.
+	stats = blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	// Add an older evaluation and assert it gets cancelled.
+	<-blocked.Block(e4)
+	out = blocked.GetDuplicates(0)
+	must.Len(t, 1, out)
+	must.Eq(t, e4, out[0])
+
+	// Verify stats such that we are only tracking one.
+	stats = blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+}
+
+func TestBlockedEvals_UnblockEscaped(t *testing.T) {
+	ci.Parallel(t)
+
+	blocked, broker := testBlockedEvals(t)
+
+	// Create an escaped eval and add it to the blocked tracker.
+	e := mock.BlockedEval()
+	e.Status = structs.EvalStatusBlocked
+	e.EscapedComputedClass = true
+	<-blocked.Block(e)
+
+	// Verify block caused the eval to be tracked
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 1, stats.TotalEscaped)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	<-blocked.Unblock("v1:123", 1000)
+	requireBlockedEvalsEnqueued(t, blocked, broker, 1)
+}
+
+func requireBlockedEvalsEnqueued(t *testing.T, blocked *BlockedEvals, broker *EvalBroker, enqueued int) {
+	t.Helper()
+
+	must.Wait(t, wait.InitialSuccess(
+		wait.ErrorFunc(func() error {
+			brokerStats := broker.Stats()
+			if brokerStats.TotalReady != enqueued {
+				return fmt.Errorf("missing enqueued evals: %#v", brokerStats)
+			}
+			return nil
+		}),
+		wait.Timeout(500*time.Millisecond),
+		wait.Gap(10*time.Millisecond),
+	))
+
+	blocked.pruneStats(time.Now().UTC())
+
+	// verfiy stats have been cleared
+	stats := blocked.Stats()
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 0, stats.BlockedResources.ByJob)
+}
+
+func TestBlockedEvals_UnblockEligible(t *testing.T) {
+	ci.Parallel(t)
+	blocked, broker := testBlockedEvals(t)
+
+	// Create a blocked eval that is eligible on a specific node class and add
+	// it to the blocked tracker.
+	e := mock.BlockedEval()
+	e.Status = structs.EvalStatusBlocked
+	e.ClassEligibility = map[string]bool{"v1:123": true}
+	<-blocked.Block(e)
+
+	// Verify block caused the eval to be tracked
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+
+	blocked.Unblock("v1:123", 1000)
+	requireBlockedEvalsEnqueued(t, blocked, broker, 1)
+}
+
+func TestBlockedEvals_UnblockIneligible(t *testing.T) {
+	ci.Parallel(t)
+	blocked, broker := testBlockedEvals(t)
+
+	// Create a blocked eval that is ineligible on a specific node class and add
+	// it to the blocked tracker.
+	e := mock.BlockedEval()
+	e.ClassEligibility = map[string]bool{"v1:123": false}
+	<-blocked.Block(e)
+
+	// Verify block caused the eval to be tracked
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	// Should do nothing
+	<-blocked.Unblock("v1:123", 1000)
+
+	// Verify Unblock didn't cause an enqueue
+	brokerStats := broker.Stats()
+	must.Eq(t, 0, brokerStats.TotalReady, must.Sprintf("eval unblocked: %#v", brokerStats))
+
+	// Prune old and empty metrics.
+	blocked.pruneStats(time.Now().UTC())
+
+	stats = blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+}
+
+func TestBlockedEvals_UnblockUnknown(t *testing.T) {
+	ci.Parallel(t)
+	blocked, broker := testBlockedEvals(t)
+
+	// Create a blocked eval that is ineligible on a specific node class and add
+	// it to the blocked tracker.
+	e := mock.BlockedEval()
+	e.ClassEligibility = map[string]bool{"v1:123": true, "v1:456": false}
+	<-blocked.Block(e)
+
+	// Verify block caused the eval to be tracked.
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	// Should unblock because the eval hasn't seen this node class.
+	<-blocked.Unblock("v1:789", 1000)
+	requireBlockedEvalsEnqueued(t, blocked, broker, 1)
+}
+
+func TestBlockedEvals_UnblockEligible_Quota(t *testing.T) {
+	ci.Parallel(t)
+
+	blocked, broker := testBlockedEvals(t)
+
+	// Create a blocked eval that is eligible for a particular quota.
+	e := mock.BlockedEval()
+	e.QuotaLimitReached = "foo"
+	<-blocked.Block(e)
+
+	// Verify block caused the eval to be tracked.
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 1, stats.TotalQuotaLimit)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	<-blocked.UnblockQuota("foo", 1000)
+	requireBlockedEvalsEnqueued(t, blocked, broker, 1)
+}
+
+// The quota here is incidental. The eval is blocked due to something else,
+// e.g. cpu exhausted, but there happens to also be a quota on the namespace.
+func TestBlockedEvals_UnblockEligible_IncidentalQuota(t *testing.T) {
+	ci.Parallel(t)
+	blocked, broker := testBlockedEvals(t)
+
+	e := mock.BlockedEval()
+	e.Status = structs.EvalStatusBlocked
+	e.QuotaLimitReached = "" // explicitly not blocked due to quota limit
+	<-blocked.Block(e)
+
+	// Verify block caused the eval to be tracked.
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+	// but not due to quota.
+	must.Eq(t, 0, stats.TotalQuotaLimit)
+
+	// When unblocking, the quota name from the alloc is passed in,
+	// regardless of the cause of the initial blockage.
+	// Since the initial block in this test was due to something else,
+	// it should be unblocked without regard to quota.
+	<-blocked.UnblockQuota("foo", 1000)
+	requireBlockedEvalsEnqueued(t, blocked, broker, 1)
+}
+
+func TestBlockedEvals_UnblockIneligible_Quota(t *testing.T) {
+	ci.Parallel(t)
+	blocked, broker := testBlockedEvals(t)
+
+	// Create a blocked eval that is eligible on a specific quota.
+	e := mock.BlockedEval()
+	e.QuotaLimitReached = "foo"
+	<-blocked.Block(e)
+
+	// Verify block caused the eval to be tracked.
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 1, stats.TotalQuotaLimit)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	// Should do nothing and have no evals enqueued
+	<-blocked.UnblockQuota("bar", 1000)
+	brokerStats := broker.Stats()
+	must.Eq(t, 0, brokerStats.TotalReady)
+
+	// Prune old and empty metrics.
+	blocked.pruneStats(time.Now().UTC())
+
+	stats = blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.Eq(t, 1, stats.TotalQuotaLimit)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+}
+
+func TestBlockedEvals_Reblock(t *testing.T) {
+	ci.Parallel(t)
+	blocked, broker := testBlockedEvals(t)
+
+	// Create an evaluation, Enqueue/Dequeue it to get a token
+	e := mock.BlockedEval()
+	e.SnapshotIndex = 500
+	e.ClassEligibility = map[string]bool{"v1:123": true, "v1:456": false}
+	broker.Enqueue(e)
+
+	_, token, err := broker.Dequeue([]string{e.Type}, time.Second)
+	must.NoError(t, err)
+
+	// Reblock the evaluation
+	<-blocked.Reblock(e, token)
+
+	// Verify block caused the eval to be tracked
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	// Should unblock because the eval
+	<-blocked.Unblock("v1:123", 1000)
+
+	brokerStats := broker.Stats()
+	must.Eq(t, 0, brokerStats.TotalReady)
+	must.Eq(t, 1, brokerStats.TotalUnacked)
+
+	// Ack the evaluation which should cause the reblocked eval to transition
+	// to ready
+	err = broker.Ack(e.ID, token)
+	must.NoError(t, err)
+
+	requireBlockedEvalsEnqueued(t, blocked, broker, 1)
+}
+
+// Test the block case in which the eval should be immediately unblocked since
+// it is escaped and old
+func TestBlockedEvals_Block_ImmediateUnblock_Escaped(t *testing.T) {
+	ci.Parallel(t)
+	blocked, broker := testBlockedEvals(t)
+
+	// Do an unblock prior to blocking
+	<-blocked.Unblock("v1:123", 1000)
+
+	// Create a blocked eval that is eligible on a specific node class and add
+	// it to the blocked tracker.
+	e := mock.BlockedEval()
+	e.EscapedComputedClass = true
+	e.SnapshotIndex = 900
+	<-blocked.Block(e)
+
+	// Verify block caused the eval to be immediately unblocked
+	stats := blocked.Stats()
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 0, stats.BlockedResources.ByJob)
+
+	requireBlockedEvalsEnqueued(t, blocked, broker, 1)
+}
+
+// Test the block case in which the eval should be immediately unblocked since
+// there is an unblock on an unseen class that occurred while it was in the
+// scheduler
+func TestBlockedEvals_Block_ImmediateUnblock_UnseenClass_After(t *testing.T) {
+	ci.Parallel(t)
+
+	blocked, broker := testBlockedEvals(t)
+
+	// Do an unblock prior to blocking
+	<-blocked.Unblock("v1:123", 1000)
+
+	// Create a blocked eval that is eligible on a specific node class and add
+	// it to the blocked tracker.
+	e := mock.BlockedEval()
+	e.EscapedComputedClass = false
+	e.SnapshotIndex = 900
+	<-blocked.Block(e)
+
+	// Verify block caused the eval to be immediately unblocked
+	stats := blocked.Stats()
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 0, stats.BlockedResources.ByJob)
+
+	requireBlockedEvalsEnqueued(t, blocked, broker, 1)
+}
+
+// Test the block case in which the eval should not immediately unblock since
+// there is an unblock on an unseen class that occurred before it was in the
+// scheduler
+func TestBlockedEvals_Block_ImmediateUnblock_UnseenClass_Before(t *testing.T) {
+	ci.Parallel(t)
+	blocked, _ := testBlockedEvals(t)
+
+	// Do an unblock prior to blocking
+	<-blocked.Unblock("v1:123", 500)
+
+	// Create a blocked eval that is eligible on a specific node class and add
+	// it to the blocked tracker.
+	e := mock.BlockedEval()
+	e.EscapedComputedClass = false
+	e.SnapshotIndex = 900
+	<-blocked.Block(e)
+
+	// Verify block caused the eval to be immediately unblocked
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+}
+
+// Test the block case in which the eval should be immediately unblocked since
+// it a class it is eligible for has been unblocked
+func TestBlockedEvals_Block_ImmediateUnblock_SeenClass(t *testing.T) {
+	ci.Parallel(t)
+	blocked, broker := testBlockedEvals(t)
+
+	// Do an unblock prior to blocking
+	<-blocked.Unblock("v1:123", 1000)
+
+	// Create a blocked eval that is eligible on a specific node class and add
+	// it to the blocked tracker.
+	e := mock.BlockedEval()
+	e.ClassEligibility = map[string]bool{"v1:123": true, "v1:456": false}
+	e.SnapshotIndex = 900
+	<-blocked.Block(e)
+
+	// Verify block caused the eval to be immediately unblocked
+	stats := blocked.Stats()
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 0, stats.BlockedResources.ByJob)
+
+	requireBlockedEvalsEnqueued(t, blocked, broker, 1)
+}
+
+// Test the block case in which the eval should be immediately unblocked since
+// it a quota has changed that it is using
+func TestBlockedEvals_Block_ImmediateUnblock_Quota(t *testing.T) {
+	ci.Parallel(t)
+	blocked, broker := testBlockedEvals(t)
+
+	// Do an unblock prior to blocking
+	<-blocked.UnblockQuota("my-quota", 1000)
+
+	// Create a blocked eval that is eligible on a specific node class and add
+	// it to the blocked tracker.
+	e := mock.BlockedEval()
+	e.QuotaLimitReached = "my-quota"
+	e.SnapshotIndex = 900
+	<-blocked.Block(e)
+
+	// Verify block caused the eval to be immediately unblocked
+	stats := blocked.Stats()
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.Eq(t, 0, stats.TotalQuotaLimit)
+	must.MapLen(t, 0, stats.BlockedResources.ByJob)
+
+	requireBlockedEvalsEnqueued(t, blocked, broker, 1)
+}
+
+func TestBlockedEvals_UnblockFailed(t *testing.T) {
+	ci.Parallel(t)
+
+	blocked, broker := testBlockedEvals(t)
+
+	// Create blocked evals that are due to failures
+	e := mock.BlockedEval()
+	e.TriggeredBy = structs.EvalTriggerMaxPlans
+	e.EscapedComputedClass = true
+	<-blocked.Block(e)
+
+	e2 := mock.BlockedEval()
+	e2.Status = structs.EvalStatusBlocked
+	e2.TriggeredBy = structs.EvalTriggerMaxPlans
+	e2.ClassEligibility = map[string]bool{"v1:123": true, "v1:456": false}
+	<-blocked.Block(e2)
+
+	e3 := mock.BlockedEval()
+	e3.TriggeredBy = structs.EvalTriggerMaxPlans
+	e3.QuotaLimitReached = "foo"
+	<-blocked.Block(e3)
+
+	// Trigger an unblock fail
+	blocked.UnblockFailed()
+
+	// Prune old and empty metrics.
+	blocked.pruneStats(time.Now().UTC())
+
+	// Verify UnblockFailed caused the eval to be immediately unblocked
+	stats := blocked.Stats()
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.Eq(t, 0, stats.TotalQuotaLimit)
+	must.MapLen(t, 0, stats.BlockedResources.ByJob)
+
+	requireBlockedEvalsEnqueued(t, blocked, broker, 3)
+
+	// Reblock an eval for the same job and check that it gets tracked.
+	<-blocked.Block(e)
+	stats = blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 1, stats.TotalEscaped)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+}
+
+func TestBlockedEvals_Untrack(t *testing.T) {
+	ci.Parallel(t)
+	blocked, _ := testBlockedEvals(t)
+
+	// Create blocked eval and add to the blocked tracker.
+	e := mock.BlockedEval()
+	e.ClassEligibility = map[string]bool{"v1:123": false, "v1:456": false}
+	e.SnapshotIndex = 1000
+	<-blocked.Block(e)
+
+	// Verify block did track
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	// Untrack and verify
+	<-blocked.Untrack(e.JobID, e.Namespace)
+	blocked.pruneStats(time.Now().UTC())
+
+	stats = blocked.Stats()
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.MapLen(t, 0, stats.BlockedResources.ByJob)
+}
+
+func TestBlockedEvals_Untrack_Quota(t *testing.T) {
+	ci.Parallel(t)
+	blocked, _ := testBlockedEvals(t)
+
+	// Create a blocked eval and add it to the blocked tracker.
+	e := mock.BlockedEval()
+	e.QuotaLimitReached = "foo"
+	e.SnapshotIndex = 1000
+	<-blocked.Block(e)
+
+	// Verify block did track
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.Eq(t, 1, stats.TotalQuotaLimit)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	// Untrack and verify
+	<-blocked.Untrack(e.JobID, e.Namespace)
+	stats = blocked.Stats()
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.Eq(t, 0, stats.TotalQuotaLimit)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	now := time.Now().UTC()
+	blocked.pruneStats(now)
+	stats = blocked.Stats()
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.Eq(t, 0, stats.TotalQuotaLimit)
+	must.MapLen(t, 0, stats.BlockedResources.ByJob)
+}
+
+func TestBlockedEvals_UnblockNode(t *testing.T) {
+	ci.Parallel(t)
+
+	blocked, broker := testBlockedEvals(t)
+
+	// Create a blocked evals and add it to the blocked tracker.
+	e := mock.BlockedEval()
+	e.Type = structs.JobTypeSystem
+	e.NodeID = "foo"
+	e.SnapshotIndex = 999
+	<-blocked.Block(e)
+
+	// Verify block did track
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	<-blocked.UnblockNode("foo")
+	requireBlockedEvalsEnqueued(t, blocked, broker, 1)
+
+	blocked.pruneStats(time.Now().UTC())
+	stats = blocked.Stats()
+	must.MapEmpty(t, blocked.system.byNode)
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.MapLen(t, 0, stats.BlockedResources.ByJob)
+}
+
+func TestBlockedEvals_SystemUntrack(t *testing.T) {
+	ci.Parallel(t)
+	blocked, _ := testBlockedEvals(t)
+
+	// Create a blocked evals and add it to the blocked tracker.
+	e := mock.Eval()
+	e.Type = structs.JobTypeSystem
+	e.NodeID = "foo"
+	<-blocked.Block(e)
+
+	// Verify block did track
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.Eq(t, 0, stats.TotalQuotaLimit)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	// Untrack and verify
+	<-blocked.Untrack(e.JobID, e.Namespace)
+	blocked.pruneStats(time.Now().UTC())
+
+	stats = blocked.Stats()
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.Eq(t, 0, stats.TotalQuotaLimit)
+	must.MapLen(t, 0, stats.BlockedResources.ByJob)
+}
+
+func TestBlockedEvals_SystemDisableFlush(t *testing.T) {
+	ci.Parallel(t)
+	blocked, _ := testBlockedEvals(t)
+
+	// Create a blocked evals and add it to the blocked tracker.
+	e := mock.Eval()
+	e.Type = structs.JobTypeSystem
+	e.NodeID = "foo"
+	<-blocked.Block(e)
+
+	// Verify block did track
+	stats := blocked.Stats()
+	must.Eq(t, 1, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.Eq(t, 0, stats.TotalQuotaLimit)
+	must.MapLen(t, 1, stats.BlockedResources.ByJob)
+
+	// Disable empties
+	blocked.SetEnabled(false)
+	stats = blocked.Stats()
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+	must.Eq(t, 0, stats.TotalQuotaLimit)
+	must.MapLen(t, 0, stats.BlockedResources.ByJob)
+	must.MapEmpty(t, blocked.system.evals)
+	must.MapEmpty(t, blocked.system.byJob)
+	must.MapEmpty(t, blocked.system.byNode)
+}
+
+// TestBlockedEvals_UnblockBackpressure verifies that calling Unblock methods on
+// BlockedEvals doesn't result in backpressure that could impact the FSM.
+func TestBlockedEvals_UnblockBackpressure(t *testing.T) {
+	pctx, cancel := context.WithCancel(context.TODO())
+	t.Cleanup(cancel)
+
+	broker, err := NewEvalBroker(pctx, time.Second, 1, 1, 1)
+	must.NoError(t, err)
+	broker.SetEnabled(true)
+	blockedEvals := NewBlockedEvals(broker, testlog.HCLogger(t))
+	blockedEvals.SetEnabled(true)
+
+	for range 10000 {
+		eval := mock.Eval()
+		eval.QueuedAllocations = map[string]int{"web": 1}
+		eval.ClassEligibility = map[string]bool{"foo": false}
+		eval.EscapedComputedClass = true
+		eval.Status = structs.EvalStatusBlocked
+
+		<-blockedEvals.Block(eval)
+	}
+	stats := blockedEvals.Stats()
+	must.Eq(t, 10000, stats.TotalBlocked)
+	must.Eq(t, 10000, stats.TotalEscaped)
+
+	stopCtx, stopCancel := context.WithCancel(pctx)
+	t.Cleanup(stopCancel)
+
+	var wg sync.WaitGroup
+	errors := make(chan error, 144)
+
+	// this loop continuously calls unblock to trigger any contention we might
+	// have; these calls should return quickly but be no-ops in watchCapacity
+	// once we've unblocked all 10000 blocked evals
+	//
+	// Note that this test isn't particularly useful in CI where we have low
+	// GOMAXPROCS. And with the fix it's intended to exercise, these Unblock
+	// calls return too quickly for even highly concurrent continuous Reblock
+	// loops to cause meaningful contention.
+	for i := range 144 {
+		wg.Go(func() {
+			index := uint64(1000 * i)
+			for {
+				ctx, cancel := context.WithTimeout(stopCtx, 100*time.Millisecond)
+				defer cancel()
+				errCh := make(chan error)
+				index++
+				go func() {
+					blockedEvals.Unblock("foo", index)
+					close(errCh)
+				}()
+
+				select {
+				case <-ctx.Done():
+					if ctx.Err() == context.DeadlineExceeded {
+						errors <- fmt.Errorf("Unblock took more than 100ms to complete")
+					}
+					return
+				case <-errCh:
+					cancel()
+					continue
+				}
+			}
+
+		})
+	}
+
+	// wait until all 10000 evals are unblocked or 5 seconds pass, whichever is
+	// longer, and collect any errors from goroutines that would indicate
+	// they've blocked for longer than expected
+	timeout := time.After(time.Second * 5)
+	poll := time.NewTicker(10 * time.Millisecond)
+	errCount := 0
+	var lastErr error
+
+DONE:
+	for {
+		select {
+		case <-timeout:
+			break DONE
+		case <-poll.C:
+			stats = blockedEvals.Stats()
+			if stats.TotalBlocked == 0 && stats.TotalEscaped == 0 {
+				break DONE
+			}
+		case err := <-errors:
+			if err != nil {
+				lastErr = err
+				errCount++
+			}
+		}
+	}
+
+	must.NoError(t, lastErr, must.Sprintf("got %d errors", errCount))
+	stats = blockedEvals.Stats()
+	must.Eq(t, 0, stats.TotalBlocked)
+	must.Eq(t, 0, stats.TotalEscaped)
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/client_agent_endpoint.go

@@ -0,0 +1,610 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"time"
+
+	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-msgpack/v2/codec"
+	sframer "github.com/hashicorp/nomad/client/lib/streamframer"
+	cstructs "github.com/hashicorp/nomad/client/structs"
+	"github.com/hashicorp/nomad/command/agent/host"
+	"github.com/hashicorp/nomad/command/agent/monitor"
+	"github.com/hashicorp/nomad/command/agent/pprof"
+	"github.com/hashicorp/nomad/helper/pointer"
+	"github.com/hashicorp/nomad/nomad/peers"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+type Agent struct {
+	srv *Server
+}
+
+func NewAgentEndpoint(srv *Server) *Agent {
+	return &Agent{srv: srv}
+}
+
+func (a *Agent) register() {
+	a.srv.streamingRpcs.Register("Agent.Monitor", a.monitor)
+	a.srv.streamingRpcs.Register("Agent.MonitorExport", a.monitorExport)
+}
+
+func (a *Agent) Profile(args *structs.AgentPprofRequest, reply *structs.AgentPprofResponse) error {
+	authErr := a.srv.Authenticate(nil, args)
+	a.srv.MeasureRPCRate("agent", structs.RateMetricRead, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+
+	// Check ACL for agent write
+	aclObj, err := a.srv.ResolveACL(args)
+	if err != nil {
+		return err
+	} else if !aclObj.AllowAgentWrite() {
+		// we're not checking AllowAgentDebug here because the target might not
+		// be this server, and the server doesn't know if enable_debug has been
+		// set on the target
+		return structs.ErrPermissionDenied
+	}
+
+	// Forward to different region if necessary
+	// this would typically be done in a.srv.forward() but since
+	// we are targeting a specific server, not just the leader
+	// we must manually handle region forwarding here.
+	region := args.RequestRegion()
+	if region == "" {
+		return fmt.Errorf("missing target RPC")
+	}
+
+	if region != a.srv.Region() {
+		// Mark that we are forwarding
+		args.SetForwarded()
+		return a.srv.forwardRegion(region, "Agent.Profile", args, reply)
+	}
+
+	// Targeting a node, forward request to node
+	if args.NodeID != "" {
+		return a.forwardProfileClient(args, reply)
+	}
+
+	// Handle serverID not equal to ours
+	if args.ServerID != "" {
+		serverToFwd, err := a.forwardFor(args.ServerID, region)
+		if err != nil {
+			return err
+		}
+		if serverToFwd != nil {
+			return a.srv.forwardServer(serverToFwd, "Agent.Profile", args, reply)
+		}
+	}
+
+	// This server is the target, so now we can check for AllowAgentDebug
+	if !aclObj.AllowAgentDebug(a.srv.GetConfig().EnableDebug) {
+		return structs.ErrPermissionDenied
+	}
+
+	// Process the request on this server
+	var resp []byte
+	var headers map[string]string
+
+	// Determine which profile to run and generate profile.
+	// Blocks for args.Seconds
+	// Our RPC endpoints currently don't support context
+	// or request cancellation so using server shutdownCtx as a
+	// best effort.
+	switch args.ReqType {
+	case pprof.CPUReq:
+		resp, headers, err = pprof.CPUProfile(a.srv.shutdownCtx, args.Seconds)
+	case pprof.CmdReq:
+		resp, headers, err = pprof.Cmdline()
+	case pprof.LookupReq:
+		resp, headers, err = pprof.Profile(args.Profile, args.Debug, args.GC)
+	case pprof.TraceReq:
+		resp, headers, err = pprof.Trace(a.srv.shutdownCtx, args.Seconds)
+	default:
+		err = structs.NewErrRPCCoded(404, "Unknown profile request type")
+	}
+
+	if err != nil {
+		if pprof.IsErrProfileNotFound(err) {
+			return structs.NewErrRPCCoded(404, err.Error())
+		}
+		return structs.NewErrRPCCoded(500, err.Error())
+	}
+
+	// Copy profile response to reply
+	reply.Payload = resp
+	reply.HTTPHeaders = headers
+	reply.AgentID = a.srv.serf.LocalMember().Name
+
+	return nil
+}
+
+func (a *Agent) monitor(conn io.ReadWriteCloser) {
+	defer conn.Close()
+
+	// Decode args
+	var args cstructs.MonitorRequest
+	decoder := codec.NewDecoder(conn, structs.MsgpackHandle)
+	encoder := codec.NewEncoder(conn, structs.MsgpackHandle)
+
+	if err := decoder.Decode(&args); err != nil {
+		handleStreamResultError(err, pointer.Of(int64(500)), encoder)
+		return
+	}
+	authErr := a.srv.Authenticate(nil, &args)
+	a.srv.MeasureRPCRate("agent", structs.RateMetricRead, &args)
+	if authErr != nil {
+		handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
+		return
+	}
+
+	// Check agent read permissions
+	if aclObj, err := a.srv.ResolveACL(&args); err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	} else if !aclObj.AllowAgentRead() {
+		handleStreamResultError(structs.ErrPermissionDenied, pointer.Of(int64(403)), encoder)
+		return
+	}
+
+	logLevel := log.LevelFromString(args.LogLevel)
+	if args.LogLevel == "" {
+		logLevel = log.LevelFromString("INFO")
+	}
+
+	if logLevel == log.NoLevel {
+		handleStreamResultError(errors.New("Unknown log level"), pointer.Of(int64(400)), encoder)
+		return
+	}
+
+	// Targeting a node, forward request to node
+	if args.NodeID != "" {
+		a.forwardMonitorClient(conn, args, encoder, decoder, args.NodeID, "Agent.Monitor")
+		// forwarded request has ended, return
+		return
+	}
+
+	region := args.RequestRegion()
+	if region == "" {
+		handleStreamResultError(fmt.Errorf("missing target region"), pointer.Of(int64(400)), encoder)
+		return
+	}
+	if region != a.srv.Region() {
+		// Mark that we are forwarding
+		args.SetForwarded()
+	}
+
+	// Try to forward request to remote region/server
+	if args.ServerID != "" {
+		serverToFwd, err := a.forwardFor(args.ServerID, region)
+		if err != nil {
+			handleStreamResultError(err, pointer.Of(int64(400)), encoder)
+			return
+		}
+		if serverToFwd != nil {
+			// Empty ServerID to prevent forwarding loop
+			args.ServerID = ""
+			a.forwardMonitorServer(conn, serverToFwd, args, encoder, decoder, "Agent.Monitor")
+			return
+		}
+	}
+
+	// NodeID was empty, ServerID was equal to this server,  monitor this server
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	m := monitor.New(512, a.srv.logger, &log.LoggerOptions{
+		Level:           logLevel,
+		JSONFormat:      args.LogJSON,
+		IncludeLocation: args.LogIncludeLocation,
+	})
+
+	frames := make(chan *sframer.StreamFrame, 32)
+	errCh := make(chan error)
+	var buf bytes.Buffer
+	frameCodec := codec.NewEncoder(&buf, structs.JsonHandle)
+
+	framer := sframer.NewStreamFramer(frames, 1*time.Second, 200*time.Millisecond, 1024)
+	framer.Run()
+	defer framer.Destroy()
+
+	// goroutine to detect remote side closing
+	go func() {
+		if _, err := conn.Read(nil); err != nil {
+			// One end of the pipe explicitly closed, exit
+			cancel()
+			return
+		}
+		<-ctx.Done()
+	}()
+
+	logCh := m.Start()
+	defer m.Stop()
+	initialOffset := int64(0)
+
+	// receive logs and build frames
+	go func() {
+		defer framer.Destroy()
+	LOOP:
+		for {
+			select {
+			case log := <-logCh:
+				if err := framer.Send("", "log", log, initialOffset); err != nil {
+					select {
+					case errCh <- err:
+					case <-ctx.Done():
+					}
+					break LOOP
+				}
+			case <-ctx.Done():
+				break LOOP
+			}
+		}
+	}()
+	streamEncoder := monitor.NewStreamEncoder(&buf, conn, encoder, frameCodec, args.PlainText)
+	streamErr := streamEncoder.EncodeStream(frames, errCh, ctx, framer, false)
+	if streamErr != nil {
+		handleStreamResultError(streamErr, pointer.Of(int64(500)), encoder)
+		return
+	}
+}
+
+func (a *Agent) monitorExport(conn io.ReadWriteCloser) {
+	defer conn.Close()
+	// Decode args
+	var args cstructs.MonitorExportRequest
+	decoder := codec.NewDecoder(conn, structs.MsgpackHandle)
+	encoder := codec.NewEncoder(conn, structs.MsgpackHandle)
+
+	if err := decoder.Decode(&args); err != nil {
+		handleStreamResultError(err, pointer.Of(int64(500)), encoder)
+		return
+	}
+	authErr := a.srv.Authenticate(nil, &args)
+	a.srv.MeasureRPCRate("agent", structs.RateMetricRead, &args)
+	if authErr != nil {
+		handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
+		return
+	}
+
+	// Check agent read permissions
+	if aclObj, err := a.srv.ResolveACL(&args); err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	} else if !aclObj.AllowAgentRead() {
+		handleStreamResultError(structs.ErrPermissionDenied, pointer.Of(int64(403)), encoder)
+		return
+	}
+
+	// Targeting a node, forward request to node
+	if args.NodeID != "" {
+		a.forwardMonitorClient(conn, args, encoder, decoder, args.NodeID, "Agent.MonitorExport")
+		// forwarded request has ended, return
+		return
+	}
+
+	region := args.RequestRegion()
+	if region == "" {
+		handleStreamResultError(fmt.Errorf("missing target region"), pointer.Of(int64(400)), encoder)
+		return
+	}
+	if region != a.srv.Region() {
+		// Mark that we are forwarding
+		args.SetForwarded()
+	}
+
+	// Try to forward request to remote region/server
+	if args.ServerID != "" {
+		serverToFwd, err := a.forwardFor(args.ServerID, region)
+		if err != nil {
+			handleStreamResultError(err, pointer.Of(int64(400)), encoder)
+			return
+		}
+		if serverToFwd != nil {
+			//empty args.ServerID to prevent forwarding loop
+			args.ServerID = ""
+			a.forwardMonitorServer(conn, serverToFwd, args, encoder, decoder, "Agent.MonitorExport")
+			return
+		}
+	}
+
+	nomadLogPath := a.srv.GetConfig().LogFile
+	if args.OnDisk && nomadLogPath == "" {
+		handleStreamResultError(errors.New("No nomad log file defined"), pointer.Of(int64(400)), encoder)
+	}
+	// NodeID was empty, ServerID was equal to this server,  monitor this server
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	opts := monitor.MonitorExportOpts{
+		Logger:       a.srv.logger,
+		LogsSince:    args.LogsSince,
+		ServiceName:  args.ServiceName,
+		NomadLogPath: nomadLogPath,
+		OnDisk:       args.OnDisk,
+		Follow:       args.Follow,
+		Context:      ctx,
+	}
+
+	frames := make(chan *sframer.StreamFrame, 32)
+	errCh := make(chan error)
+	var buf bytes.Buffer
+	frameSize := 1024
+	frameCodec := codec.NewEncoder(&buf, structs.JsonHandle)
+
+	framer := sframer.NewStreamFramer(frames, 1*time.Second, 200*time.Millisecond, frameSize)
+	framer.Run()
+	defer framer.Destroy()
+
+	// goroutine to detect remote side closing
+	go func() {
+		if _, err := conn.Read(nil); err != nil {
+			// One end of the pipe explicitly closed, exit
+			cancel()
+			return
+		}
+		<-ctx.Done()
+	}()
+	m, err := monitor.NewExportMonitor(opts)
+	if err != nil {
+		handleStreamResultError(err, pointer.Of(int64(500)), encoder)
+		return
+	}
+
+	var eofCancelCh chan error
+
+	streamCh := m.Start()
+	initialOffset := int64(0)
+	eofCancel := !opts.Follow
+
+	streamEncoder := monitor.NewStreamEncoder(&buf, conn, encoder, frameCodec, args.PlainText)
+	// receive logs and build frames
+	streamReader := monitor.NewStreamReader(streamCh, framer, int64(frameSize))
+	go func() {
+		defer framer.Destroy()
+		if err := streamReader.StreamFixed(ctx, initialOffset, "", 0, eofCancelCh, eofCancel); err != nil {
+			select {
+			case errCh <- err:
+			case <-ctx.Done():
+			}
+		}
+	}()
+
+	streamErr := streamEncoder.EncodeStream(frames, errCh, ctx, framer, true)
+	if streamErr != nil {
+		handleStreamResultError(streamErr, pointer.Of(int64(500)), encoder)
+		return
+	}
+}
+
+// forwardFor returns a serverParts for a request to be forwarded to.
+// A response of nil, nil indicates that the current server is equal to the
+// serverID and region so the request should not be forwarded.
+func (a *Agent) forwardFor(serverID, region string) (*peers.Parts, error) {
+	var target *peers.Parts
+	var err error
+
+	if serverID == "leader" {
+		isLeader, remoteLeader := a.srv.getLeader()
+		if !isLeader && remoteLeader != nil {
+			target = remoteLeader
+		} else if !isLeader && remoteLeader == nil {
+			return nil, structs.ErrNoLeader
+		} else if isLeader {
+			// This server is current leader do not forward
+			return nil, nil
+		}
+	} else {
+		target, err = a.srv.getServer(region, serverID)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Unable to find a server
+	if target == nil {
+		return nil, fmt.Errorf("unknown nomad server %s", serverID)
+	}
+
+	// ServerID is this current server,
+	// No need to forward request
+	if target.Name == a.srv.LocalMember().Name {
+		return nil, nil
+	}
+
+	return target, nil
+}
+
+func (a *Agent) forwardMonitorClient(conn io.ReadWriteCloser, args any, encoder *codec.Encoder, decoder *codec.Decoder, nodeID string, endpoint string) {
+	// Get the Connection to the client either by fowarding to another server
+	// or creating direct stream
+	state, srv, err := a.findClientConn(nodeID)
+	if err != nil {
+		handleStreamResultError(err, pointer.Of(int64(500)), encoder)
+		return
+	}
+
+	var clientConn net.Conn
+
+	if state == nil {
+		conn, err := a.srv.streamingRpc(srv, endpoint)
+		if err != nil {
+			handleStreamResultError(err, nil, encoder)
+			return
+		}
+
+		clientConn = conn
+	} else {
+		stream, err := NodeStreamingRpc(state.Session, endpoint)
+		if err != nil {
+			handleStreamResultError(err, nil, encoder)
+			return
+		}
+		clientConn = stream
+	}
+	defer clientConn.Close()
+
+	// Send the Request
+	outEncoder := codec.NewEncoder(clientConn, structs.MsgpackHandle)
+	if err := outEncoder.Encode(args); err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	}
+
+	structs.Bridge(conn, clientConn)
+}
+
+func (a *Agent) forwardMonitorServer(conn io.ReadWriteCloser, server *peers.Parts, args any, encoder *codec.Encoder, decoder *codec.Decoder, endpoint string) {
+	serverConn, err := a.srv.streamingRpc(server, "Agent.Monitor")
+	if err != nil {
+		handleStreamResultError(err, pointer.Of(int64(500)), encoder)
+		return
+	}
+	defer serverConn.Close()
+
+	// Send the Request
+	outEncoder := codec.NewEncoder(serverConn, structs.MsgpackHandle)
+	if err := outEncoder.Encode(args); err != nil {
+		handleStreamResultError(err, pointer.Of(int64(500)), encoder)
+		return
+	}
+
+	structs.Bridge(conn, serverConn)
+}
+
+func (a *Agent) forwardProfileClient(args *structs.AgentPprofRequest, reply *structs.AgentPprofResponse) error {
+	state, srv, err := a.findClientConn(args.NodeID)
+
+	if err != nil {
+		return err
+	}
+
+	if srv != nil {
+		return a.srv.forwardServer(srv, "Agent.Profile", args, reply)
+	}
+
+	// NodeRpc
+	rpcErr := NodeRpc(state.Session, "Agent.Profile", args, reply)
+	if rpcErr != nil {
+		return rpcErr
+	}
+
+	return nil
+}
+
+// Host returns data about the agent's host system for the `debug` command.
+func (a *Agent) Host(args *structs.HostDataRequest, reply *structs.HostDataResponse) error {
+	authErr := a.srv.Authenticate(nil, args)
+	a.srv.MeasureRPCRate("agent", structs.RateMetricRead, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	aclObj, err := a.srv.ResolveACL(args)
+	if err != nil {
+		return err
+	}
+	if !aclObj.AllowAgentDebug(a.srv.GetConfig().EnableDebug) {
+		return structs.ErrPermissionDenied
+	}
+
+	// Forward to different region if necessary
+	// this would typically be done in a.srv.forward() but since
+	// we are targeting a specific server, not just the leader
+	// we must manually handle region forwarding here.
+	region := args.RequestRegion()
+	if region == "" {
+		return fmt.Errorf("missing target RPC")
+	}
+
+	if region != a.srv.Region() {
+		// Mark that we are forwarding
+		args.SetForwarded()
+		return a.srv.forwardRegion(region, "Agent.Host", args, reply)
+	}
+
+	// Targeting a client node, forward request to node
+	if args.NodeID != "" {
+		client, srv, err := a.findClientConn(args.NodeID)
+
+		if err != nil {
+			return err
+		}
+
+		if srv != nil {
+			return a.srv.forwardServer(srv, "Agent.Host", args, reply)
+		}
+
+		return NodeRpc(client.Session, "Agent.Host", args, reply)
+	}
+
+	// Handle serverID not equal to ours
+	if args.ServerID != "" {
+		srv, err := a.forwardFor(args.ServerID, region)
+		if err != nil {
+			return err
+		}
+		if srv != nil {
+			return a.srv.forwardServer(srv, "Agent.Host", args, reply)
+		}
+	}
+
+	data, err := host.MakeHostData()
+	if err != nil {
+		return err
+	}
+
+	reply.AgentID = a.srv.serf.LocalMember().Name
+	reply.HostData = data
+	return nil
+}
+
+// findClientConn is a helper that returns a connection to the client node or, if the client
+// is connected to a different server, a serverParts describing the server to which the
+// client bound RPC should be forwarded.
+func (a *Agent) findClientConn(nodeID string) (*nodeConnState, *peers.Parts, error) {
+	snap, err := a.srv.State().Snapshot()
+	if err != nil {
+		return nil, nil, structs.NewErrRPCCoded(500, err.Error())
+	}
+
+	node, err := snap.NodeByID(nil, nodeID)
+	if err != nil {
+		return nil, nil, structs.NewErrRPCCoded(500, err.Error())
+	}
+
+	if node == nil {
+		err := fmt.Errorf("Unknown node %q", nodeID)
+		return nil, nil, structs.NewErrRPCCoded(404, err.Error())
+	}
+
+	if err := nodeSupportsRpc(node); err != nil {
+		return nil, nil, structs.NewErrRPCCoded(400, err.Error())
+	}
+
+	// Get the Connection to the client either by fowarding to another server
+	// or creating direct stream
+	state, ok := a.srv.getNodeConn(nodeID)
+	if ok {
+		return state, nil, nil
+	}
+
+	// Determine the server that has a connection to the node
+	srv, err := a.srv.serverWithNodeConn(nodeID, a.srv.Region())
+	if err != nil {
+		code := 500
+		if structs.IsErrNoNodeConn(err) {
+			code = 404
+		}
+		return nil, nil, structs.NewErrRPCCoded(code, err.Error())
+	}
+
+	return nil, srv, nil
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/client_alloc_endpoint.go

@@ -0,0 +1,639 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/hashicorp/go-hclog"
+	metrics "github.com/hashicorp/go-metrics/compat"
+	"github.com/hashicorp/go-msgpack/v2/codec"
+
+	"github.com/hashicorp/nomad/acl"
+	cstructs "github.com/hashicorp/nomad/client/structs"
+	"github.com/hashicorp/nomad/helper/pointer"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+// ClientAllocations is used to forward RPC requests to the targeted Nomad client's
+// Allocation endpoint.
+type ClientAllocations struct {
+	srv    *Server
+	logger hclog.Logger
+}
+
+func NewClientAllocationsEndpoint(srv *Server) *ClientAllocations {
+	return &ClientAllocations{srv: srv, logger: srv.logger.Named("client_allocs")}
+}
+
+func (a *ClientAllocations) register() {
+	a.srv.streamingRpcs.Register("Allocations.Exec", a.exec)
+}
+
+// GarbageCollectAll is used to garbage collect all allocations on a client.
+func (a *ClientAllocations) GarbageCollectAll(args *structs.NodeSpecificRequest, reply *structs.GenericResponse) error {
+	// We only allow stale reads since the only potentially stale information is
+	// the Node registration and the cost is fairly high for adding another hop
+	// in the forwarding chain.
+	args.QueryOptions.AllowStale = true
+
+	authErr := a.srv.Authenticate(nil, args)
+
+	// Potentially forward to a different region.
+	if done, err := a.srv.forward("ClientAllocations.GarbageCollectAll", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "client_allocations", "garbage_collect_all"}, time.Now())
+
+	// Check node read permissions
+	if aclObj, err := a.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNodeWrite() {
+		return structs.ErrPermissionDenied
+	}
+
+	// Verify the arguments.
+	if args.NodeID == "" {
+		return errors.New("missing NodeID")
+	}
+
+	// Make sure Node is valid and new enough to support RPC
+	snap, err := a.srv.State().Snapshot()
+	if err != nil {
+		return err
+	}
+
+	_, err = getNodeForRpc(snap, args.NodeID)
+	if err != nil {
+		return err
+	}
+
+	// Get the connection to the client
+	state, ok := a.srv.getNodeConn(args.NodeID)
+	if !ok {
+		return findNodeConnAndForward(a.srv, args.NodeID, "ClientAllocations.GarbageCollectAll", args, reply)
+	}
+
+	// Make the RPC
+	return NodeRpc(state.Session, "Allocations.GarbageCollectAll", args, reply)
+}
+
+// Signal is used to send a signal to an allocation on a client.
+func (a *ClientAllocations) Signal(args *structs.AllocSignalRequest, reply *structs.GenericResponse) error {
+	// We only allow stale reads since the only potentially stale information is
+	// the Node registration and the cost is fairly high for adding another hope
+	// in the forwarding chain.
+	args.QueryOptions.AllowStale = true
+
+	authErr := a.srv.Authenticate(nil, args)
+
+	// Potentially forward to a different region.
+	if done, err := a.srv.forward("ClientAllocations.Signal", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "client_allocations", "signal"}, time.Now())
+
+	// Verify the arguments.
+	if args.AllocID == "" {
+		return errors.New("missing AllocID")
+	}
+
+	// Find the allocation
+	snap, err := a.srv.State().Snapshot()
+	if err != nil {
+		return err
+	}
+
+	alloc, err := getAlloc(snap, args.AllocID)
+	if err != nil {
+		return err
+	}
+
+	// Check namespace alloc-lifecycle permission.
+	if aclObj, err := a.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNsOp(alloc.Namespace, acl.NamespaceCapabilityAllocLifecycle) {
+		return structs.ErrPermissionDenied
+	}
+
+	// Make sure Node is valid and new enough to support RPC
+	_, err = getNodeForRpc(snap, alloc.NodeID)
+	if err != nil {
+		return err
+	}
+
+	// Get the connection to the client
+	state, ok := a.srv.getNodeConn(alloc.NodeID)
+	if !ok {
+		return findNodeConnAndForward(a.srv, alloc.NodeID, "ClientAllocations.Signal", args, reply)
+	}
+
+	// Make the RPC
+	return NodeRpc(state.Session, "Allocations.Signal", args, reply)
+}
+
+func (a *ClientAllocations) SetPauseState(args *structs.AllocPauseRequest, reply *structs.GenericResponse) error {
+	args.QueryOptions.AllowStale = true
+	authErr := a.srv.Authenticate(nil, args)
+
+	// Potentially forward to a different region.
+	if done, err := a.srv.forward("ClientAllocations.SetPauseState", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "client_allocations", "pause_set"}, time.Now())
+
+	// Verify the arguments.
+	if args.AllocID == "" {
+		return errors.New("missing AllocID")
+	}
+
+	// Find the allocation.
+	snap, err := a.srv.State().Snapshot()
+	if err != nil {
+		return err
+	}
+
+	alloc, err := getAlloc(snap, args.AllocID)
+	if err != nil {
+		return err
+	}
+
+	// Check namespace submit-job permission.
+	if aclObj, err := a.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNsOpAnyOf(alloc.Namespace,
+		acl.NamespaceCapabilitySubmitJob,
+		acl.NamespaceCapabilityPauseAllocation,
+	) {
+		return structs.ErrPermissionDenied
+	}
+
+	// Make sure the node is valid and new enough to support RPC
+	_, err = getNodeForRpc(snap, alloc.NodeID)
+	if err != nil {
+		return err
+	}
+
+	// Get the connection to the client
+	state, ok := a.srv.getNodeConn(alloc.NodeID)
+	if !ok {
+		return findNodeConnAndForward(a.srv, alloc.NodeID, "ClientAllocations.SetPauseState", args, reply)
+	}
+
+	// Make the RPC
+	return NodeRpc(state.Session, "Allocations.SetPauseState", args, reply)
+}
+
+func (a *ClientAllocations) GetPauseState(args *structs.AllocGetPauseStateRequest, reply *structs.AllocGetPauseStateResponse) error {
+	args.QueryOptions.AllowStale = true
+	authErr := a.srv.Authenticate(nil, args)
+
+	// Potentially forward to a different region.
+	if done, err := a.srv.forward("ClientAllocations.GetPauseState", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricRead, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "client_allocations", "pause_get"}, time.Now())
+
+	// Verify the arguments.
+	if args.AllocID == "" {
+		return errors.New("missing AllocID")
+	}
+
+	// Find the allocation.
+	snap, err := a.srv.State().Snapshot()
+	if err != nil {
+		return err
+	}
+
+	alloc, err := getAlloc(snap, args.AllocID)
+	if err != nil {
+		return err
+	}
+
+	// Check namespace read-job permission.
+	if aclObj, err := a.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNsOp(alloc.Namespace, acl.NamespaceCapabilityReadJob) {
+		return structs.ErrPermissionDenied
+	}
+
+	// Make sure the node is valid and new enough to support RPC
+	_, err = getNodeForRpc(snap, alloc.NodeID)
+	if err != nil {
+		return err
+	}
+
+	// Get the connection to the client
+	state, ok := a.srv.getNodeConn(alloc.NodeID)
+	if !ok {
+		return findNodeConnAndForward(a.srv, alloc.NodeID, "ClientAllocations.GetPauseState", args, reply)
+	}
+
+	// Make the RPC
+	return NodeRpc(state.Session, "Allocations.GetPauseState", args, reply)
+}
+
+// GarbageCollect is used to garbage collect an allocation on a client.
+func (a *ClientAllocations) GarbageCollect(args *structs.AllocSpecificRequest, reply *structs.GenericResponse) error {
+	// We only allow stale reads since the only potentially stale information is
+	// the Node registration and the cost is fairly high for adding another hop
+	// in the forwarding chain.
+	args.QueryOptions.AllowStale = true
+
+	authErr := a.srv.Authenticate(nil, args)
+
+	// Potentially forward to a different region.
+	if done, err := a.srv.forward("ClientAllocations.GarbageCollect", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "client_allocations", "garbage_collect"}, time.Now())
+
+	// Verify the arguments.
+	if args.AllocID == "" {
+		return errors.New("missing AllocID")
+	}
+
+	// Find the allocation
+	snap, err := a.srv.State().Snapshot()
+	if err != nil {
+		return err
+	}
+
+	alloc, err := getAlloc(snap, args.AllocID)
+	if err != nil {
+		return err
+	}
+
+	// Check namespace submit-job permission.
+	if aclObj, err := a.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNsOpAnyOf(alloc.Namespace,
+		acl.NamespaceCapabilitySubmitJob,
+		acl.NamespaceCapabilityGCAllocation,
+	) {
+		return structs.ErrPermissionDenied
+	}
+
+	// Make sure Node is valid and new enough to support RPC
+	_, err = getNodeForRpc(snap, alloc.NodeID)
+	if err != nil {
+		return err
+	}
+
+	// Get the connection to the client
+	state, ok := a.srv.getNodeConn(alloc.NodeID)
+	if !ok {
+		return findNodeConnAndForward(a.srv, alloc.NodeID, "ClientAllocations.GarbageCollect", args, reply)
+	}
+
+	// Make the RPC
+	return NodeRpc(state.Session, "Allocations.GarbageCollect", args, reply)
+}
+
+// Restart is used to trigger a restart of an allocation or a subtask on a client.
+func (a *ClientAllocations) Restart(args *structs.AllocRestartRequest, reply *structs.GenericResponse) error {
+	// We only allow stale reads since the only potentially stale information is
+	// the Node registration and the cost is fairly high for adding another hop
+	// in the forwarding chain.
+	args.QueryOptions.AllowStale = true
+
+	authErr := a.srv.Authenticate(nil, args)
+
+	// Potentially forward to a different region.
+	if done, err := a.srv.forward("ClientAllocations.Restart", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "client_allocations", "restart"}, time.Now())
+
+	// Find the allocation
+	snap, err := a.srv.State().Snapshot()
+	if err != nil {
+		return err
+	}
+
+	alloc, err := getAlloc(snap, args.AllocID)
+	if err != nil {
+		return err
+	}
+
+	// Check for namespace alloc-lifecycle permissions.
+	if aclObj, err := a.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNsOp(alloc.Namespace, acl.NamespaceCapabilityAllocLifecycle) {
+		return structs.ErrPermissionDenied
+	}
+
+	// Make sure Node is valid and new enough to support RPC
+	_, err = getNodeForRpc(snap, alloc.NodeID)
+	if err != nil {
+		return err
+	}
+
+	// Get the connection to the client
+	state, ok := a.srv.getNodeConn(alloc.NodeID)
+	if !ok {
+		return findNodeConnAndForward(a.srv, alloc.NodeID, "ClientAllocations.Restart", args, reply)
+	}
+
+	// Make the RPC
+	return NodeRpc(state.Session, "Allocations.Restart", args, reply)
+}
+
+// Stats is used to collect allocation statistics
+func (a *ClientAllocations) Stats(args *cstructs.AllocStatsRequest, reply *cstructs.AllocStatsResponse) error {
+	// We only allow stale reads since the only potentially stale information is
+	// the Node registration and the cost is fairly high for adding another hop
+	// in the forwarding chain.
+	args.QueryOptions.AllowStale = true
+
+	authErr := a.srv.Authenticate(nil, args)
+
+	// Potentially forward to a different region.
+	if done, err := a.srv.forward("ClientAllocations.Stats", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricRead, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "client_allocations", "stats"}, time.Now())
+
+	// Find the allocation
+	snap, err := a.srv.State().Snapshot()
+	if err != nil {
+		return err
+	}
+
+	alloc, err := getAlloc(snap, args.AllocID)
+	if err != nil {
+		return err
+	}
+
+	// Check for namespace read-job permissions.
+	if aclObj, err := a.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNsOp(alloc.Namespace, acl.NamespaceCapabilityReadJob) {
+		return structs.ErrPermissionDenied
+	}
+
+	// Make sure Node is valid and new enough to support RPC
+	_, err = getNodeForRpc(snap, alloc.NodeID)
+	if err != nil {
+		return err
+	}
+
+	// Get the connection to the client
+	state, ok := a.srv.getNodeConn(alloc.NodeID)
+	if !ok {
+		return findNodeConnAndForward(a.srv, alloc.NodeID, "ClientAllocations.Stats", args, reply)
+	}
+
+	// Make the RPC
+	return NodeRpc(state.Session, "Allocations.Stats", args, reply)
+}
+
+// Checks is the server implementation of the allocation checks RPC. The
+// ultimate response is provided by the node running the allocation. This RPC
+// is needed to handle queries which hit the server agent API directly, or via
+// another node which is not running the allocation.
+func (a *ClientAllocations) Checks(args *cstructs.AllocChecksRequest, reply *cstructs.AllocChecksResponse) error {
+
+	// We only allow stale reads since the only potentially stale information
+	// is the Node registration and the cost is fairly high for adding another
+	// hop in the forwarding chain.
+	args.QueryOptions.AllowStale = true
+
+	authErr := a.srv.Authenticate(nil, args)
+
+	// Potentially forward to a different region.
+	if done, err := a.srv.forward("ClientAllocations.Checks", args, args, reply); done {
+		return err
+	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricRead, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "client_allocations", "checks"}, time.Now())
+
+	// Grab the state snapshot, as we need this to perform lookups for a number
+	// of objects, all things being well.
+	snap, err := a.srv.State().Snapshot()
+	if err != nil {
+		return err
+	}
+
+	// Get the full allocation object, so we have information such as the
+	// namespace and node ID.
+	alloc, err := getAlloc(snap, args.AllocID)
+	if err != nil {
+		return err
+	}
+
+	// Check for namespace read-job permissions.
+	if aclObj, err := a.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNsOp(alloc.Namespace, acl.NamespaceCapabilityReadJob) {
+		return structs.ErrPermissionDenied
+	}
+
+	// Make sure Node is valid and new enough to support RPC.
+	if _, err = getNodeForRpc(snap, alloc.NodeID); err != nil {
+		return err
+	}
+
+	// Get the connection to the client.
+	state, ok := a.srv.getNodeConn(alloc.NodeID)
+	if !ok {
+		return findNodeConnAndForward(a.srv, alloc.NodeID, "ClientAllocations.Checks", args, reply)
+	}
+
+	// Make the RPC
+	return NodeRpc(state.Session, "Allocations.Checks", args, reply)
+}
+
+// exec is used to execute command in a running task
+func (a *ClientAllocations) exec(conn io.ReadWriteCloser) {
+	defer conn.Close()
+	defer metrics.MeasureSince([]string{"nomad", "alloc", "exec"}, time.Now())
+
+	// Decode the arguments
+	var args cstructs.AllocExecRequest
+	decoder := codec.NewDecoder(conn, structs.MsgpackHandle)
+	encoder := codec.NewEncoder(conn, structs.MsgpackHandle)
+
+	if err := decoder.Decode(&args); err != nil {
+		handleStreamResultError(err, pointer.Of(int64(500)), encoder)
+		return
+	}
+
+	authErr := a.srv.Authenticate(nil, &args)
+
+	// Check if we need to forward to a different region
+	if r := args.RequestRegion(); r != a.srv.Region() {
+		forwardRegionStreamingRpc(a.srv, conn, encoder, &args, "Allocations.Exec",
+			args.AllocID, &args.QueryOptions)
+		return
+	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, &args)
+	if authErr != nil {
+		handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
+		return
+	}
+
+	// Verify the arguments.
+	if args.AllocID == "" {
+		handleStreamResultError(errors.New("missing AllocID"), pointer.Of(int64(400)), encoder)
+		return
+	}
+
+	// Retrieve the allocation
+	snap, err := a.srv.State().Snapshot()
+	if err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	}
+
+	alloc, err := getAlloc(snap, args.AllocID)
+	if structs.IsErrUnknownAllocation(err) {
+		handleStreamResultError(err, pointer.Of(int64(404)), encoder)
+		return
+	}
+	if err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	}
+
+	// Check node read permissions
+	if aclObj, err := a.srv.ResolveACL(&args); err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	} else if !aclObj.AllowNsOp(alloc.Namespace, acl.NamespaceCapabilityAllocExec) {
+		// client ultimately checks if AllocNodeExec is required
+		handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
+		return
+	}
+
+	if alloc.ClientTerminalStatus() {
+		handleStreamResultError(fmt.Errorf("exec not possible, client status of allocation %s is %s", alloc.ID, alloc.ClientStatus),
+			pointer.Of(int64(http.StatusBadRequest)), encoder)
+		return
+	}
+
+	// Handle job ID if requested.
+	if args.JobID != "" {
+		// Verify job exists.
+		job, err := snap.JobByID(nil, args.Namespace, args.JobID)
+		if err != nil {
+			handleStreamResultError(err,
+				pointer.Of(int64(http.StatusInternalServerError)), encoder)
+			return
+		}
+		if job == nil {
+			handleStreamResultError(
+				fmt.Errorf("job %s not found in namespace %s", args.JobID, args.Namespace),
+				pointer.Of(int64(http.StatusNotFound)), encoder)
+			return
+		}
+
+		// Verify requested allocation belongs to the job.
+		if args.JobID != alloc.JobID {
+			handleStreamResultError(
+				fmt.Errorf("job %s does not have allocation %s", args.JobID, alloc.ID),
+				pointer.Of(int64(http.StatusBadRequest)), encoder,
+			)
+		}
+	}
+
+	nodeID := alloc.NodeID
+
+	// Make sure Node is valid and new enough to support RPC
+	node, err := snap.NodeByID(nil, nodeID)
+	if err != nil {
+		handleStreamResultError(err, pointer.Of(int64(500)), encoder)
+		return
+	}
+
+	if node == nil {
+		err := fmt.Errorf("Unknown node %q", nodeID)
+		handleStreamResultError(err, pointer.Of(int64(400)), encoder)
+		return
+	}
+
+	if err := nodeSupportsRpc(node); err != nil {
+		handleStreamResultError(err, pointer.Of(int64(400)), encoder)
+		return
+	}
+
+	// Get the connection to the client either by forwarding to another server
+	// or creating a direct stream
+	var clientConn net.Conn
+	state, ok := a.srv.getNodeConn(nodeID)
+	if !ok {
+		// Determine the Server that has a connection to the node.
+		srv, err := a.srv.serverWithNodeConn(nodeID, a.srv.Region())
+		if err != nil {
+			var code *int64
+			if structs.IsErrNoNodeConn(err) {
+				code = pointer.Of(int64(404))
+			}
+			handleStreamResultError(err, code, encoder)
+			return
+		}
+
+		// Get a connection to the server
+		conn, err := a.srv.streamingRpc(srv, "Allocations.Exec")
+		if err != nil {
+			handleStreamResultError(err, nil, encoder)
+			return
+		}
+
+		clientConn = conn
+	} else {
+		stream, err := NodeStreamingRpc(state.Session, "Allocations.Exec")
+		if err != nil {
+			handleStreamResultError(err, nil, encoder)
+			return
+		}
+		clientConn = stream
+	}
+	defer clientConn.Close()
+
+	// Send the request.
+	outEncoder := codec.NewEncoder(clientConn, structs.MsgpackHandle)
+	if err := outEncoder.Encode(args); err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	}
+
+	structs.Bridge(conn, clientConn)
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/client_csi_endpoint.go

@@ -0,0 +1,360 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+	"time"
+
+	log "github.com/hashicorp/go-hclog"
+	memdb "github.com/hashicorp/go-memdb"
+	metrics "github.com/hashicorp/go-metrics/compat"
+
+	cstructs "github.com/hashicorp/nomad/client/structs"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+// ClientCSI is used to forward RPC requests to the targed Nomad client's
+// CSIController endpoint.
+type ClientCSI struct {
+	srv    *Server
+	ctx    *RPCContext
+	logger log.Logger
+}
+
+func NewClientCSIEndpoint(srv *Server, ctx *RPCContext) *ClientCSI {
+	return &ClientCSI{srv: srv, ctx: ctx, logger: srv.logger.Named("client_csi")}
+}
+
+func (a *ClientCSI) ControllerAttachVolume(args *cstructs.ClientCSIControllerAttachVolumeRequest, reply *cstructs.ClientCSIControllerAttachVolumeResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "attach_volume"}, time.Now())
+
+	err := a.sendCSIControllerRPC(args.PluginID,
+		"CSI.ControllerAttachVolume",
+		"ClientCSI.ControllerAttachVolume",
+		structs.RateMetricWrite,
+		args, reply)
+	if err != nil {
+		return fmt.Errorf("controller attach volume: %v", err)
+	}
+	return nil
+}
+
+func (a *ClientCSI) ControllerValidateVolume(args *cstructs.ClientCSIControllerValidateVolumeRequest, reply *cstructs.ClientCSIControllerValidateVolumeResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "validate_volume"}, time.Now())
+
+	err := a.sendCSIControllerRPC(args.PluginID,
+		"CSI.ControllerValidateVolume",
+		"ClientCSI.ControllerValidateVolume",
+		structs.RateMetricWrite,
+		args, reply)
+	if err != nil {
+		return fmt.Errorf("controller validate volume: %v", err)
+	}
+	return nil
+}
+
+func (a *ClientCSI) ControllerDetachVolume(args *cstructs.ClientCSIControllerDetachVolumeRequest, reply *cstructs.ClientCSIControllerDetachVolumeResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "detach_volume"}, time.Now())
+
+	err := a.sendCSIControllerRPC(args.PluginID,
+		"CSI.ControllerDetachVolume",
+		"ClientCSI.ControllerDetachVolume",
+		structs.RateMetricWrite,
+		args, reply)
+	if err != nil {
+		return fmt.Errorf("controller detach volume: %v", err)
+	}
+	return nil
+}
+
+func (a *ClientCSI) ControllerCreateVolume(args *cstructs.ClientCSIControllerCreateVolumeRequest, reply *cstructs.ClientCSIControllerCreateVolumeResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "create_volume"}, time.Now())
+
+	err := a.sendCSIControllerRPC(args.PluginID,
+		"CSI.ControllerCreateVolume",
+		"ClientCSI.ControllerCreateVolume",
+		structs.RateMetricWrite,
+		args, reply)
+	if err != nil {
+		return fmt.Errorf("controller create volume: %v", err)
+	}
+	return nil
+}
+
+func (a *ClientCSI) ControllerExpandVolume(args *cstructs.ClientCSIControllerExpandVolumeRequest, reply *cstructs.ClientCSIControllerExpandVolumeResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "expand_volume"}, time.Now())
+
+	err := a.sendCSIControllerRPC(args.PluginID,
+		"CSI.ControllerExpandVolume",
+		"ClientCSI.ControllerExpandVolume",
+		structs.RateMetricWrite,
+		args, reply)
+	if err != nil {
+		return fmt.Errorf("controller expand volume: %v", err)
+	}
+	return nil
+}
+
+func (a *ClientCSI) ControllerDeleteVolume(args *cstructs.ClientCSIControllerDeleteVolumeRequest, reply *cstructs.ClientCSIControllerDeleteVolumeResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "delete_volume"}, time.Now())
+
+	err := a.sendCSIControllerRPC(args.PluginID,
+		"CSI.ControllerDeleteVolume",
+		"ClientCSI.ControllerDeleteVolume",
+		structs.RateMetricWrite,
+		args, reply)
+	if err != nil {
+		return fmt.Errorf("controller delete volume: %v", err)
+	}
+	return nil
+}
+
+func (a *ClientCSI) ControllerListVolumes(args *cstructs.ClientCSIControllerListVolumesRequest, reply *cstructs.ClientCSIControllerListVolumesResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "list_volumes"}, time.Now())
+
+	err := a.sendCSIControllerRPC(args.PluginID,
+		"CSI.ControllerListVolumes",
+		"ClientCSI.ControllerListVolumes",
+		structs.RateMetricList,
+		args, reply)
+	if err != nil {
+		return fmt.Errorf("controller list volumes: %v", err)
+	}
+	return nil
+}
+
+func (a *ClientCSI) ControllerCreateSnapshot(args *cstructs.ClientCSIControllerCreateSnapshotRequest, reply *cstructs.ClientCSIControllerCreateSnapshotResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "create_snapshot"}, time.Now())
+
+	err := a.sendCSIControllerRPC(args.PluginID,
+		"CSI.ControllerCreateSnapshot",
+		"ClientCSI.ControllerCreateSnapshot",
+		structs.RateMetricWrite,
+		args, reply)
+	if err != nil {
+		return fmt.Errorf("controller create snapshot: %v", err)
+	}
+	return nil
+}
+
+func (a *ClientCSI) ControllerDeleteSnapshot(args *cstructs.ClientCSIControllerDeleteSnapshotRequest, reply *cstructs.ClientCSIControllerDeleteSnapshotResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "delete_snapshot"}, time.Now())
+
+	err := a.sendCSIControllerRPC(args.PluginID,
+		"CSI.ControllerDeleteSnapshot",
+		"ClientCSI.ControllerDeleteSnapshot",
+		structs.RateMetricWrite,
+		args, reply)
+	if err != nil {
+		return fmt.Errorf("controller delete snapshot: %v", err)
+	}
+	return nil
+}
+
+func (a *ClientCSI) ControllerListSnapshots(args *cstructs.ClientCSIControllerListSnapshotsRequest, reply *cstructs.ClientCSIControllerListSnapshotsResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "list_snapshots"}, time.Now())
+
+	err := a.sendCSIControllerRPC(args.PluginID,
+		"CSI.ControllerListSnapshots",
+		"ClientCSI.ControllerListSnapshots",
+		structs.RateMetricList,
+		args, reply)
+	if err != nil {
+		return fmt.Errorf("controller list snapshots: %v", err)
+	}
+	return nil
+}
+
+func (a *ClientCSI) sendCSIControllerRPC(pluginID, method, fwdMethod, op string, args cstructs.CSIControllerRequest, reply interface{}) error {
+
+	// client requests aren't RequestWithIdentity, so we use a placeholder here
+	// to populate the identity data for metrics
+	identityReq := &structs.GenericRequest{}
+
+	aclObj, err := a.srv.AuthenticateServerOnly(a.ctx, identityReq)
+	a.srv.MeasureRPCRate("client_csi", op, identityReq)
+
+	if err != nil || !aclObj.AllowServerOp() {
+		return structs.ErrPermissionDenied
+	}
+
+	clientIDs, err := a.clientIDsForController(pluginID)
+	if err != nil {
+		return err
+	}
+
+	for _, clientID := range clientIDs {
+		args.SetControllerNodeID(clientID)
+
+		state, ok := a.srv.getNodeConn(clientID)
+		if !ok {
+			return findNodeConnAndForward(a.srv,
+				clientID, fwdMethod, args, reply)
+		}
+
+		err = NodeRpc(state.Session, method, args, reply)
+		if err == nil {
+			return nil
+		}
+		if a.isRetryable(err) {
+			a.logger.Debug("failed to reach controller on client",
+				"nodeID", clientID, "error", err)
+			continue
+		}
+		return err
+	}
+	return err
+}
+
+// we can retry the same RPC on a different controller in the cases where the
+// client has stopped and been GC'd, or where the controller has stopped but
+// we don't have the fingerprint update yet
+func (a *ClientCSI) isRetryable(err error) bool {
+	// TODO: msgpack-rpc mangles the error so we lose the wrapping,
+	// but if that can be fixed upstream we should use that here instead
+	return strings.Contains(err.Error(), "CSI client error (retryable)") ||
+		strings.Contains(err.Error(), "Unknown node")
+}
+
+func (a *ClientCSI) NodeDetachVolume(args *cstructs.ClientCSINodeDetachVolumeRequest, reply *cstructs.ClientCSINodeDetachVolumeResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_csi_node", "detach_volume"}, time.Now())
+	return a.sendCSINodeRPC(
+		args.NodeID,
+		"CSI.NodeDetachVolume",
+		"ClientCSI.NodeDetachVolume",
+		structs.RateMetricWrite,
+		args,
+		reply,
+	)
+}
+
+func (a *ClientCSI) NodeExpandVolume(args *cstructs.ClientCSINodeExpandVolumeRequest, reply *cstructs.ClientCSINodeExpandVolumeResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_csi_node", "expand_volume"}, time.Now())
+	return a.sendCSINodeRPC(
+		args.Claim.NodeID,
+		"CSI.NodeExpandVolume",
+		"ClientCSI.NodeExpandVolume",
+		structs.RateMetricWrite,
+		args,
+		reply,
+	)
+}
+
+func (a *ClientCSI) sendCSINodeRPC(nodeID, method, fwdMethod, op string, args any, reply any) error {
+	// client requests aren't RequestWithIdentity, so we use a placeholder here
+	// to populate the identity data for metrics
+	identityReq := &structs.GenericRequest{}
+	aclObj, err := a.srv.AuthenticateServerOnly(a.ctx, identityReq)
+	a.srv.MeasureRPCRate("client_csi", op, identityReq)
+
+	if err != nil || !aclObj.AllowServerOp() {
+		return structs.ErrPermissionDenied
+	}
+
+	// Make sure Node is valid and new enough to support RPC
+	snap, err := a.srv.State().Snapshot()
+	if err != nil {
+		return err
+	}
+
+	_, err = getNodeForRpc(snap, nodeID)
+	if err != nil {
+		return err
+	}
+
+	// Get the connection to the client
+	state, ok := a.srv.getNodeConn(nodeID)
+	if !ok {
+		return findNodeConnAndForward(a.srv, nodeID, fwdMethod, args, reply)
+	}
+
+	// Make the RPC
+	if err := NodeRpc(state.Session, method, args, reply); err != nil {
+		return fmt.Errorf("%s error: %w", method, err)
+	}
+	return nil
+}
+
+// clientIDsForController returns a sorted list of client IDs where the
+// controller plugin is expected to be running.
+func (a *ClientCSI) clientIDsForController(pluginID string) ([]string, error) {
+
+	snap, err := a.srv.State().Snapshot()
+	if err != nil {
+		return nil, err
+	}
+
+	if pluginID == "" {
+		return nil, fmt.Errorf("missing plugin ID")
+	}
+
+	ws := memdb.NewWatchSet()
+
+	// note: plugin IDs are not scoped to region but volumes are. so any Nomad
+	// client we get for a controller is already in the same region for the
+	// volume.
+	plugin, err := snap.CSIPluginByID(ws, pluginID)
+	if err != nil {
+		return nil, fmt.Errorf("error getting plugin: %s, %v", pluginID, err)
+	}
+	if plugin == nil {
+		return nil, fmt.Errorf("plugin missing: %s", pluginID)
+	}
+
+	clientIDs := []string{}
+
+	if len(plugin.Controllers) == 0 {
+		return nil, fmt.Errorf("failed to find instances of controller plugin %q", pluginID)
+	}
+
+	var merr error
+	for clientID, controller := range plugin.Controllers {
+		if !controller.IsController() {
+			// we don't have separate types for CSIInfo depending on whether
+			// it's a controller or node. this error should never make it to
+			// production
+			merr = errors.Join(merr, fmt.Errorf(
+				"plugin instance %q is not a controller but was registered as one - this is always a bug", controller.AllocID))
+			continue
+		}
+
+		if !controller.Healthy {
+			merr = errors.Join(merr, fmt.Errorf(
+				"plugin instance %q is not healthy", controller.AllocID))
+			continue
+		}
+
+		node, err := getNodeForRpc(snap, clientID)
+		if err != nil || node == nil {
+			merr = errors.Join(merr, fmt.Errorf(
+				"cannot find node %q for plugin instance %q", clientID, controller.AllocID))
+			continue
+		}
+
+		if node.Status != structs.NodeStatusReady {
+			merr = errors.Join(merr, fmt.Errorf(
+				"node %q for plugin instance %q is not ready", clientID, controller.AllocID))
+			continue
+		}
+
+		clientIDs = append(clientIDs, clientID)
+	}
+
+	if len(clientIDs) == 0 {
+		return nil, fmt.Errorf("failed to find clients running controller plugin %q: %v",
+			pluginID, merr)
+	}
+
+	// Many plugins don't handle concurrent requests as described in the spec,
+	// and have undocumented expectations of using k8s-specific sidecars to
+	// leader elect. Sort the client IDs so that we prefer sending requests to
+	// the same controller to hack around this.
+	slices.Sort(clientIDs)
+
+	return clientIDs, nil
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/client_csi_endpoint_test.go

@@ -0,0 +1,558 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"fmt"
+	"net/rpc"
+	"testing"
+	"time"
+
+	memdb "github.com/hashicorp/go-memdb"
+	msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc/v2"
+	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/client"
+	"github.com/hashicorp/nomad/client/config"
+	cstructs "github.com/hashicorp/nomad/client/structs"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/hashicorp/nomad/nomad/mock"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/testutil"
+	"github.com/shoenig/test/must"
+)
+
+// MockClientCSI is a mock for the nomad.ClientCSI RPC server (see
+// nomad/client_csi_endpoint.go). This can be used with a TestRPCOnlyClient to
+// return specific plugin responses back to server RPCs for testing. Note that
+// responses that have no bodies have no "Next*Response" field and will always
+// return an empty response body.
+type MockClientCSI struct {
+	NextValidateError                  error
+	NextAttachError                    error
+	NextAttachResponse                 *cstructs.ClientCSIControllerAttachVolumeResponse
+	NextDetachError                    error
+	NextCreateError                    error
+	NextCreateResponse                 *cstructs.ClientCSIControllerCreateVolumeResponse
+	NextDeleteError                    error
+	NextListExternalError              error
+	NextListExternalResponse           *cstructs.ClientCSIControllerListVolumesResponse
+	NextCreateSnapshotError            error
+	NextCreateSnapshotResponse         *cstructs.ClientCSIControllerCreateSnapshotResponse
+	NextDeleteSnapshotError            error
+	NextListExternalSnapshotsError     error
+	NextListExternalSnapshotsResponse  *cstructs.ClientCSIControllerListSnapshotsResponse
+	NextControllerExpandVolumeError    error
+	NextControllerExpandVolumeResponse *cstructs.ClientCSIControllerExpandVolumeResponse
+	NextNodeDetachError                error
+	NextNodeExpandError                error
+	LastNodeExpandRequest              *cstructs.ClientCSINodeExpandVolumeRequest
+	LastDeleteSnapshotRequest          *cstructs.ClientCSIControllerDeleteSnapshotRequest
+}
+
+func newMockClientCSI() *MockClientCSI {
+	return &MockClientCSI{
+		NextAttachResponse:                 &cstructs.ClientCSIControllerAttachVolumeResponse{},
+		NextCreateResponse:                 &cstructs.ClientCSIControllerCreateVolumeResponse{},
+		NextListExternalResponse:           &cstructs.ClientCSIControllerListVolumesResponse{},
+		NextCreateSnapshotResponse:         &cstructs.ClientCSIControllerCreateSnapshotResponse{},
+		NextListExternalSnapshotsResponse:  &cstructs.ClientCSIControllerListSnapshotsResponse{},
+		NextControllerExpandVolumeResponse: &cstructs.ClientCSIControllerExpandVolumeResponse{},
+	}
+}
+
+func (c *MockClientCSI) ControllerValidateVolume(req *cstructs.ClientCSIControllerValidateVolumeRequest, resp *cstructs.ClientCSIControllerValidateVolumeResponse) error {
+	return c.NextValidateError
+}
+
+func (c *MockClientCSI) ControllerAttachVolume(req *cstructs.ClientCSIControllerAttachVolumeRequest, resp *cstructs.ClientCSIControllerAttachVolumeResponse) error {
+	*resp = *c.NextAttachResponse
+	return c.NextAttachError
+}
+
+func (c *MockClientCSI) ControllerDetachVolume(req *cstructs.ClientCSIControllerDetachVolumeRequest, resp *cstructs.ClientCSIControllerDetachVolumeResponse) error {
+	return c.NextDetachError
+}
+
+func (c *MockClientCSI) ControllerCreateVolume(req *cstructs.ClientCSIControllerCreateVolumeRequest, resp *cstructs.ClientCSIControllerCreateVolumeResponse) error {
+	*resp = *c.NextCreateResponse
+	return c.NextCreateError
+}
+
+func (c *MockClientCSI) ControllerDeleteVolume(req *cstructs.ClientCSIControllerDeleteVolumeRequest, resp *cstructs.ClientCSIControllerDeleteVolumeResponse) error {
+	return c.NextDeleteError
+}
+
+func (c *MockClientCSI) ControllerListVolumes(req *cstructs.ClientCSIControllerListVolumesRequest, resp *cstructs.ClientCSIControllerListVolumesResponse) error {
+	*resp = *c.NextListExternalResponse
+	return c.NextListExternalError
+}
+
+func (c *MockClientCSI) ControllerCreateSnapshot(req *cstructs.ClientCSIControllerCreateSnapshotRequest, resp *cstructs.ClientCSIControllerCreateSnapshotResponse) error {
+	*resp = *c.NextCreateSnapshotResponse
+	return c.NextCreateSnapshotError
+}
+
+func (c *MockClientCSI) ControllerDeleteSnapshot(req *cstructs.ClientCSIControllerDeleteSnapshotRequest, resp *cstructs.ClientCSIControllerDeleteSnapshotResponse) error {
+	c.LastDeleteSnapshotRequest = req
+	return c.NextDeleteSnapshotError
+}
+
+func (c *MockClientCSI) ControllerListSnapshots(req *cstructs.ClientCSIControllerListSnapshotsRequest, resp *cstructs.ClientCSIControllerListSnapshotsResponse) error {
+	*resp = *c.NextListExternalSnapshotsResponse
+	return c.NextListExternalSnapshotsError
+}
+
+func (c *MockClientCSI) ControllerExpandVolume(req *cstructs.ClientCSIControllerExpandVolumeRequest, resp *cstructs.ClientCSIControllerExpandVolumeResponse) error {
+	*resp = *c.NextControllerExpandVolumeResponse
+	return c.NextControllerExpandVolumeError
+}
+
+func (c *MockClientCSI) NodeDetachVolume(req *cstructs.ClientCSINodeDetachVolumeRequest, resp *cstructs.ClientCSINodeDetachVolumeResponse) error {
+	return c.NextNodeDetachError
+}
+
+func (c *MockClientCSI) NodeExpandVolume(req *cstructs.ClientCSINodeExpandVolumeRequest, resp *cstructs.ClientCSINodeExpandVolumeResponse) error {
+	c.LastNodeExpandRequest = req
+	return c.NextNodeExpandError
+}
+
+func TestClientCSIController_AttachVolume_Local(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupLocal(t)
+
+	req := &cstructs.ClientCSIControllerAttachVolumeRequest{
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerAttachVolume", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_AttachVolume_Forwarded(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupForward(t)
+
+	req := &cstructs.ClientCSIControllerAttachVolumeRequest{
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerAttachVolume", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_DetachVolume_Local(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupLocal(t)
+
+	req := &cstructs.ClientCSIControllerDetachVolumeRequest{
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDetachVolume", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_DetachVolume_Forwarded(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupForward(t)
+
+	req := &cstructs.ClientCSIControllerDetachVolumeRequest{
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDetachVolume", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_ValidateVolume_Local(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupLocal(t)
+
+	req := &cstructs.ClientCSIControllerValidateVolumeRequest{
+		VolumeID:           "test",
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerValidateVolume", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_ValidateVolume_Forwarded(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupForward(t)
+
+	req := &cstructs.ClientCSIControllerValidateVolumeRequest{
+		VolumeID:           "test",
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerValidateVolume", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_CreateVolume_Local(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupLocal(t)
+
+	req := &cstructs.ClientCSIControllerCreateVolumeRequest{
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerCreateVolume", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_CreateVolume_Forwarded(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupForward(t)
+
+	req := &cstructs.ClientCSIControllerCreateVolumeRequest{
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerCreateVolume", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_DeleteVolume_Local(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupLocal(t)
+
+	req := &cstructs.ClientCSIControllerDeleteVolumeRequest{
+		ExternalVolumeID:   "test",
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDeleteVolume", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_DeleteVolume_Forwarded(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupForward(t)
+
+	req := &cstructs.ClientCSIControllerDeleteVolumeRequest{
+		ExternalVolumeID:   "test",
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDeleteVolume", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_ListVolumes_Local(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupLocal(t)
+
+	req := &cstructs.ClientCSIControllerListVolumesRequest{
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerListVolumes", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_ListVolumes_Forwarded(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupForward(t)
+
+	req := &cstructs.ClientCSIControllerListVolumesRequest{
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerListVolumes", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_CreateSnapshot_Local(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupLocal(t)
+
+	req := &cstructs.ClientCSIControllerCreateSnapshotRequest{
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerCreateSnapshot", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_CreateSnapshot_Forwarded(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupForward(t)
+
+	req := &cstructs.ClientCSIControllerCreateSnapshotRequest{
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerCreateSnapshot", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_DeleteSnapshot_Local(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupLocal(t)
+
+	req := &cstructs.ClientCSIControllerDeleteSnapshotRequest{
+		ID:                 "test",
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDeleteSnapshot", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_DeleteSnapshot_Forwarded(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupForward(t)
+
+	req := &cstructs.ClientCSIControllerDeleteSnapshotRequest{
+		ID:                 "test",
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDeleteSnapshot", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_ListSnapshots_Local(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupLocal(t)
+
+	req := &cstructs.ClientCSIControllerListSnapshotsRequest{
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerListSnapshots", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSIController_ListSnapshots_Forwarded(t *testing.T) {
+	ci.Parallel(t)
+	codec := setupForward(t)
+
+	req := &cstructs.ClientCSIControllerListSnapshotsRequest{
+		CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"},
+	}
+
+	var resp structs.GenericResponse
+	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerListSnapshots", req, &resp)
+	must.ErrorContains(t, err, "no plugins registered for type")
+}
+
+func TestClientCSI_NodeForControllerPlugin(t *testing.T) {
+	ci.Parallel(t)
+	srv, shutdown := TestServer(t, func(c *Config) {})
+	testutil.WaitForLeader(t, srv.RPC)
+	defer shutdown()
+
+	plugins := map[string]*structs.CSIInfo{
+		"minnie": {PluginID: "minnie",
+			Healthy:                  true,
+			ControllerInfo:           &structs.CSIControllerInfo{},
+			NodeInfo:                 &structs.CSINodeInfo{},
+			RequiresControllerPlugin: true,
+		},
+	}
+	state := srv.fsm.State()
+
+	node1 := mock.Node()
+	node1.Attributes["nomad.version"] = "0.11.0" // client RPCs not supported on early versions
+	node1.CSIControllerPlugins = plugins
+	node2 := mock.Node()
+	node2.CSIControllerPlugins = plugins
+	node2.ID = uuid.Generate()
+	node3 := mock.Node()
+	node3.ID = uuid.Generate()
+
+	err := state.UpsertNode(structs.MsgTypeTestSetup, 1002, node1)
+	must.NoError(t, err)
+	err = state.UpsertNode(structs.MsgTypeTestSetup, 1003, node2)
+	must.NoError(t, err)
+	err = state.UpsertNode(structs.MsgTypeTestSetup, 1004, node3)
+	must.NoError(t, err)
+
+	ws := memdb.NewWatchSet()
+
+	plugin, err := state.CSIPluginByID(ws, "minnie")
+	must.NoError(t, err)
+
+	clientCSI := NewClientCSIEndpoint(srv, nil)
+	nodeIDs, err := clientCSI.clientIDsForController(plugin.ID)
+	must.NoError(t, err)
+	must.Len(t, 1, nodeIDs)
+	// only node1 has both the controller and a recent Nomad version
+	must.Eq(t, nodeIDs[0], node1.ID)
+}
+
+// sets up a pair of servers, each with one client, and registers a plugin to the clients.
+// returns a RPC client to the leader and a cleanup function.
+func setupForward(t *testing.T) rpc.ClientCodec {
+
+	s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 })
+	s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 })
+	t.Cleanup(cleanupS1)
+	t.Cleanup(cleanupS2)
+	TestJoin(t, s1, s2)
+
+	testutil.WaitForLeader(t, s1.RPC)
+	testutil.WaitForLeader(t, s2.RPC)
+	codec := rpcClient(t, s1)
+
+	c1, cleanupC1 := client.TestClient(t, func(c *config.Config) {
+		c.Servers = []string{s1.config.RPCAddr.String()}
+	})
+	t.Cleanup(func() { cleanupC1() })
+
+	// Wait for client initialization
+	select {
+	case <-c1.Ready():
+	case <-time.After(10 * time.Second):
+		t.Fatal("client timedout on initialize")
+	}
+
+	c2, cleanupC2 := client.TestClient(t, func(c *config.Config) {
+		c.Servers = []string{s2.config.RPCAddr.String()}
+	})
+	t.Cleanup(func() { cleanupC2() })
+	select {
+	case <-c2.Ready():
+	case <-time.After(10 * time.Second):
+		t.Fatal("client timedout on initialize")
+	}
+
+	s1.nodeConnsLock.Lock()
+	delete(s1.nodeConns, c2.NodeID())
+	s1.nodeConnsLock.Unlock()
+
+	s2.nodeConnsLock.Lock()
+	delete(s2.nodeConns, c1.NodeID())
+	s2.nodeConnsLock.Unlock()
+
+	waitForNodes(t, s2, 1, 2)
+
+	plugins := map[string]*structs.CSIInfo{
+		"minnie": {PluginID: "minnie",
+			Healthy:                  true,
+			ControllerInfo:           &structs.CSIControllerInfo{},
+			NodeInfo:                 &structs.CSINodeInfo{},
+			RequiresControllerPlugin: true,
+		},
+	}
+
+	// update w/ plugin
+	node1 := c1.Node()
+	node1.Attributes["nomad.version"] = "0.11.0" // client RPCs not supported on early versions
+	node1.CSIControllerPlugins = plugins
+
+	s1.fsm.state.UpsertNode(structs.MsgTypeTestSetup, 1000, node1)
+
+	return codec
+}
+
+// sets up a single server with a client, and registers a plugin to the client.
+func setupLocal(t *testing.T) rpc.ClientCodec {
+	var err error
+	s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 1 })
+	t.Cleanup(cleanupS1)
+
+	testutil.WaitForLeader(t, s1.RPC)
+	testutil.WaitForKeyring(t, s1.RPC, s1.config.Region)
+	codec := rpcClient(t, s1)
+
+	mockCSI := newMockClientCSI()
+	mockCSI.NextValidateError = fmt.Errorf("no plugins registered for type")
+	mockCSI.NextAttachError = fmt.Errorf("no plugins registered for type")
+	mockCSI.NextDetachError = fmt.Errorf("no plugins registered for type")
+	mockCSI.NextCreateError = fmt.Errorf("no plugins registered for type")
+	mockCSI.NextDeleteError = fmt.Errorf("no plugins registered for type")
+	mockCSI.NextListExternalError = fmt.Errorf("no plugins registered for type")
+	mockCSI.NextCreateSnapshotError = fmt.Errorf("no plugins registered for type")
+	mockCSI.NextDeleteSnapshotError = fmt.Errorf("no plugins registered for type")
+	mockCSI.NextListExternalSnapshotsError = fmt.Errorf("no plugins registered for type")
+
+	c1, cleanupC1 := client.TestRPCOnlyClient(t, nil, s1.config.RPCAddr,
+		map[string]interface{}{"CSI": mockCSI},
+	)
+	t.Cleanup(cleanupC1)
+
+	node1 := c1.UpdateConfig(func(c *config.Config) {
+		c.Node.Attributes["nomad.version"] = "0.11.0" // client RPCs not supported on early versions
+	}).Node
+
+	req := &structs.NodeRegisterRequest{
+		Node:         node1,
+		WriteRequest: structs.WriteRequest{Region: "global"},
+	}
+	var resp structs.NodeUpdateResponse
+	err = c1.RPC("Node.Register", req, &resp)
+	must.NoError(t, err, must.Sprint("could not register client node"))
+
+	waitForNodes(t, s1, 1, 1)
+
+	plugins := map[string]*structs.CSIInfo{
+		"minnie": {PluginID: "minnie",
+			Healthy:                  true,
+			ControllerInfo:           &structs.CSIControllerInfo{},
+			NodeInfo:                 &structs.CSINodeInfo{},
+			RequiresControllerPlugin: true,
+		},
+	}
+
+	// update w/ plugin
+	node1 = c1.UpdateConfig(func(c *config.Config) {
+		c.Node.CSIControllerPlugins = plugins
+	}).Node
+	s1.fsm.state.UpsertNode(structs.MsgTypeTestSetup, 1000, node1)
+
+	return codec
+}
+
+// waitForNodes waits until the server is connected to connectedNodes
+// clients and totalNodes clients are in the state store
+func waitForNodes(t *testing.T, s *Server, connectedNodes, totalNodes int) {
+	codec := rpcClient(t, s)
+
+	testutil.WaitForResult(func() (bool, error) {
+		connNodes := s.connectedNodes()
+		if len(connNodes) != connectedNodes {
+			return false, fmt.Errorf("expected %d connected nodes but found %d", connectedNodes, len(connNodes))
+		}
+
+		get := &structs.NodeListRequest{
+			QueryOptions: structs.QueryOptions{Region: "global"},
+		}
+		var resp structs.NodeListResponse
+		err := msgpackrpc.CallWithCodec(codec, "Node.List", get, &resp)
+		if err != nil {
+			return false, err
+		}
+
+		if err != nil {
+			return false, fmt.Errorf("failed to list nodes: %v", err)
+		}
+		if len(resp.Nodes) != totalNodes {
+			return false, fmt.Errorf("expected %d total nodes but found %d", totalNodes, len(resp.Nodes))
+		}
+		return true, nil
+	}, func(err error) {
+		must.NoError(t, err)
+	})
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/client_fs_endpoint.go

@@ -0,0 +1,475 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"time"
+
+	log "github.com/hashicorp/go-hclog"
+	metrics "github.com/hashicorp/go-metrics/compat"
+	cstructs "github.com/hashicorp/nomad/client/structs"
+	"github.com/hashicorp/nomad/helper/pointer"
+
+	"github.com/hashicorp/go-msgpack/v2/codec"
+	"github.com/hashicorp/nomad/acl"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+// FileSystem endpoint is used for accessing the logs and filesystem of
+// allocations from a Node.
+type FileSystem struct {
+	srv    *Server
+	logger log.Logger
+}
+
+func NewFileSystemEndpoint(srv *Server) *FileSystem {
+	return &FileSystem{srv: srv, logger: srv.logger.Named("client_fs")}
+}
+
+func (f *FileSystem) register() {
+	f.srv.streamingRpcs.Register("FileSystem.Logs", f.logs)
+	f.srv.streamingRpcs.Register("FileSystem.Stream", f.stream)
+}
+
+// handleStreamResultError is a helper for sending an error with a potential
+// error code. The transmission of the error is ignored if the error has been
+// generated by the closing of the underlying transport.
+func handleStreamResultError(err error, code *int64, encoder *codec.Encoder) {
+	// Nothing to do as the conn is closed
+	if err == io.EOF || strings.Contains(err.Error(), "closed") {
+		return
+	}
+
+	// Attempt to send the error
+	encoder.Encode(&cstructs.StreamErrWrapper{
+		Error: cstructs.NewRpcError(err, code),
+	})
+}
+
+// forwardRegionStreamingRpc is used to make a streaming RPC to a different
+// region. It looks up the allocation in the remote region to determine what
+// remote server can route the request.
+func forwardRegionStreamingRpc(fsrv *Server, conn io.ReadWriteCloser,
+	encoder *codec.Encoder, args interface{}, method, allocID string, qo *structs.QueryOptions) {
+	// Request the allocation from the target region
+	allocReq := &structs.AllocSpecificRequest{
+		AllocID:      allocID,
+		QueryOptions: *qo,
+	}
+	var allocResp structs.SingleAllocResponse
+	if err := fsrv.forwardRegion(qo.RequestRegion(), "Alloc.GetAlloc", allocReq, &allocResp); err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	}
+
+	if allocResp.Alloc == nil {
+		handleStreamResultError(structs.NewErrUnknownAllocation(allocID), pointer.Of(int64(404)), encoder)
+		return
+	}
+
+	// Determine the Server that has a connection to the node.
+	srv, err := fsrv.serverWithNodeConn(allocResp.Alloc.NodeID, qo.RequestRegion())
+	if err != nil {
+		var code *int64
+		if structs.IsErrNoNodeConn(err) {
+			code = pointer.Of(int64(404))
+		}
+		handleStreamResultError(err, code, encoder)
+		return
+	}
+
+	// Get a connection to the server
+	srvConn, err := fsrv.streamingRpc(srv, method)
+	if err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	}
+	defer srvConn.Close()
+
+	// Send the request.
+	outEncoder := codec.NewEncoder(srvConn, structs.MsgpackHandle)
+	if err := outEncoder.Encode(args); err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	}
+
+	structs.Bridge(conn, srvConn)
+}
+
+// List is used to list the contents of an allocation's directory.
+func (f *FileSystem) List(args *cstructs.FsListRequest, reply *cstructs.FsListResponse) error {
+	// We only allow stale reads since the only potentially stale information is
+	// the Node registration and the cost is fairly high for adding another hope
+	// in the forwarding chain.
+	args.QueryOptions.AllowStale = true
+
+	authErr := f.srv.Authenticate(nil, args)
+
+	// Potentially forward to a different region.
+	if done, err := f.srv.forward("FileSystem.List", args, args, reply); done {
+		return err
+	}
+	f.srv.MeasureRPCRate("file_system", structs.RateMetricList, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "file_system", "list"}, time.Now())
+
+	// Verify the arguments.
+	if args.AllocID == "" {
+		return errors.New("missing allocation ID")
+	}
+
+	// Lookup the allocation
+	snap, err := f.srv.State().Snapshot()
+	if err != nil {
+		return err
+	}
+
+	alloc, err := getAlloc(snap, args.AllocID)
+	if err != nil {
+		return err
+	}
+
+	// Check namespace filesystem read permissions
+	allowNsOp := acl.NamespaceValidator(acl.NamespaceCapabilityReadFS)
+	aclObj, err := f.srv.ResolveACL(args)
+	if err != nil {
+		return err
+	} else if !allowNsOp(aclObj, alloc.Namespace) {
+		return structs.ErrPermissionDenied
+	}
+
+	// Make sure Node is valid and new enough to support RPC
+	_, err = getNodeForRpc(snap, alloc.NodeID)
+	if err != nil {
+		return err
+	}
+
+	// Get the connection to the client
+	state, ok := f.srv.getNodeConn(alloc.NodeID)
+	if !ok {
+		return findNodeConnAndForward(f.srv, alloc.NodeID, "FileSystem.List", args, reply)
+	}
+
+	// Make the RPC
+	return NodeRpc(state.Session, "FileSystem.List", args, reply)
+}
+
+// Stat is used to stat a file in the allocation's directory.
+func (f *FileSystem) Stat(args *cstructs.FsStatRequest, reply *cstructs.FsStatResponse) error {
+	// We only allow stale reads since the only potentially stale information is
+	// the Node registration and the cost is fairly high for adding another hope
+	// in the forwarding chain.
+	args.QueryOptions.AllowStale = true
+
+	authErr := f.srv.Authenticate(nil, args)
+
+	// Potentially forward to a different region.
+	if done, err := f.srv.forward("FileSystem.Stat", args, args, reply); done {
+		return err
+	}
+	f.srv.MeasureRPCRate("file_system", structs.RateMetricRead, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "file_system", "stat"}, time.Now())
+
+	// Verify the arguments.
+	if args.AllocID == "" {
+		return errors.New("missing allocation ID")
+	}
+
+	// Lookup the allocation
+	snap, err := f.srv.State().Snapshot()
+	if err != nil {
+		return err
+	}
+
+	alloc, err := getAlloc(snap, args.AllocID)
+	if err != nil {
+		return err
+	}
+
+	// Check filesystem read permissions
+	if aclObj, err := f.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNsOp(alloc.Namespace, acl.NamespaceCapabilityReadFS) {
+		return structs.ErrPermissionDenied
+	}
+
+	// Make sure Node is valid and new enough to support RPC
+	_, err = getNodeForRpc(snap, alloc.NodeID)
+	if err != nil {
+		return err
+	}
+
+	// Get the connection to the client
+	state, ok := f.srv.getNodeConn(alloc.NodeID)
+	if !ok {
+		return findNodeConnAndForward(f.srv, alloc.NodeID, "FileSystem.Stat", args, reply)
+	}
+
+	// Make the RPC
+	return NodeRpc(state.Session, "FileSystem.Stat", args, reply)
+}
+
+// stream is is used to stream the contents of file in an allocation's
+// directory.
+func (f *FileSystem) stream(conn io.ReadWriteCloser) {
+	defer conn.Close()
+	defer metrics.MeasureSince([]string{"nomad", "file_system", "stream"}, time.Now())
+
+	// Decode the arguments
+	var args cstructs.FsStreamRequest
+	decoder := codec.NewDecoder(conn, structs.MsgpackHandle)
+	encoder := codec.NewEncoder(conn, structs.MsgpackHandle)
+
+	if err := decoder.Decode(&args); err != nil {
+		handleStreamResultError(err, pointer.Of(int64(500)), encoder)
+		return
+	}
+
+	authErr := f.srv.Authenticate(nil, &args)
+
+	// Check if we need to forward to a different region
+	if r := args.RequestRegion(); r != f.srv.Region() {
+		forwardRegionStreamingRpc(f.srv, conn, encoder, &args, "FileSystem.Stream",
+			args.AllocID, &args.QueryOptions)
+		return
+	}
+	f.srv.MeasureRPCRate("file_system", structs.RateMetricRead, &args)
+	if authErr != nil {
+		handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
+		return
+	}
+
+	// Verify the arguments.
+	if args.AllocID == "" {
+		handleStreamResultError(errors.New("missing AllocID"), pointer.Of(int64(400)), encoder)
+		return
+	}
+
+	// Retrieve the allocation
+	snap, err := f.srv.State().Snapshot()
+	if err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	}
+
+	alloc, err := getAlloc(snap, args.AllocID)
+	if structs.IsErrUnknownAllocation(err) {
+		handleStreamResultError(structs.NewErrUnknownAllocation(args.AllocID), pointer.Of(int64(404)), encoder)
+		return
+	}
+	if err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	}
+
+	// Check namespace read-fs permissions.
+	if aclObj, err := f.srv.ResolveACL(&args); err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	} else if !aclObj.AllowNsOp(alloc.Namespace, acl.NamespaceCapabilityReadFS) {
+		handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
+		return
+	}
+
+	nodeID := alloc.NodeID
+
+	// Make sure Node is valid and new enough to support RPC
+	node, err := snap.NodeByID(nil, nodeID)
+	if err != nil {
+		handleStreamResultError(err, pointer.Of(int64(500)), encoder)
+		return
+	}
+
+	if node == nil {
+		err := fmt.Errorf("Unknown node %q", nodeID)
+		handleStreamResultError(err, pointer.Of(int64(400)), encoder)
+		return
+	}
+
+	if err := nodeSupportsRpc(node); err != nil {
+		handleStreamResultError(err, pointer.Of(int64(400)), encoder)
+		return
+	}
+
+	// Get the connection to the client either by forwarding to another server
+	// or creating a direct stream
+	var clientConn net.Conn
+	state, ok := f.srv.getNodeConn(nodeID)
+	if !ok {
+		// Determine the Server that has a connection to the node.
+		srv, err := f.srv.serverWithNodeConn(nodeID, f.srv.Region())
+		if err != nil {
+			var code *int64
+			if structs.IsErrNoNodeConn(err) {
+				code = pointer.Of(int64(404))
+			}
+			handleStreamResultError(err, code, encoder)
+			return
+		}
+
+		// Get a connection to the server
+		conn, err := f.srv.streamingRpc(srv, "FileSystem.Stream")
+		if err != nil {
+			handleStreamResultError(err, nil, encoder)
+			return
+		}
+
+		clientConn = conn
+	} else {
+		stream, err := NodeStreamingRpc(state.Session, "FileSystem.Stream")
+		if err != nil {
+			handleStreamResultError(err, nil, encoder)
+			return
+		}
+		clientConn = stream
+	}
+	defer clientConn.Close()
+
+	// Send the request.
+	outEncoder := codec.NewEncoder(clientConn, structs.MsgpackHandle)
+	if err := outEncoder.Encode(args); err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	}
+
+	structs.Bridge(conn, clientConn)
+}
+
+// logs is used to access an task's logs for a given allocation
+func (f *FileSystem) logs(conn io.ReadWriteCloser) {
+	defer conn.Close()
+	defer metrics.MeasureSince([]string{"nomad", "file_system", "logs"}, time.Now())
+
+	// Decode the arguments
+	var args cstructs.FsLogsRequest
+	decoder := codec.NewDecoder(conn, structs.MsgpackHandle)
+	encoder := codec.NewEncoder(conn, structs.MsgpackHandle)
+
+	if err := decoder.Decode(&args); err != nil {
+		handleStreamResultError(err, pointer.Of(int64(500)), encoder)
+		return
+	}
+
+	authErr := f.srv.Authenticate(nil, &args)
+
+	// Check if we need to forward to a different region
+	if r := args.RequestRegion(); r != f.srv.Region() {
+		forwardRegionStreamingRpc(f.srv, conn, encoder, &args, "FileSystem.Logs",
+			args.AllocID, &args.QueryOptions)
+		return
+	}
+	f.srv.MeasureRPCRate("file_system", structs.RateMetricRead, &args)
+	if authErr != nil {
+		handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
+		return
+	}
+
+	// Verify the arguments.
+	if args.AllocID == "" {
+		handleStreamResultError(structs.ErrMissingAllocID, pointer.Of(int64(400)), encoder)
+		return
+	}
+
+	// Retrieve the allocation
+	snap, err := f.srv.State().Snapshot()
+	if err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	}
+
+	alloc, err := getAlloc(snap, args.AllocID)
+	if structs.IsErrUnknownAllocation(err) {
+		handleStreamResultError(structs.NewErrUnknownAllocation(args.AllocID), pointer.Of(int64(404)), encoder)
+		return
+	}
+	if err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	}
+
+	// Check namespace read-logs *or* read-fs permissions.
+	allowNsOp := acl.NamespaceValidator(
+		acl.NamespaceCapabilityReadFS, acl.NamespaceCapabilityReadLogs)
+	aclObj, err := f.srv.ResolveACL(&args)
+	if err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	} else if !allowNsOp(aclObj, alloc.Namespace) {
+		handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
+		return
+	}
+
+	nodeID := alloc.NodeID
+
+	// Make sure Node is valid and new enough to support RPC
+	node, err := snap.NodeByID(nil, nodeID)
+	if err != nil {
+		handleStreamResultError(err, pointer.Of(int64(500)), encoder)
+		return
+	}
+
+	if node == nil {
+		err := fmt.Errorf("Unknown node %q", nodeID)
+		handleStreamResultError(err, pointer.Of(int64(400)), encoder)
+		return
+	}
+
+	if err := nodeSupportsRpc(node); err != nil {
+		handleStreamResultError(err, pointer.Of(int64(400)), encoder)
+		return
+	}
+
+	// Get the connection to the client either by forwarding to another server
+	// or creating a direct stream
+	var clientConn net.Conn
+	state, ok := f.srv.getNodeConn(nodeID)
+	if !ok {
+		// Determine the Server that has a connection to the node.
+		srv, err := f.srv.serverWithNodeConn(nodeID, f.srv.Region())
+		if err != nil {
+			var code *int64
+			if structs.IsErrNoNodeConn(err) {
+				code = pointer.Of(int64(404))
+			}
+			handleStreamResultError(err, code, encoder)
+			return
+		}
+
+		// Get a connection to the server
+		conn, err := f.srv.streamingRpc(srv, "FileSystem.Logs")
+		if err != nil {
+			handleStreamResultError(err, nil, encoder)
+			return
+		}
+
+		clientConn = conn
+	} else {
+		stream, err := NodeStreamingRpc(state.Session, "FileSystem.Logs")
+		if err != nil {
+			handleStreamResultError(err, nil, encoder)
+			return
+		}
+		clientConn = stream
+	}
+	defer clientConn.Close()
+
+	// Send the request.
+	outEncoder := codec.NewEncoder(clientConn, structs.MsgpackHandle)
+	if err := outEncoder.Encode(args); err != nil {
+		handleStreamResultError(err, nil, encoder)
+		return
+	}
+
+	structs.Bridge(conn, clientConn)
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/client_host_volume_endpoint.go

@@ -0,0 +1,96 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"fmt"
+	"time"
+
+	log "github.com/hashicorp/go-hclog"
+	metrics "github.com/hashicorp/go-metrics/compat"
+	cstructs "github.com/hashicorp/nomad/client/structs"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+// ClientHostVolume is the client RPC endpoint for host volumes
+type ClientHostVolume struct {
+	srv    *Server
+	ctx    *RPCContext
+	logger log.Logger
+}
+
+func NewClientHostVolumeEndpoint(srv *Server, ctx *RPCContext) *ClientHostVolume {
+	return &ClientHostVolume{srv: srv, ctx: ctx, logger: srv.logger.Named("client_host_volume")}
+}
+
+func (c *ClientHostVolume) Create(args *cstructs.ClientHostVolumeCreateRequest, reply *cstructs.ClientHostVolumeCreateResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_host_node", "create"}, time.Now())
+	return c.sendVolumeRPC(
+		args.NodeID,
+		"HostVolume.Create",
+		"ClientHostVolume.Create",
+		structs.RateMetricWrite,
+		args,
+		reply,
+	)
+}
+
+func (c *ClientHostVolume) Register(args *cstructs.ClientHostVolumeRegisterRequest, reply *cstructs.ClientHostVolumeRegisterResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_host_node", "register"}, time.Now())
+	return c.sendVolumeRPC(
+		args.NodeID,
+		"HostVolume.Register",
+		"ClientHostVolume.Register",
+		structs.RateMetricWrite,
+		args,
+		reply,
+	)
+}
+
+func (c *ClientHostVolume) Delete(args *cstructs.ClientHostVolumeDeleteRequest, reply *cstructs.ClientHostVolumeDeleteResponse) error {
+	defer metrics.MeasureSince([]string{"nomad", "client_host_volume", "delete"}, time.Now())
+	return c.sendVolumeRPC(
+		args.NodeID,
+		"HostVolume.Delete",
+		"ClientHostVolume.Delete",
+		structs.RateMetricWrite,
+		args,
+		reply,
+	)
+}
+
+func (c *ClientHostVolume) sendVolumeRPC(nodeID, method, fwdMethod, op string, args any, reply any) error {
+	// client requests aren't RequestWithIdentity, so we use a placeholder here
+	// to populate the identity data for metrics
+	identityReq := &structs.GenericRequest{}
+	aclObj, err := c.srv.AuthenticateServerOnly(c.ctx, identityReq)
+	c.srv.MeasureRPCRate("client_host_volume", op, identityReq)
+
+	if err != nil || !aclObj.AllowServerOp() {
+		return structs.ErrPermissionDenied
+	}
+
+	// Make sure Node is valid and new enough to support RPC
+	snap, err := c.srv.State().Snapshot()
+	if err != nil {
+		return err
+	}
+
+	_, err = getNodeForRpc(snap, nodeID)
+	if err != nil {
+		return err
+	}
+
+	// Get the connection to the client
+	state, ok := c.srv.getNodeConn(nodeID)
+	if !ok {
+		return findNodeConnAndForward(c.srv, nodeID, fwdMethod, args, reply)
+	}
+
+	// Make the RPC
+	if err := NodeRpc(state.Session, method, args, reply); err != nil {
+		return fmt.Errorf("%s error: %w", method, err)
+	}
+	return nil
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/client_identity_endpoint.go

@@ -0,0 +1,73 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"time"
+
+	metrics "github.com/hashicorp/go-metrics/compat"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+type NodeIdentity struct {
+	srv *Server
+}
+
+func newNodeIdentityEndpoint(srv *Server) *NodeIdentity {
+	return &NodeIdentity{
+		srv: srv,
+	}
+}
+
+func (n *NodeIdentity) Get(args *structs.NodeIdentityGetReq, reply *structs.NodeIdentityGetResp) error {
+
+	// Prevent infinite loop between the leader and the follower with the target
+	// node connection.
+	args.QueryOptions.AllowStale = true
+
+	authErr := n.srv.Authenticate(nil, args)
+	if done, err := n.srv.forward(structs.NodeIdentityGetRPCMethod, args, args, reply); done {
+		return err
+	}
+	n.srv.MeasureRPCRate("client_identity", structs.RateMetricRead, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "client_identity", "get"}, time.Now())
+
+	// Check node read permissions
+	if aclObj, err := n.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNodeRead() {
+		return structs.ErrPermissionDenied
+	}
+
+	return n.srv.forwardClientRPC(structs.NodeIdentityGetRPCMethod, args.NodeID, args, reply)
+}
+
+func (n *NodeIdentity) Renew(args *structs.NodeIdentityRenewReq, reply *structs.NodeIdentityRenewResp) error {
+
+	// Prevent infinite loop between the leader and the follower with the target
+	// node connection.
+	args.QueryOptions.AllowStale = true
+
+	authErr := n.srv.Authenticate(nil, args)
+	if done, err := n.srv.forward(structs.NodeIdentityRenewRPCMethod, args, args, reply); done {
+		return err
+	}
+	n.srv.MeasureRPCRate("client_identity", structs.RateMetricWrite, args)
+	if authErr != nil {
+		return structs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "client_identity", "renew"}, time.Now())
+
+	// Check node write permissions
+	if aclObj, err := n.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNodeWrite() {
+		return structs.ErrPermissionDenied
+	}
+
+	return n.srv.forwardClientRPC(structs.NodeIdentityRenewRPCMethod, args.NodeID, args, reply)
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/client_identity_endpoint_test.go

@@ -0,0 +1,155 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"testing"
+
+	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/client"
+	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/testutil"
+	"github.com/shoenig/test/must"
+)
+
+func TestNodeIdentity_Get_Forward(t *testing.T) {
+	ci.Parallel(t)
+
+	servers := []*Server{}
+	for range 3 {
+		s, cleanup := TestServer(t, func(c *Config) {
+			c.BootstrapExpect = 3
+			c.NumSchedulers = 0
+		})
+		t.Cleanup(cleanup)
+		servers = append(servers, s)
+	}
+
+	TestJoin(t, servers...)
+	leader := testutil.WaitForLeaders(t, servers[0].RPC, servers[1].RPC, servers[2].RPC)
+
+	followers := []string{}
+	for _, s := range servers {
+		if addr := s.config.RPCAddr.String(); addr != leader {
+			followers = append(followers, addr)
+		}
+	}
+	t.Logf("leader=%s followers=%q", leader, followers)
+
+	clients := make([]*client.Client, 4)
+
+	for i := range 4 {
+		c, cleanup := client.TestClient(t, func(c *config.Config) {
+			c.Servers = followers
+		})
+		t.Cleanup(func() { _ = cleanup() })
+		clients[i] = c
+	}
+	for _, c := range clients {
+		testutil.WaitForClient(t, servers[0].RPC, c.NodeID(), c.Region())
+	}
+
+	agentRPCs := []func(string, any, any) error{}
+	nodeIDs := make([]string, 0, len(clients))
+
+	// Build list of agents and node IDs
+	for _, s := range servers {
+		agentRPCs = append(agentRPCs, s.RPC)
+	}
+
+	for _, c := range clients {
+		agentRPCs = append(agentRPCs, c.RPC)
+		nodeIDs = append(nodeIDs, c.NodeID())
+	}
+
+	// Iterate through all the agent RPCs to ensure that the renew RPC will
+	// succeed, no matter which agent we connect to.
+	for _, agentRPC := range agentRPCs {
+		for _, nodeID := range nodeIDs {
+			args := &structs.NodeIdentityGetReq{
+				NodeID: nodeID,
+				QueryOptions: structs.QueryOptions{
+					Region: clients[0].Region(),
+				},
+			}
+			must.NoError(t,
+				agentRPC(structs.NodeIdentityGetRPCMethod,
+					args,
+					&structs.NodeIdentityGetResp{},
+				),
+			)
+		}
+	}
+}
+
+func TestNodeIdentity_Renew_Forward(t *testing.T) {
+	ci.Parallel(t)
+
+	servers := []*Server{}
+	for i := 0; i < 3; i++ {
+		s, cleanup := TestServer(t, func(c *Config) {
+			c.BootstrapExpect = 3
+			c.NumSchedulers = 0
+		})
+		t.Cleanup(cleanup)
+		servers = append(servers, s)
+	}
+
+	TestJoin(t, servers...)
+	leader := testutil.WaitForLeaders(t, servers[0].RPC, servers[1].RPC, servers[2].RPC)
+
+	followers := []string{}
+	for _, s := range servers {
+		if addr := s.config.RPCAddr.String(); addr != leader {
+			followers = append(followers, addr)
+		}
+	}
+	t.Logf("leader=%s followers=%q", leader, followers)
+
+	clients := make([]*client.Client, 4)
+
+	for i := 0; i < 4; i++ {
+		c, cleanup := client.TestClient(t, func(c *config.Config) {
+			c.Servers = followers
+		})
+		t.Cleanup(func() { _ = cleanup() })
+		clients[i] = c
+	}
+	for _, c := range clients {
+		testutil.WaitForClient(t, servers[0].RPC, c.NodeID(), c.Region())
+	}
+
+	agentRPCs := []func(string, any, any) error{}
+	nodeIDs := make([]string, 0, len(clients))
+
+	// Build list of agents and node IDs
+	for _, s := range servers {
+		agentRPCs = append(agentRPCs, s.RPC)
+	}
+
+	for _, c := range clients {
+		agentRPCs = append(agentRPCs, c.RPC)
+		nodeIDs = append(nodeIDs, c.NodeID())
+	}
+
+	// Iterate through all the agent RPCs to ensure that the renew RPC will
+	// succeed, no matter which agent we connect to.
+	for _, agentRPC := range agentRPCs {
+		for _, nodeID := range nodeIDs {
+			args := &structs.NodeIdentityRenewReq{
+				NodeID: nodeID,
+				QueryOptions: structs.QueryOptions{
+					Region: clients[0].Region(),
+				},
+			}
+			must.NoError(t,
+				agentRPC(structs.NodeIdentityRenewRPCMethod,
+					args,
+					&structs.NodeIdentityRenewResp{},
+				),
+			)
+		}
+	}
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/client_meta_endpoint.go

@@ -0,0 +1,80 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"time"
+
+	log "github.com/hashicorp/go-hclog"
+	metrics "github.com/hashicorp/go-metrics/compat"
+	"github.com/hashicorp/nomad/nomad/structs"
+	nstructs "github.com/hashicorp/nomad/nomad/structs"
+)
+
+type NodeMeta struct {
+	srv    *Server
+	logger log.Logger
+}
+
+func newNodeMetaEndpoint(srv *Server) *NodeMeta {
+	n := &NodeMeta{
+		srv:    srv,
+		logger: srv.logger.Named("node_meta"),
+	}
+	return n
+}
+
+func (n *NodeMeta) Apply(args *structs.NodeMetaApplyRequest, reply *structs.NodeMetaResponse) error {
+	const method = "NodeMeta.Apply"
+
+	// Prevent infinite loop between leader and
+	// follower-with-the-target-node-connection.
+	args.QueryOptions.AllowStale = true
+
+	authErr := n.srv.Authenticate(nil, args)
+	if done, err := n.srv.forward(method, args, args, reply); done {
+		return err
+	}
+	n.srv.MeasureRPCRate("node_meta", nstructs.RateMetricRead, args)
+	if authErr != nil {
+		return nstructs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "client_meta", "apply"}, time.Now())
+
+	// Check node write permissions
+	if aclObj, err := n.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNodeWrite() {
+		return nstructs.ErrPermissionDenied
+	}
+
+	return n.srv.forwardClientRPC(method, args.NodeID, args, reply)
+}
+
+func (n *NodeMeta) Read(args *structs.NodeSpecificRequest, reply *structs.NodeMetaResponse) error {
+	const method = "NodeMeta.Read"
+
+	// Prevent infinite loop between leader and
+	// follower-with-the-target-node-connection.
+	args.QueryOptions.AllowStale = true
+
+	authErr := n.srv.Authenticate(nil, args)
+	if done, err := n.srv.forward(method, args, args, reply); done {
+		return err
+	}
+	n.srv.MeasureRPCRate("node_meta", nstructs.RateMetricRead, args)
+	if authErr != nil {
+		return nstructs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "client_meta", "read"}, time.Now())
+
+	// Check node read permissions
+	if aclObj, err := n.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNodeRead() {
+		return nstructs.ErrPermissionDenied
+	}
+
+	return n.srv.forwardClientRPC(method, args.NodeID, args, reply)
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/client_meta_endpoint_test.go

@@ -0,0 +1,132 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"testing"
+
+	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/client"
+	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/helper/pointer"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/testutil"
+	"github.com/shoenig/test/must"
+)
+
+// TestNodeMeta_Forward asserts that Client RPCs do not result in infinite
+// loops. For example in a cluster with 1 Leader, 2 Followers, and a Node
+// connected to Follower 1:
+//
+// If a NodeMeta.Apply RPC with AllowStale=false is received by Follower 1, it
+// will honor AllowStale=false and forward the request to the Leader.
+//
+// The Leader will accept the RPC, notice that Follower 1 has a connection to
+// the Node, and the Leader will send the request back to Follower 1.
+//
+// Follower 1, ever respectful of AllowStale=false, will forward it back to the
+// Leader.
+//
+// The Leader, being unable to forward to the Node, will send it back to
+// Follower 1.
+//
+// This argument will continue until one of the Servers runs out of memory or
+// patience and stomps away in anger (crashes). Like any good argument the
+// ending is never pretty as the Servers will suffer CPU starvation and
+// potentially Raft flapping before anyone actually OOMs.
+//
+// See https://github.com/hashicorp/nomad/issues/16517 for details.
+//
+// If test fails it will do so spectacularly by consuming all available CPU and
+// potentially all available memory. Running it in a VM or container is
+// suggested.
+func TestNodeMeta_Forward(t *testing.T) {
+	ci.Parallel(t)
+
+	servers := []*Server{}
+	for i := 0; i < 3; i++ {
+		s, cleanup := TestServer(t, func(c *Config) {
+			c.BootstrapExpect = 3
+			c.NumSchedulers = 0
+		})
+		t.Cleanup(cleanup)
+		servers = append(servers, s)
+	}
+
+	TestJoin(t, servers...)
+	leader := testutil.WaitForLeaders(t, servers[0].RPC, servers[1].RPC, servers[2].RPC)
+
+	followers := []string{}
+	for _, s := range servers {
+		if addr := s.config.RPCAddr.String(); addr != leader {
+			followers = append(followers, addr)
+		}
+	}
+	t.Logf("leader=%s followers=%q", leader, followers)
+
+	clients := []*client.Client{}
+	for i := 0; i < 4; i++ {
+		c, cleanup := client.TestClient(t, func(c *config.Config) {
+			// Clients will rebalance across all servers, but try to get them to use
+			// followers to ensure we don't hit the loop in #16517
+			c.Servers = followers
+		})
+		defer cleanup()
+		clients = append(clients, c)
+	}
+	for _, c := range clients {
+		testutil.WaitForClient(t, servers[0].RPC, c.NodeID(), c.Region())
+	}
+
+	agentRPCs := []func(string, any, any) error{}
+	nodeIDs := make([]string, 0, len(clients))
+
+	// Build list of agents and node IDs
+	for _, s := range servers {
+		agentRPCs = append(agentRPCs, s.RPC)
+	}
+
+	for _, c := range clients {
+		agentRPCs = append(agentRPCs, c.RPC)
+		nodeIDs = append(nodeIDs, c.NodeID())
+	}
+
+	region := clients[0].Region()
+
+	// Apply metadata to every client through every agent to ensure forwarding
+	// always works regardless of path taken.
+	for _, rpc := range agentRPCs {
+		for _, nodeID := range nodeIDs {
+			args := &structs.NodeMetaApplyRequest{
+				// Intentionally don't set QueryOptions.AllowStale to exercise #16517
+				QueryOptions: structs.QueryOptions{
+					Region: region,
+				},
+				NodeID: nodeID,
+				Meta:   map[string]*string{"testing": pointer.Of("123")},
+			}
+			reply := &structs.NodeMetaResponse{}
+			must.NoError(t, rpc("NodeMeta.Apply", args, reply))
+			must.MapNotEmpty(t, reply.Meta)
+		}
+	}
+
+	for _, rpc := range agentRPCs {
+		for _, nodeID := range nodeIDs {
+			args := &structs.NodeSpecificRequest{
+				// Intentionally don't set QueryOptions.AllowStale to exercise #16517
+				QueryOptions: structs.QueryOptions{
+					Region: region,
+				},
+				NodeID: nodeID,
+			}
+			reply := &structs.NodeMetaResponse{}
+			must.NoError(t, rpc("NodeMeta.Read", args, reply))
+			must.MapNotEmpty(t, reply.Meta)
+			must.Eq(t, reply.Meta["testing"], "123")
+			must.MapNotEmpty(t, reply.Dynamic)
+			must.Eq(t, *reply.Dynamic["testing"], "123")
+		}
+	}
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/client_rpc.go

@@ -0,0 +1,323 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/hashicorp/go-msgpack/v2/codec"
+	multierror "github.com/hashicorp/go-multierror"
+	msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc/v2"
+	"github.com/hashicorp/nomad/helper/pool"
+	"github.com/hashicorp/nomad/nomad/peers"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/yamux"
+)
+
+// nodeConnState is used to track connection information about a Nomad Client.
+type nodeConnState struct {
+	// Session holds the multiplexed yamux Session for dialing back.
+	Session *yamux.Session
+
+	// Established is when the connection was established.
+	Established time.Time
+
+	// Ctx is the full RPC context
+	Ctx *RPCContext
+}
+
+// getNodeConn returns the connection to the given node and whether it exists.
+func (s *Server) getNodeConn(nodeID string) (*nodeConnState, bool) {
+	s.nodeConnsLock.RLock()
+	defer s.nodeConnsLock.RUnlock()
+	conns, ok := s.nodeConns[nodeID]
+	if !ok {
+		return nil, false
+	}
+
+	// Return the latest conn
+	var state *nodeConnState
+	for _, conn := range conns {
+		if state == nil || state.Established.Before(conn.Established) {
+			state = conn
+		}
+	}
+
+	// Shouldn't happen but rather be safe
+	if state == nil {
+		s.logger.Named("client_rpc").Warn("node exists in node connection map without any connection", "node_id", nodeID)
+		return nil, false
+	}
+
+	return state, ok
+}
+
+// connectedNodes returns the set of nodes we have a connection with.
+func (s *Server) connectedNodes() map[string]time.Time {
+	s.nodeConnsLock.RLock()
+	defer s.nodeConnsLock.RUnlock()
+	nodes := make(map[string]time.Time, len(s.nodeConns))
+	for nodeID, conns := range s.nodeConns {
+		for _, conn := range conns {
+			if nodes[nodeID].Before(conn.Established) {
+				nodes[nodeID] = conn.Established
+			}
+		}
+	}
+	return nodes
+}
+
+// addNodeConn adds the mapping between a node and its session.
+func (s *Server) addNodeConn(ctx *RPCContext) {
+	// Hotpath the no-op
+	if ctx == nil || ctx.NodeID == "" {
+		return
+	}
+
+	s.nodeConnsLock.Lock()
+	defer s.nodeConnsLock.Unlock()
+
+	// Capture the tracked connections so far
+	currentConns := s.nodeConns[ctx.NodeID]
+
+	// Check if we already have the connection. If we do, just update the
+	// establish time.
+	for _, c := range currentConns {
+		if c.Ctx.Conn.LocalAddr().String() == ctx.Conn.LocalAddr().String() &&
+			c.Ctx.Conn.RemoteAddr().String() == ctx.Conn.RemoteAddr().String() {
+			c.Established = time.Now()
+			return
+		}
+	}
+
+	// Add the new conn
+	s.nodeConns[ctx.NodeID] = append(s.nodeConns[ctx.NodeID], &nodeConnState{
+		Session:     ctx.Session,
+		Established: time.Now(),
+		Ctx:         ctx,
+	})
+}
+
+// removeNodeConn removes the mapping between a node and its session.
+func (s *Server) removeNodeConn(ctx *RPCContext) {
+	// Hotpath the no-op
+	if ctx == nil || ctx.NodeID == "" {
+		return
+	}
+
+	s.nodeConnsLock.Lock()
+	defer s.nodeConnsLock.Unlock()
+	conns, ok := s.nodeConns[ctx.NodeID]
+	if !ok {
+		return
+	}
+
+	// It is important that we check that the connection being removed is the
+	// actual stored connection for the client. It is possible for the client to
+	// dial various addresses that all route to the same server. The most common
+	// case for this is the original address the client uses to connect to the
+	// server differs from the advertised address sent by the heartbeat.
+	for i, conn := range conns {
+		if conn.Ctx.Conn.LocalAddr().String() == ctx.Conn.LocalAddr().String() &&
+			conn.Ctx.Conn.RemoteAddr().String() == ctx.Conn.RemoteAddr().String() {
+
+			if len(conns) == 1 {
+				// We are deleting the last conn, remove it from the map
+				delete(s.nodeConns, ctx.NodeID)
+			} else {
+				// Slice out the connection we are deleting
+				s.nodeConns[ctx.NodeID] = append(s.nodeConns[ctx.NodeID][:i], s.nodeConns[ctx.NodeID][i+1:]...)
+			}
+
+			return
+		}
+	}
+}
+
+// serverWithNodeConn is used to determine which remote server has the most
+// recent connection to the given node. The local server is not queried.
+// ErrNoNodeConn is returned if all local peers could be queried but did not
+// have a connection to the node. Otherwise if a connection could not be found
+// and there were RPC errors, an error is returned.
+func (s *Server) serverWithNodeConn(nodeID, region string) (*peers.Parts, error) {
+	// We skip ourselves.
+	selfAddr := s.LocalMember().Addr.String()
+
+	// Build the request
+	req := &structs.NodeSpecificRequest{
+		NodeID: nodeID,
+		QueryOptions: structs.QueryOptions{
+			Region: s.config.Region,
+		},
+	}
+
+	// Select the list of servers to check based on what region we are querying
+	var targets []*peers.Parts
+	if region == s.Region() {
+		targets = s.peersCache.LocalPeers()
+	} else {
+		targets = s.peersCache.RegionPeers(region)
+		if targets == nil {
+			return nil, structs.ErrNoRegionPath
+		}
+	}
+
+	// connections is used to store the servers that have connections to the
+	// requested node.
+	var mostRecentServer *peers.Parts
+	var mostRecent time.Time
+
+	var rpcErr multierror.Error
+	for _, server := range targets {
+		if server.Addr.String() == selfAddr {
+			continue
+		}
+
+		// Make the RPC
+		var resp structs.NodeConnQueryResponse
+		err := s.connPool.RPC(s.config.Region, server.Addr, "Status.HasNodeConn", &req, &resp)
+		if err != nil {
+			multierror.Append(&rpcErr, fmt.Errorf("failed querying server %q: %v", server.Addr.String(), err))
+			continue
+		}
+
+		if resp.Connected && resp.Established.After(mostRecent) {
+			mostRecentServer = server
+			mostRecent = resp.Established
+		}
+	}
+
+	// Return an error if there is no route to the node.
+	if mostRecentServer == nil {
+		if err := rpcErr.ErrorOrNil(); err != nil {
+			return nil, err
+		}
+
+		return nil, structs.ErrNoNodeConn
+	}
+
+	return mostRecentServer, nil
+}
+
+// forwardClientRPC forwards the RPC specified by method to the node specified
+// by nodeID. Must be done after region forwarding, metrics, and permissions
+// checks.
+//
+// This is a wrapper method for getNodeForRpc, getNodeConn, etc that Client
+// RPCs which only need Servers to forward requests can use.
+func (s *Server) forwardClientRPC(method, nodeID string, args, reply any) error {
+	if nodeID == "" {
+		return errors.New("missing NodeID")
+	}
+
+	// Check if the node even exists and is compatible with NodeRpc
+	snap, err := s.State().Snapshot()
+	if err != nil {
+		return err
+	}
+
+	// Make sure Node is new enough to support RPC
+	_, err = getNodeForRpc(snap, nodeID)
+	if err != nil {
+		return err
+	}
+
+	// Get the connection to the client
+	state, ok := s.getNodeConn(nodeID)
+	if !ok {
+		// Make the RPC via another server
+		return findNodeConnAndForward(s, nodeID, method, args, reply)
+	}
+
+	// Make the RPC
+	return NodeRpc(state.Session, method, args, reply)
+}
+
+// NodeRpc is used to make an RPC call to a node. The method takes the
+// Yamux session for the node and the method to be called.
+func NodeRpc(session *yamux.Session, method string, args, reply interface{}) error {
+	// Open a new session
+	stream, err := session.Open()
+	if err != nil {
+		return fmt.Errorf("session open: %v", err)
+	}
+	defer stream.Close()
+
+	// Write the RpcNomad byte to set the mode
+	if _, err := stream.Write([]byte{byte(pool.RpcNomad)}); err != nil {
+		stream.Close()
+		return fmt.Errorf("set mode: %v", err)
+	}
+
+	// Make the RPC
+	err = msgpackrpc.CallWithCodec(pool.NewClientCodec(stream), method, args, reply)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// NodeStreamingRpc is used to make a streaming RPC call to a node. The method
+// takes the Yamux session for the node and the method to be called. It conducts
+// the initial handshake and returns a connection to be used or an error. It is
+// the callers responsibility to close the connection if there is no error.
+func NodeStreamingRpc(session *yamux.Session, method string) (net.Conn, error) {
+	// Open a new session
+	stream, err := session.Open()
+	if err != nil {
+		return nil, err
+	}
+
+	// Write the RpcNomad byte to set the mode
+	if _, err := stream.Write([]byte{byte(pool.RpcStreaming)}); err != nil {
+		stream.Close()
+		return nil, err
+	}
+
+	// Send the header
+	encoder := codec.NewEncoder(stream, structs.MsgpackHandle)
+	decoder := codec.NewDecoder(stream, structs.MsgpackHandle)
+	header := structs.StreamingRpcHeader{
+		Method: method,
+	}
+	if err := encoder.Encode(header); err != nil {
+		stream.Close()
+		return nil, err
+	}
+
+	// Wait for the acknowledgement
+	var ack structs.StreamingRpcAck
+	if err := decoder.Decode(&ack); err != nil {
+		stream.Close()
+		return nil, err
+	}
+
+	if ack.Error != "" {
+		stream.Close()
+		return nil, errors.New(ack.Error)
+	}
+
+	return stream, nil
+}
+
+// findNodeConnAndForward is a helper for finding the server with a connection
+// to the given node and forwarding the RPC to the correct server. This does not
+// work for streaming RPCs.
+func findNodeConnAndForward(srv *Server, nodeID, method string, args, reply interface{}) error {
+	// Determine the Server that has a connection to the node.
+	srvWithConn, err := srv.serverWithNodeConn(nodeID, srv.Region())
+	if err != nil {
+		return err
+	}
+
+	if srvWithConn == nil {
+		return structs.ErrNoNodeConn
+	}
+
+	return srv.forwardServer(srvWithConn, method, args, reply)
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/client_rpc_test.go

@@ -0,0 +1,315 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"net"
+	"strings"
+	"testing"
+
+	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/client"
+	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+type namedConnWrapper struct {
+	net.Conn
+	name string
+}
+
+type namedAddr string
+
+func (n namedAddr) String() string  { return string(n) }
+func (n namedAddr) Network() string { return string(n) }
+
+func (n namedConnWrapper) LocalAddr() net.Addr {
+	return namedAddr(n.name)
+}
+
+func TestServer_removeNodeConn_differentAddrs(t *testing.T) {
+	ci.Parallel(t)
+	require := require.New(t)
+
+	s1, cleanupS1 := TestServer(t, nil)
+	defer cleanupS1()
+	testutil.WaitForLeader(t, s1.RPC)
+
+	p1, p2 := net.Pipe()
+	w1 := namedConnWrapper{
+		Conn: p1,
+		name: "a",
+	}
+	w2 := namedConnWrapper{
+		Conn: p2,
+		name: "b",
+	}
+
+	// Add the connections
+	nodeID := uuid.Generate()
+	ctx1 := &RPCContext{
+		Conn:   w1,
+		NodeID: nodeID,
+	}
+	ctx2 := &RPCContext{
+		Conn:   w2,
+		NodeID: nodeID,
+	}
+
+	s1.addNodeConn(ctx1)
+	s1.addNodeConn(ctx2)
+	require.Len(s1.connectedNodes(), 1)
+	require.Len(s1.nodeConns[nodeID], 2)
+
+	// Check that the value is the second conn.
+	state, ok := s1.getNodeConn(nodeID)
+	require.True(ok)
+	require.Equal(state.Ctx.Conn.LocalAddr().String(), w2.name)
+
+	// Delete the first
+	s1.removeNodeConn(ctx1)
+	require.Len(s1.connectedNodes(), 1)
+	require.Len(s1.nodeConns[nodeID], 1)
+
+	// Check that the value is the second conn.
+	state, ok = s1.getNodeConn(nodeID)
+	require.True(ok)
+	require.Equal(state.Ctx.Conn.LocalAddr().String(), w2.name)
+
+	// Delete the second
+	s1.removeNodeConn(ctx2)
+	require.Len(s1.connectedNodes(), 0)
+
+	_, ok = s1.getNodeConn(nodeID)
+	require.False(ok)
+}
+
+func TestServerWithNodeConn_NoPath(t *testing.T) {
+	ci.Parallel(t)
+	require := require.New(t)
+
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
+	defer cleanupS1()
+	s2, cleanupS2 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
+	defer cleanupS2()
+	TestJoin(t, s1, s2)
+	testutil.WaitForLeader(t, s1.RPC)
+	testutil.WaitForLeader(t, s2.RPC)
+
+	nodeID := uuid.Generate()
+	srv, err := s1.serverWithNodeConn(nodeID, s1.Region())
+	require.Nil(srv)
+	require.EqualError(err, structs.ErrNoNodeConn.Error())
+}
+
+func TestServerWithNodeConn_NoPath_Region(t *testing.T) {
+	ci.Parallel(t)
+	require := require.New(t)
+
+	s1, cleanupS1 := TestServer(t, nil)
+	defer cleanupS1()
+	testutil.WaitForLeader(t, s1.RPC)
+
+	nodeID := uuid.Generate()
+	srv, err := s1.serverWithNodeConn(nodeID, "fake-region")
+	require.Nil(srv)
+	require.EqualError(err, structs.ErrNoRegionPath.Error())
+}
+
+func TestServerWithNodeConn_Path(t *testing.T) {
+	ci.Parallel(t)
+	require := require.New(t)
+
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
+	defer cleanupS1()
+	s2, cleanupS2 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
+	defer cleanupS2()
+	TestJoin(t, s1, s2)
+	testutil.WaitForLeader(t, s1.RPC)
+	testutil.WaitForLeader(t, s2.RPC)
+
+	// Create a fake connection for the node on server 2
+	nodeID := uuid.Generate()
+	s2.addNodeConn(&RPCContext{
+		NodeID: nodeID,
+	})
+
+	srv, err := s1.serverWithNodeConn(nodeID, s1.Region())
+	require.NotNil(srv)
+	require.Equal(srv.Addr.String(), s2.config.RPCAddr.String())
+	require.Nil(err)
+}
+
+func TestServerWithNodeConn_Path_Region(t *testing.T) {
+	ci.Parallel(t)
+	require := require.New(t)
+
+	s1, cleanupS1 := TestServer(t, nil)
+	defer cleanupS1()
+	s2, cleanupS2 := TestServer(t, func(c *Config) {
+		c.Region = "two"
+	})
+	defer cleanupS2()
+	TestJoin(t, s1, s2)
+	testutil.WaitForLeader(t, s1.RPC)
+	testutil.WaitForLeader(t, s2.RPC)
+
+	// Create a fake connection for the node on server 2
+	nodeID := uuid.Generate()
+	s2.addNodeConn(&RPCContext{
+		NodeID: nodeID,
+	})
+
+	srv, err := s1.serverWithNodeConn(nodeID, s2.Region())
+	require.NotNil(srv)
+	require.Equal(srv.Addr.String(), s2.config.RPCAddr.String())
+	require.Nil(err)
+}
+
+func TestServerWithNodeConn_Path_Newest(t *testing.T) {
+	ci.Parallel(t)
+	require := require.New(t)
+
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
+	defer cleanupS1()
+	s2, cleanupS2 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
+	defer cleanupS2()
+	s3, cleanupS3 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
+	defer cleanupS3()
+	TestJoin(t, s1, s2, s3)
+	testutil.WaitForLeader(t, s1.RPC)
+	testutil.WaitForLeader(t, s2.RPC)
+	testutil.WaitForLeader(t, s3.RPC)
+
+	// Create a fake connection for the node on server 2 and 3
+	nodeID := uuid.Generate()
+	s2.addNodeConn(&RPCContext{
+		NodeID: nodeID,
+	})
+	s3.addNodeConn(&RPCContext{
+		NodeID: nodeID,
+	})
+
+	srv, err := s1.serverWithNodeConn(nodeID, s1.Region())
+	require.NotNil(srv)
+	require.Equal(srv.Addr.String(), s3.config.RPCAddr.String())
+	require.Nil(err)
+}
+
+func TestServerWithNodeConn_PathAndErr(t *testing.T) {
+	ci.Parallel(t)
+	require := require.New(t)
+
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
+	defer cleanupS1()
+	s2, cleanupS2 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
+	defer cleanupS2()
+	s3, cleanupS3 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
+	defer cleanupS3()
+	TestJoin(t, s1, s2, s3)
+	testutil.WaitForLeader(t, s1.RPC)
+	testutil.WaitForLeader(t, s2.RPC)
+	testutil.WaitForLeader(t, s3.RPC)
+
+	// Create a fake connection for the node on server 2
+	nodeID := uuid.Generate()
+	s2.addNodeConn(&RPCContext{
+		NodeID: nodeID,
+	})
+
+	// Shutdown the RPC layer for server 3
+	s3.rpcListener.Close()
+
+	srv, err := s1.serverWithNodeConn(nodeID, s1.Region())
+	require.NotNil(srv)
+	require.Equal(srv.Addr.String(), s2.config.RPCAddr.String())
+	require.Nil(err)
+}
+
+func TestServerWithNodeConn_NoPathAndErr(t *testing.T) {
+	ci.Parallel(t)
+	require := require.New(t)
+
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
+	defer cleanupS1()
+	s2, cleanupS2 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
+	defer cleanupS2()
+	s3, cleanupS3 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
+	defer cleanupS3()
+	TestJoin(t, s1, s2, s3)
+	testutil.WaitForLeader(t, s1.RPC)
+	testutil.WaitForLeader(t, s2.RPC)
+	testutil.WaitForLeader(t, s3.RPC)
+
+	// Shutdown the RPC layer for server 3
+	s3.rpcListener.Close()
+
+	srv, err := s1.serverWithNodeConn(uuid.Generate(), s1.Region())
+	require.Nil(srv)
+	require.NotNil(err)
+
+	// the exact error seems to be dependent on timing and raft protocol version
+	if !strings.Contains(err.Error(), "failed querying") && !strings.Contains(err.Error(), "No path to node") {
+		require.Contains(err.Error(), "failed querying")
+	}
+}
+
+func TestNodeStreamingRpc_badEndpoint(t *testing.T) {
+	ci.Parallel(t)
+	require := require.New(t)
+	s1, cleanupS1 := TestServer(t, nil)
+	defer cleanupS1()
+	testutil.WaitForLeader(t, s1.RPC)
+
+	c, cleanupC := client.TestClient(t, func(c *config.Config) {
+		c.Servers = []string{s1.config.RPCAddr.String()}
+	})
+	defer cleanupC()
+
+	// Wait for the client to connect
+	testutil.WaitForResult(func() (bool, error) {
+		nodes := s1.connectedNodes()
+		return len(nodes) == 1, nil
+	}, func(err error) {
+		t.Fatalf("should have a clients")
+	})
+
+	state, ok := s1.getNodeConn(c.NodeID())
+	require.True(ok)
+
+	conn, err := NodeStreamingRpc(state.Session, "Bogus")
+	require.Nil(conn)
+	require.NotNil(err)
+	require.Contains(err.Error(), "Bogus")
+	require.True(structs.IsErrUnknownMethod(err))
+}

bench/cve-zero/samples/GHSA-hx53-77qj-8663/source/nomad/client_stats_endpoint.go

@@ -0,0 +1,53 @@
+// Copyright IBM Corp. 2015, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package nomad
+
+import (
+	"time"
+
+	log "github.com/hashicorp/go-hclog"
+	metrics "github.com/hashicorp/go-metrics/compat"
+	nstructs "github.com/hashicorp/nomad/nomad/structs"
+
+	"github.com/hashicorp/nomad/client/structs"
+)
+
+// ClientStats is used to forward RPC requests to the targed Nomad client's
+// ClientStats endpoint.
+type ClientStats struct {
+	srv    *Server
+	logger log.Logger
+}
+
+func NewClientStatsEndpoint(srv *Server) *ClientStats {
+	return &ClientStats{srv: srv, logger: srv.logger.Named("client_stats")}
+}
+
+func (s *ClientStats) Stats(args *nstructs.NodeSpecificRequest, reply *structs.ClientStatsResponse) error {
+
+	// We only allow stale reads since the only potentially stale information is
+	// the Node registration and the cost is fairly high for adding another hope
+	// in the forwarding chain.
+	args.QueryOptions.AllowStale = true
+	authErr := s.srv.Authenticate(nil, args)
+
+	// Potentially forward to a different region.
+	if done, err := s.srv.forward("ClientStats.Stats", args, args, reply); done {
+		return err
+	}
+	s.srv.MeasureRPCRate("client_stats", nstructs.RateMetricRead, args)
+	if authErr != nil {
+		return nstructs.ErrPermissionDenied
+	}
+	defer metrics.MeasureSince([]string{"nomad", "client_stats", "stats"}, time.Now())
+
+	// Check node read permissions
+	if aclObj, err := s.srv.ResolveACL(args); err != nil {
+		return err
+	} else if !aclObj.AllowNodeRead() {
+		return nstructs.ErrPermissionDenied
+	}
+
+	return s.srv.forwardClientRPC("ClientStats.Stats", args.NodeID, args, reply)
+}