T3MP3ST public launch build

This commit is contained in:
t3mp3st contributors committed 2026-07-03 10:57:20 -04:00
1 parent 972247b881
commit 184c7d4e59
1589 files changed
+57150 -506

No files matched your search

@@ -0,0 +1,466 @@
// Copyright IBM Corp. 2015, 2025
// SPDX-License-Identifier: BUSL-1.1
package hostvolumemanager
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"time"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/go-version"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper"
)
const (
// environment variables for external plugins
EnvOperation = "DHV_OPERATION"
EnvVolumesDir = "DHV_VOLUMES_DIR"
EnvPluginDir = "DHV_PLUGIN_DIR"
EnvCreatedPath = "DHV_CREATED_PATH"
EnvNamespace = "DHV_NAMESPACE"
EnvVolumeName = "DHV_VOLUME_NAME"
EnvVolumeID = "DHV_VOLUME_ID"
EnvNodeID = "DHV_NODE_ID"
EnvNodePool = "DHV_NODE_POOL"
EnvCapacityMin = "DHV_CAPACITY_MIN_BYTES"
EnvCapacityMax = "DHV_CAPACITY_MAX_BYTES"
EnvParameters = "DHV_PARAMETERS"
)
// HostVolumePlugin manages the lifecycle of volumes.
type HostVolumePlugin interface {
Fingerprint(ctx context.Context) (*PluginFingerprint, error)
Create(ctx context.Context, req *cstructs.ClientHostVolumeCreateRequest) (*HostVolumePluginCreateResponse, error)
Delete(ctx context.Context, req *cstructs.ClientHostVolumeDeleteRequest) error
}
// PluginFingerprint gets set on the node for volume scheduling.
// Plugins are expected to respond to 'fingerprint' calls with json that
// unmarshals to this struct.
type PluginFingerprint struct {
Version *version.Version `json:"version"`
}
// HostVolumePluginCreateResponse returns values to the server that may be shown
// to the user or stored in server state. Plugins are expected to respond to
// 'create' calls with json that unmarshals to this struct.
type HostVolumePluginCreateResponse struct {
Path string `json:"path"`
SizeBytes int64 `json:"bytes"`
Error string `json:"error"`
}
// HostVolumePluginDeleteResponse returns values to the server that may be shown
// to the user or stored in server state. Plugins are expected to respond to
// 'delete' calls with json that unmarshals to this struct.
type HostVolumePluginDeleteResponse struct {
Error string `json:"error"`
}
const HostVolumePluginMkdirID = "mkdir"
const HostVolumePluginMkdirVersion = "0.0.1"
// HostVolumePluginMkdirParams represents the parameters{} that the "mkdir"
// plugin will accept.
type HostVolumePluginMkdirParams struct {
Uid int
Gid int
Mode os.FileMode
}
var _ HostVolumePlugin = &HostVolumePluginMkdir{}
// HostVolumePluginMkdir is a plugin that creates a directory within the
// specified VolumesDir. It is built-in to Nomad, so is always available.
type HostVolumePluginMkdir struct {
ID string
VolumesDir string
log hclog.Logger
}
func (p *HostVolumePluginMkdir) Fingerprint(_ context.Context) (*PluginFingerprint, error) {
v, err := version.NewVersion(HostVolumePluginMkdirVersion)
return &PluginFingerprint{
Version: v,
}, err
}
func (p *HostVolumePluginMkdir) Create(_ context.Context,
req *cstructs.ClientHostVolumeCreateRequest) (*HostVolumePluginCreateResponse, error) {
path := filepath.Join(p.VolumesDir, req.ID)
log := p.log.With(
"operation", "create",
"volume_id", req.ID,
"path", path)
log.Debug("running plugin")
resp := &HostVolumePluginCreateResponse{
Path: path,
// "mkdir" volumes, being simple directories, have unrestricted size
SizeBytes: 0,
}
if _, err := os.Stat(path); err == nil {
// already exists
return resp, nil
} else if !os.IsNotExist(err) {
// doesn't exist, but some other path error
log.Error("error with path", "error", err)
return nil, err
}
params, err := decodeMkdirParams(req.Parameters)
if err != nil {
log.Error("error with parameters", "error", err)
return nil, err
}
err = os.MkdirAll(path, params.Mode)
if err != nil {
log.Error("error creating directory", "error", err)
return nil, fmt.Errorf("error creating directory: %w", err)
}
// os.MkdirAll perms are applied after umask, so the new directory may not
// have the exact permissions requested.
err = os.Chmod(path, params.Mode)
if err != nil {
log.Error("error setting directory permission mode", "error", err)
return nil, fmt.Errorf("error setting directory permission mode: %w", err)
}
if runtime.GOOS != "windows" {
// Chown note: A uid or gid of -1 means to not change that value.
if err = os.Chown(path, params.Uid, params.Gid); err != nil {
log.Error("error changing owner/group", "error", err, "uid", params.Uid, "gid", params.Gid)
// Failing to change ownership is fatal for this plugin. Since we have
// already created the directory, we should attempt to clean it.
// Otherwise, the operator must do this manually.
if err := os.RemoveAll(path); err != nil {
log.Error("failed to remove directory after create failure",
"error", err)
}
return nil, fmt.Errorf("error changing owner/group: %w", err)
}
}
log.Debug("plugin ran successfully")
return resp, nil
}
func decodeMkdirParams(in map[string]string) (HostVolumePluginMkdirParams, error) {
// default values if their associated keys are not in the input map
out := HostVolumePluginMkdirParams{
Mode: 0o700, // "0700"
Uid: -1, // this default translates to "do not set" in os.Chown
Gid: -1, // ditto
}
var err error
for param, val := range in {
switch param {
case "mode":
// mode needs special treatment - it's octal. note that this does
// not check whether it's a *reasonable* mode for a directory.
// that will be discovered during MkdirAll and subsequent usage
// by workloads (which we cannot predict).
var number uint64
number, err = strconv.ParseUint(val, 8, 32)
out.Mode = os.FileMode(number)
case "uid":
out.Uid, err = strconv.Atoi(val)
case "gid":
out.Gid, err = strconv.Atoi(val)
default:
err = fmt.Errorf("unknown mkdir parameter: %q", param)
}
if err != nil {
return out, fmt.Errorf("invalid value for %q: %w", param, err)
}
}
return out, nil
}
func (p *HostVolumePluginMkdir) Delete(_ context.Context, req *cstructs.ClientHostVolumeDeleteRequest) error {
path := filepath.Join(p.VolumesDir, req.ID)
log := p.log.With(
"operation", "delete",
"volume_id", req.ID,
"path", path)
log.Debug("running plugin")
err := os.RemoveAll(path)
if err != nil {
log.Error("error with plugin", "error", err)
return err
}
log.Debug("plugin ran successfully")
return nil
}
var _ HostVolumePlugin = &HostVolumePluginExternal{}
// NewHostVolumePluginExternal returns an external host volume plugin
// if the specified executable exists on disk.
func NewHostVolumePluginExternal(log hclog.Logger,
pluginDir, filename, volumesDir, nodePool string) (*HostVolumePluginExternal, error) {
// this should only be called with already-detected executables,
// but we'll double-check it anyway, so we can provide a tidy error message
// if it has changed between fingerprinting and execution.
executable := filepath.Join(pluginDir, filename)
f, err := os.Stat(executable)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("%w: %q", ErrPluginNotExists, filename)
}
return nil, err
}
if !helper.IsExecutable(f) {
return nil, fmt.Errorf("%w: %q", ErrPluginNotExecutable, filename)
}
return &HostVolumePluginExternal{
ID: filename,
Executable: executable,
VolumesDir: volumesDir,
PluginDir: pluginDir,
NodePool: nodePool,
log: log,
}, nil
}
// HostVolumePluginExternal calls an executable on disk. All operations
// *must* be idempotent, and safe to be called concurrently per volume.
// For each call, the executable's stdout and stderr may be logged, so plugin
// authors should not include any sensitive information in their plugin outputs.
type HostVolumePluginExternal struct {
ID string
Executable string
VolumesDir string
PluginDir string
NodePool string
log hclog.Logger
}
// Fingerprint calls the executable with the following parameters:
// arguments: $1=fingerprint
// environment:
// - DHV_OPERATION=fingerprint
//
// Response should be valid JSON on stdout, with a "version" key, e.g.:
// {"version": "0.0.1"}
// The version value should be a valid version number as allowed by
// version.NewVersion()
//
// Must complete within 5 seconds
func (p *HostVolumePluginExternal) Fingerprint(ctx context.Context) (*PluginFingerprint, error) {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, p.Executable, "fingerprint")
cmd.Env = []string{EnvOperation + "=fingerprint"}
stdout, stderr, err := runCommand(cmd)
log := p.log.With(
"operation", "fingerprint",
"stdout", string(stdout),
"stderr", string(stderr),
)
if err != nil {
log.Error("error running plugin", "error", err)
return nil, fmt.Errorf("error fingerprinting plugin %q: %w", p.ID, err)
}
fprint := &PluginFingerprint{}
if err := json.Unmarshal(stdout, fprint); err != nil {
err = fmt.Errorf("error parsing fingerprint output as json: %w", err)
log.Error("error parsing plugin output", "error", err)
return nil, err
}
return fprint, nil
}
// Create calls the executable with the following parameters:
// arguments: $1=create
// environment:
// - DHV_OPERATION=create
// - DHV_VOLUMES_DIR={directory to put the volume in}
// - DHV_PLUGIN_DIR={path to directory containing plugins}
// - DHV_NAMESPACE={volume namespace}
// - DHV_VOLUME_NAME={name from the volume specification}
// - DHV_VOLUME_ID={volume ID generated by Nomad}
// - DHV_NODE_ID={Nomad node ID}
// - DHV_NODE_POOL={Nomad node pool}
// - DHV_CAPACITY_MIN_BYTES={capacity_min from the volume spec, expressed in bytes}
// - DHV_CAPACITY_MAX_BYTES={capacity_max from the volume spec, expressed in bytes}
// - DHV_PARAMETERS={stringified json of parameters from the volume spec}
//
// Response should be valid JSON on stdout with "path" and "bytes", e.g.:
// {"path": "/path/that/was/created", "bytes": 50000000}
// "path" must be provided to confirm the requested path is what was
// created by the plugin. "bytes" is the actual size of the volume created
// by the plugin; if excluded, it will default to 0.
//
// Must complete within 60 seconds (timeout on RPC)
func (p *HostVolumePluginExternal) Create(ctx context.Context,
req *cstructs.ClientHostVolumeCreateRequest) (*HostVolumePluginCreateResponse, error) {
params, err := json.Marshal(req.Parameters)
if err != nil {
// should never happen; req.Parameters is a simple map[string]string
return nil, fmt.Errorf("error marshaling volume pramaters: %w", err)
}
envVars := []string{
fmt.Sprintf("%s=%s", EnvOperation, "create"),
fmt.Sprintf("%s=%s", EnvVolumesDir, p.VolumesDir),
fmt.Sprintf("%s=%s", EnvPluginDir, p.PluginDir),
fmt.Sprintf("%s=%s", EnvNodePool, p.NodePool),
// values from volume spec
fmt.Sprintf("%s=%s", EnvNamespace, req.Namespace),
fmt.Sprintf("%s=%s", EnvVolumeName, req.Name),
fmt.Sprintf("%s=%s", EnvVolumeID, req.ID),
fmt.Sprintf("%s=%d", EnvCapacityMin, req.RequestedCapacityMinBytes),
fmt.Sprintf("%s=%d", EnvCapacityMax, req.RequestedCapacityMaxBytes),
fmt.Sprintf("%s=%s", EnvNodeID, req.NodeID),
fmt.Sprintf("%s=%s", EnvParameters, params),
}
var pluginResp HostVolumePluginCreateResponse
log := p.log.With("volume_name", req.Name, "volume_id", req.ID)
stdout, _, err := p.runPlugin(ctx, log, "create", envVars)
if err != nil {
jsonErr := json.Unmarshal(stdout, &pluginResp)
if jsonErr != nil {
// if we got an error, we can't actually count on getting JSON, so
// optimistically look for it and return the original error
// otherwise
return nil, fmt.Errorf(
"error creating volume %q with plugin %q: %w", req.ID, p.ID, err)
}
return nil, fmt.Errorf("error creating volume %q with plugin %q: %w: %s",
req.ID, p.ID, err, pluginResp.Error)
}
err = json.Unmarshal(stdout, &pluginResp)
if err != nil {
// note: if a plugin does not return valid json, a volume may be
// created without any respective state in Nomad, since we return
// an error here after the plugin has done who-knows-what.
return nil, err
}
return &pluginResp, nil
}
// Delete calls the executable with the following parameters:
// arguments: $1=delete
// environment:
// - DHV_OPERATION=delete
// - DHV_CREATED_PATH={path that `create` returned}
// - DHV_VOLUMES_DIR={directory that volumes should be put in}
// - DHV_PLUGIN_DIR={path to directory containing plugins}
// - DHV_NAMESPACE={volume namespace}
// - DHV_VOLUME_NAME={name from the volume specification}
// - DHV_VOLUME_ID={volume ID generated by Nomad}
// - DHV_NODE_ID={Nomad node ID}
// - DHV_NODE_POOL={Nomad node pool}
// - DHV_PARAMETERS={stringified json of parameters from the volume spec}
//
// Response on stdout is discarded.
//
// Must complete within 60 seconds (timeout on RPC)
func (p *HostVolumePluginExternal) Delete(ctx context.Context,
req *cstructs.ClientHostVolumeDeleteRequest) error {
params, err := json.Marshal(req.Parameters)
if err != nil {
// should never happen; req.Parameters is a simple map[string]string
return fmt.Errorf("error marshaling volume pramaters: %w", err)
}
envVars := []string{
fmt.Sprintf("%s=%s", EnvOperation, "delete"),
fmt.Sprintf("%s=%s", EnvVolumesDir, p.VolumesDir),
fmt.Sprintf("%s=%s", EnvPluginDir, p.PluginDir),
fmt.Sprintf("%s=%s", EnvNodePool, p.NodePool),
// from create response
fmt.Sprintf("%s=%s", EnvCreatedPath, req.HostPath),
// values from volume spec
fmt.Sprintf("%s=%s", EnvNamespace, req.Namespace),
fmt.Sprintf("%s=%s", EnvVolumeName, req.Name),
fmt.Sprintf("%s=%s", EnvVolumeID, req.ID),
fmt.Sprintf("%s=%s", EnvNodeID, req.NodeID),
fmt.Sprintf("%s=%s", EnvParameters, params),
}
log := p.log.With("volume_name", req.Name, "volume_id", req.ID)
stdout, _, err := p.runPlugin(ctx, log, "delete", envVars)
if err != nil {
var pluginResp HostVolumePluginDeleteResponse
jsonErr := json.Unmarshal(stdout, &pluginResp)
if jsonErr != nil {
// if we got an error, we can't actually count on getting JSON, so
// optimistically look for it and return the original error
// otherwise
return fmt.Errorf("error reading plugin response when deleting volume %q with plugin %q: original error: %w", req.ID, p.ID, err)
}
return fmt.Errorf("error deleting volume %q with plugin %q: %w: %s",
req.ID, p.ID, err, pluginResp.Error)
}
return nil
}
// runPlugin executes the... executable
func (p *HostVolumePluginExternal) runPlugin(ctx context.Context, log hclog.Logger,
op string, env []string) (stdout, stderr []byte, err error) {
log = log.With("operation", op)
log.Debug("running plugin")
// set up plugin execution
cmd := exec.CommandContext(ctx, p.Executable, op)
cmd.Env = env
stdout, stderr, err = runCommand(cmd)
log = log.With(
"stdout", string(stdout),
"stderr", string(stderr),
)
if err != nil {
log.Error("error with plugin", "error", err)
return stdout, stderr, err
}
log.Debug("plugin ran successfully")
return stdout, stderr, nil
}
// runCommand executes the provided Cmd and captures stdout and stderr.
func runCommand(cmd *exec.Cmd) (stdout, stderr []byte, err error) {
var errBuf bytes.Buffer
cmd.Stderr = io.Writer(&errBuf)
mErr := &multierror.Error{}
stdout, err = cmd.Output()
if err != nil {
mErr = multierror.Append(mErr, err)
}
stderr, err = io.ReadAll(&errBuf)
if err != nil {
mErr = multierror.Append(mErr, err)
}
return stdout, stderr, helper.FlattenMultierror(mErr.ErrorOrNil())
}
@@ -0,0 +1,346 @@
// Copyright IBM Corp. 2015, 2025
// SPDX-License-Identifier: BUSL-1.1
package hostvolumemanager
import (
"os"
"os/user"
"path/filepath"
"runtime"
"testing"
"github.com/hashicorp/go-version"
"github.com/hashicorp/nomad/ci"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/shoenig/test/must"
)
func TestHostVolumePluginMkdir(t *testing.T) {
ci.Parallel(t)
tmp := t.TempDir()
plug := &HostVolumePluginMkdir{
ID: "test-mkdir-plugin",
VolumesDir: tmp,
log: testlog.HCLogger(t),
}
// contexts don't matter here, since they're thrown away by this plugin,
// but sending timeout contexts anyway, in case the plugin changes later.
_, err := plug.Fingerprint(timeout(t))
must.NoError(t, err)
t.Run("happy", func(t *testing.T) {
volID := "happy"
target := filepath.Join(tmp, volID)
// run multiple times, should be idempotent
for range 2 {
resp, err := plug.Create(timeout(t),
&cstructs.ClientHostVolumeCreateRequest{
ID: volID, // minimum required by this plugin
})
must.NoError(t, err)
must.Eq(t, &HostVolumePluginCreateResponse{
Path: target,
SizeBytes: 0,
}, resp)
must.DirExists(t, target)
must.DirMode(t, target, 0o700+os.ModeDir)
}
// delete should be idempotent, too
for range 2 {
err = plug.Delete(timeout(t),
&cstructs.ClientHostVolumeDeleteRequest{
ID: volID,
})
must.NoError(t, err)
must.DirNotExists(t, target)
}
})
t.Run("sad", func(t *testing.T) {
volID := "sad"
// can't mkdir inside a file
plug.VolumesDir = "host_volume_plugin_test.go"
t.Cleanup(func() {
plug.VolumesDir = tmp
})
resp, err := plug.Create(timeout(t),
&cstructs.ClientHostVolumeCreateRequest{
ID: volID, // minimum required by this plugin
})
must.ErrorContains(t, err, "host_volume_plugin_test.go/sad: not a directory")
must.Nil(t, resp)
err = plug.Delete(timeout(t),
&cstructs.ClientHostVolumeDeleteRequest{
ID: volID,
})
must.ErrorContains(t, err, "host_volume_plugin_test.go/sad: not a directory")
})
t.Run("happy params", func(t *testing.T) {
volID := "happy_params"
target := filepath.Join(tmp, volID)
currentUser, err := user.Current()
must.NoError(t, err)
// run multiple times, should be idempotent
for range 2 {
_, err := plug.Create(timeout(t),
&cstructs.ClientHostVolumeCreateRequest{
ID: volID,
Parameters: map[string]string{
"uid": currentUser.Uid,
"gid": currentUser.Gid,
"mode": "0777",
},
})
must.NoError(t, err)
must.DirExists(t, target)
must.DirMode(t, target, 0o777+os.ModeDir)
}
err = plug.Delete(timeout(t),
&cstructs.ClientHostVolumeDeleteRequest{
ID: volID,
})
must.NoError(t, err)
must.DirNotExists(t, target)
})
t.Run("sad params", func(t *testing.T) {
volID := "sad_params"
// test one representative error; decodeMkdirParams has its own tests
resp, err := plug.Create(timeout(t),
&cstructs.ClientHostVolumeCreateRequest{
ID: volID,
Parameters: map[string]string{
"mode": "invalid",
},
})
must.ErrorContains(t, err, "invalid value")
must.Nil(t, resp)
})
}
func TestDecodeMkdirParams(t *testing.T) {
ci.Parallel(t)
cases := []struct {
name string
params map[string]string
expect HostVolumePluginMkdirParams
err string
}{
{
name: "none ok",
params: nil,
expect: HostVolumePluginMkdirParams{
Uid: -1,
Gid: -1,
Mode: os.FileMode(0700),
},
},
{
name: "all ok",
params: map[string]string{
"uid": "1",
"gid": "2",
"mode": "0444",
},
expect: HostVolumePluginMkdirParams{
Uid: 1,
Gid: 2,
Mode: os.FileMode(0444),
},
},
{
name: "invalid mode: bad number",
params: map[string]string{
// this is what happens if you put mode=0700 instead of
// mode="0700" in the HCL spec.
"mode": "493",
},
err: `invalid value for "mode"`,
},
{
name: "invalid mode: string",
params: map[string]string{
"mode": "consider the lobster",
},
err: `invalid value for "mode"`,
},
{
name: "invalid uid",
params: map[string]string{
"uid": "a supposedly fun thing i'll never do again",
},
err: `invalid value for "uid"`,
},
{
name: "invalid gid",
params: map[string]string{
"gid": "surely you jest",
},
err: `invalid value for "gid"`,
},
{
name: "unknown param",
params: map[string]string{
"what": "the hell is water?",
},
err: `unknown mkdir parameter: "what"`,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, err := decodeMkdirParams(tc.params)
if tc.err != "" {
must.ErrorContains(t, err, tc.err)
} else {
must.NoError(t, err)
must.Eq(t, tc.expect, got)
}
})
}
}
func TestNewHostVolumePluginExternal(t *testing.T) {
ci.Parallel(t)
log := testlog.HCLogger(t)
var err error
_, err = NewHostVolumePluginExternal(log, ".", "non-existent", "target", "")
must.ErrorIs(t, err, ErrPluginNotExists)
_, err = NewHostVolumePluginExternal(log, ".", "host_volume_plugin_test.go", "target", "")
must.ErrorIs(t, err, ErrPluginNotExecutable)
t.Run("unix", func(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("skipped because windows") // db TODO(1.10.0)
}
p, err := NewHostVolumePluginExternal(log,
"./test_fixtures", "test_plugin.sh", "test-target", "test-pool")
must.NoError(t, err)
must.Eq(t, &HostVolumePluginExternal{
ID: "test_plugin.sh",
Executable: "test_fixtures/test_plugin.sh",
VolumesDir: "test-target",
PluginDir: "./test_fixtures",
NodePool: "test-pool",
log: log,
}, p)
})
}
func TestHostVolumePluginExternal(t *testing.T) {
ci.Parallel(t)
if runtime.GOOS == "windows" {
t.Skip("skipped because windows") // db TODO(1.10.0)
}
volID := "test-vol-id"
tmp := t.TempDir()
target := filepath.Join(tmp, volID)
expectVersion, err := version.NewVersion("0.0.2")
must.NoError(t, err)
t.Run("happy", func(t *testing.T) {
log, getLogs := logRecorder(t)
plug, err := NewHostVolumePluginExternal(log,
"./test_fixtures", "test_plugin.sh", tmp, "test-node-pool")
must.NoError(t, err)
// fingerprint
v, err := plug.Fingerprint(timeout(t))
logged := getLogs()
must.NoError(t, err, must.Sprintf("logs: %s", logged))
must.Eq(t, expectVersion, v.Version, must.Sprintf("logs: %s", logged))
// create
resp, err := plug.Create(timeout(t),
&cstructs.ClientHostVolumeCreateRequest{
Name: "test-vol-name",
ID: volID,
Namespace: "test-namespace",
NodeID: "test-node",
RequestedCapacityMinBytes: 5,
RequestedCapacityMaxBytes: 10,
Parameters: map[string]string{"key": "val"},
})
logged = getLogs()
must.NoError(t, err, must.Sprintf("logs: %s", logged))
must.Eq(t, &HostVolumePluginCreateResponse{
Path: target,
SizeBytes: 5,
}, resp)
must.DirExists(t, target)
must.StrContains(t, logged, "OPERATION=create") // stderr from `env`
must.StrContains(t, logged, `stdout="{`) // stdout from printf
// delete
err = plug.Delete(timeout(t),
&cstructs.ClientHostVolumeDeleteRequest{
Name: "test-vol-name",
ID: volID,
HostPath: resp.Path,
Namespace: "test-namespace",
NodeID: "test-node",
Parameters: map[string]string{"key": "val"},
})
logged = getLogs()
must.NoError(t, err, must.Sprintf("logs: %s", logged))
must.DirNotExists(t, target)
must.StrContains(t, logged, "OPERATION=delete") // stderr from `env`
must.StrContains(t, logged, "removed directory") // stdout from `rm -v`
})
t.Run("sad", func(t *testing.T) {
log, getLogs := logRecorder(t)
plug, err := NewHostVolumePluginExternal(log, "./test_fixtures", "test_plugin_sad.sh", tmp, "")
must.NoError(t, err)
v, err := plug.Fingerprint(timeout(t))
must.EqError(t, err, `error fingerprinting plugin "test_plugin_sad.sh": exit status 1`)
must.Nil(t, v)
logged := getLogs()
must.StrContains(t, logged, "fingerprint: sad plugin is sad")
must.StrContains(t, logged, "fingerprint: it tells you all about it in stderr")
// reset logger
log, getLogs = logRecorder(t)
plug.log = log
resp, err := plug.Create(timeout(t),
&cstructs.ClientHostVolumeCreateRequest{
ID: volID,
})
must.EqError(t, err, `error creating volume "test-vol-id" with plugin "test_plugin_sad.sh": exit status 1: create: sad plugin is sad`)
must.Nil(t, resp)
logged = getLogs()
must.StrContains(t, logged, "create: sad plugin is sad")
must.StrContains(t, logged, "create: it tells you all about it in stderr")
log, getLogs = logRecorder(t)
plug.log = log
err = plug.Delete(timeout(t),
&cstructs.ClientHostVolumeDeleteRequest{
ID: volID,
})
must.EqError(t, err, `error deleting volume "test-vol-id" with plugin "test_plugin_sad.sh": exit status 1: delete: sad plugin is sad`)
logged = getLogs()
must.StrContains(t, logged, "delete: sad plugin is sad")
must.StrContains(t, logged, "delete: it tells you all about it in stderr")
})
}
@@ -0,0 +1,377 @@
// Copyright IBM Corp. 2015, 2025
// SPDX-License-Identifier: BUSL-1.1
package hostvolumemanager
import (
"context"
"errors"
"fmt"
"os"
"sync"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-multierror"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/nomad/structs"
)
var (
ErrPluginNotExists = errors.New("no such plugin")
ErrPluginNotExecutable = errors.New("plugin not executable")
ErrVolumeNameExists = errors.New("volume name already exists on this node")
)
// HostVolumeStateManager manages the lifecycle of volumes in client state.
type HostVolumeStateManager interface {
PutDynamicHostVolume(*cstructs.HostVolumeState) error
GetDynamicHostVolumes() ([]*cstructs.HostVolumeState, error)
DeleteDynamicHostVolume(string) error
}
// Config is used to configure a HostVolumeManager.
type Config struct {
// PluginDir is where external plugins may be found.
PluginDir string
// VolumesDir is where plugins should place the directory
// that will later become a volume's HostPath
VolumesDir string
// NodePool is passed into external plugin execution environment.
NodePool string
// StateMgr manages client state to restore on agent restarts.
StateMgr HostVolumeStateManager
// UpdateNodeVols is run to update the node when a volume is created
// or deleted.
UpdateNodeVols HostVolumeNodeUpdater
}
// HostVolumeManager executes plugins, manages volume metadata in client state,
// and registers volumes with the client node.
type HostVolumeManager struct {
pluginDir string
volumesDir string
nodePool string
stateMgr HostVolumeStateManager
updateNodeVols HostVolumeNodeUpdater
builtIns map[string]HostVolumePlugin
locker *volLocker
log hclog.Logger
}
// NewHostVolumeManager includes default builtin plugins.
func NewHostVolumeManager(logger hclog.Logger, config Config) *HostVolumeManager {
logger = logger.Named("host_volume_manager")
return &HostVolumeManager{
pluginDir: config.PluginDir,
volumesDir: config.VolumesDir,
nodePool: config.NodePool,
stateMgr: config.StateMgr,
updateNodeVols: config.UpdateNodeVols,
builtIns: map[string]HostVolumePlugin{
HostVolumePluginMkdirID: &HostVolumePluginMkdir{
ID: HostVolumePluginMkdirID,
VolumesDir: config.VolumesDir,
log: logger.With("plugin_id", HostVolumePluginMkdirID),
},
},
locker: &volLocker{},
log: logger,
}
}
// Create runs the appropriate plugin for the given request, saves the request
// to state, and updates the node with the volume.
func (hvm *HostVolumeManager) Create(ctx context.Context,
req *cstructs.ClientHostVolumeCreateRequest) (*cstructs.ClientHostVolumeCreateResponse, error) {
log := hvm.log.With("volume_name", req.Name, "volume_id", req.ID)
plug, err := hvm.getPlugin(req.PluginID)
if err != nil {
return nil, err
}
// can't have two of the same volume name w/ different IDs per client node
isNewVolume, err := hvm.locker.lock(req.Name, req.ID)
if err != nil {
return nil, err
}
pluginResp, err := plug.Create(ctx, req)
if err != nil {
hvm.locker.release(req.Name)
return nil, err
}
volState := &cstructs.HostVolumeState{
ID: req.ID,
HostPath: pluginResp.Path,
CreateReq: req,
}
if err := hvm.stateMgr.PutDynamicHostVolume(volState); err != nil {
// if we fail to write to state on initial create,
// delete the volume so it isn't left lying around
// without Nomad knowing about it.
log.Error("failed to save volume in client state", "error", err)
if isNewVolume {
log.Error("initial create detected, running delete")
delErr := plug.Delete(ctx, &cstructs.ClientHostVolumeDeleteRequest{
ID: req.ID,
PluginID: req.PluginID,
NodeID: req.NodeID,
HostPath: hvm.volumesDir,
Parameters: req.Parameters,
})
if delErr != nil {
log.Warn("error deleting volume after state store failure", "error", delErr)
err = multierror.Append(err, delErr)
}
// free up the volume name whether delete succeeded or not.
hvm.locker.release(req.Name)
}
return nil, err
}
hvm.updateNodeVols(req.Name, genVolConfig(req, pluginResp.Path))
resp := &cstructs.ClientHostVolumeCreateResponse{
VolumeName: req.Name,
VolumeID: req.ID,
HostPath: pluginResp.Path,
CapacityBytes: pluginResp.SizeBytes,
}
return resp, nil
}
// Register saves the request to state, and updates the node with the volume.
func (hvm *HostVolumeManager) Register(ctx context.Context,
req *cstructs.ClientHostVolumeRegisterRequest) error {
log := hvm.log.With("volume_name", req.Name, "volume_id", req.ID)
// can't have two of the same volume name w/ different IDs per client node
if _, err := hvm.locker.lock(req.Name, req.ID); err != nil {
log.Error("duplicate volume name", "error", err)
return err
}
_, err := os.Stat(req.HostPath)
if err != nil {
hvm.locker.release(req.Name)
err = fmt.Errorf("could not verify host path for %q: %w", req.Name, err)
log.Error("error registering volume", "error", err)
return err
}
// generate a stub create request and plugin response for the fingerprint
// and client state
creq := &cstructs.ClientHostVolumeCreateRequest{
ID: req.ID,
Name: req.Name,
NodeID: req.NodeID,
Parameters: req.Parameters,
}
volState := &cstructs.HostVolumeState{
ID: req.ID,
CreateReq: creq,
HostPath: req.HostPath,
}
if err := hvm.stateMgr.PutDynamicHostVolume(volState); err != nil {
log.Error("failed to save volume in state", "error", err)
hvm.locker.release(req.Name)
return err
}
hvm.updateNodeVols(req.Name, genVolConfig(creq, req.HostPath))
return nil
}
// Delete runs the appropriate plugin for the given request, removes it from
// state, and updates the node to remove the volume.
func (hvm *HostVolumeManager) Delete(ctx context.Context,
req *cstructs.ClientHostVolumeDeleteRequest) (*cstructs.ClientHostVolumeDeleteResponse, error) {
if req.PluginID != "" {
plug, err := hvm.getPlugin(req.PluginID)
if err != nil {
return nil, err
}
err = plug.Delete(ctx, req)
if err != nil {
return nil, err
}
}
if err := hvm.stateMgr.DeleteDynamicHostVolume(req.ID); err != nil {
hvm.log.Error("failed to delete volume in state", "volume_id", req.ID, "error", err)
return nil, err // bail so a user may retry
}
// free up volume name for reuse
hvm.locker.release(req.Name)
hvm.updateNodeVols(req.Name, nil)
resp := &cstructs.ClientHostVolumeDeleteResponse{
VolumeName: req.Name,
VolumeID: req.ID,
}
return resp, nil
}
// getPlugin finds either a built-in plugin or an external plugin.
func (hvm *HostVolumeManager) getPlugin(id string) (HostVolumePlugin, error) {
if plug, ok := hvm.builtIns[id]; ok {
return plug, nil
}
log := hvm.log.With("plugin_id", id)
return NewHostVolumePluginExternal(log, hvm.pluginDir, id, hvm.volumesDir, hvm.nodePool)
}
// restoreFromState loads all volumes from client state and runs Create for
// each one, so volumes are restored upon agent restart or host reboot.
func (hvm *HostVolumeManager) restoreFromState(ctx context.Context) (VolumeMap, error) {
vols, err := hvm.stateMgr.GetDynamicHostVolumes()
if err != nil {
return nil, err
}
volumes := make(VolumeMap)
var mut sync.Mutex
if len(vols) == 0 {
return volumes, nil // nothing to do
}
group := multierror.Group{}
for _, vol := range vols {
group.Go(func() error {
var volCfg *structs.ClientHostVolumeConfig
var err error
if vol.CreateReq.PluginID == "" {
volCfg, err = hvm.restoreForRegister(vol)
} else {
volCfg, err = hvm.restoreForCreate(ctx, vol)
}
if err != nil {
return err
}
mut.Lock()
volumes[vol.CreateReq.Name] = volCfg
mut.Unlock()
return nil
})
}
mErr := group.Wait()
return volumes, helper.FlattenMultierror(mErr.ErrorOrNil())
}
// restoreForCreate restores a single volume that was previously created by
// Create, by "recreating" the volumes. Plugins have the best knowledge of their
// side effects, and they must be idempotent.
func (hvm *HostVolumeManager) restoreForCreate(ctx context.Context, vol *cstructs.HostVolumeState) (*structs.ClientHostVolumeConfig, error) {
// missing plugins with associated volumes in state are considered
// client-stopping errors. they need to be fixed by cluster admins.
plug, err := hvm.getPlugin(vol.CreateReq.PluginID)
if err != nil {
return nil, err
}
// lock the name so future creates can't produce duplicates.
_, err = hvm.locker.lock(vol.CreateReq.Name, vol.CreateReq.ID)
// state should never have duplicate vol names, and restore happens
// prior to node registration, so new creates shouldn't come in
// concurrently, but check for error just in case.
if err != nil {
hvm.log.Error("error during restore",
"volume_name", vol.CreateReq.Name,
"volume_id", vol.CreateReq.ID,
"error", err)
// don't stop the world if it does happen, because an admin
// couldn't do anything about it short of wiping client state.
return nil, nil
}
resp, err := plug.Create(ctx, vol.CreateReq)
if err != nil {
// plugin execution errors are only logged
hvm.log.Error("failed to restore",
"plugin_id", vol.CreateReq.PluginID, "volume_id", vol.ID, "error", err)
return nil, nil
}
return genVolConfig(vol.CreateReq, resp.Path), nil
}
// restoreForRegister restores a single volume that was previously created by
// Register, by converting the stored struct. It otherwise behaves the same as
// restoreForCreate.
func (hvm *HostVolumeManager) restoreForRegister(vol *cstructs.HostVolumeState) (*structs.ClientHostVolumeConfig, error) {
_, err := hvm.locker.lock(vol.CreateReq.Name, vol.CreateReq.ID)
if err != nil {
hvm.log.Error("error during restore",
"volume_name", vol.CreateReq.Name,
"volume_id", vol.CreateReq.ID,
"error", err)
return nil, nil
}
_, err = os.Stat(vol.HostPath)
if err != nil {
hvm.log.Error("failed to restore: could not verify host path",
"volume_id", vol.ID, "error", err, "path", vol.HostPath)
return nil, nil
}
return genVolConfig(vol.CreateReq, vol.HostPath), nil
}
// genVolConfig generates the host volume config for the node to report as
// available to the servers for job scheduling.
func genVolConfig(req *cstructs.ClientHostVolumeCreateRequest, hostPath string) *structs.ClientHostVolumeConfig {
return &structs.ClientHostVolumeConfig{
Name: req.Name,
ID: req.ID,
Path: hostPath,
// dynamic volumes, like CSI, have more robust `capabilities`,
// so we always set ReadOnly to false, and let the scheduler
// decide when to ignore this and check capabilities instead.
ReadOnly: false,
}
}
// volLocker is used to ensure that volumes on each node are unique by name.
// The volume scheduler will prevent this too, but only after node fingerprint,
// so we need to protect against concurrent duplicate creates.
type volLocker struct {
locks sync.Map
}
// lock the provided name, return true if it was not already locked,
// and error if it was already locked with a different ID.
func (l *volLocker) lock(name, id string) (bool, error) {
current, exists := l.locks.LoadOrStore(name, id)
if exists && id != current.(string) {
return false, ErrVolumeNameExists
}
return !exists, nil
}
func (l *volLocker) release(name string) {
l.locks.Delete(name)
}
func (l *volLocker) isLocked(name string) bool {
_, locked := l.locks.Load(name)
return locked
}
@@ -0,0 +1,441 @@
// Copyright IBM Corp. 2015, 2025
// SPDX-License-Identifier: BUSL-1.1
package hostvolumemanager
import (
"bytes"
"context"
"errors"
"io"
"path/filepath"
"sort"
"testing"
"time"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-version"
cstate "github.com/hashicorp/nomad/client/state"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/shoenig/test"
"github.com/shoenig/test/must"
)
func TestHostVolumeManager(t *testing.T) {
log := testlog.HCLogger(t)
errDB := &cstate.ErrDB{}
memDB := cstate.NewMemDB(log)
node := newFakeNode(t)
volumesDirCreate := t.TempDir()
volumesDirRegister := t.TempDir()
hvm := NewHostVolumeManager(log, Config{
PluginDir: "./test_fixtures",
VolumesDir: volumesDirCreate,
StateMgr: errDB,
UpdateNodeVols: node.updateVol,
})
plug := &fakePlugin{volsDir: volumesDirCreate}
hvm.builtIns["test-plugin"] = plug
ctx := timeout(t)
t.Run("create", func(t *testing.T) {
// plugin doesn't exist
name := "created-volume"
req := &cstructs.ClientHostVolumeCreateRequest{
Name: name,
ID: "vol-id-1",
PluginID: "nope",
RequestedCapacityMinBytes: 5,
}
_, err := hvm.Create(ctx, req)
must.ErrorIs(t, err, ErrPluginNotExists)
// error from plugin
req.PluginID = "test-plugin"
plug.createErr = errors.New("sad create")
_, err = hvm.Create(ctx, req)
must.ErrorIs(t, err, plug.createErr)
assertNotLocked(t, hvm, name)
plug.reset()
// error saving state, then error from cleanup attempt
plug.deleteErr = errors.New("sad delete")
_, err = hvm.Create(ctx, req)
must.ErrorIs(t, err, cstate.ErrDBError)
must.ErrorIs(t, err, plug.deleteErr)
assertNotLocked(t, hvm, name)
plug.reset()
// error saving state, successful cleanup
_, err = hvm.Create(ctx, req)
must.ErrorIs(t, err, cstate.ErrDBError)
must.Eq(t, "vol-id-1", plug.deleted)
assertNotLocked(t, hvm, name)
plug.reset()
// happy path
hvm.stateMgr = memDB
resp, err := hvm.Create(ctx, req)
must.NoError(t, err)
expectResp := &cstructs.ClientHostVolumeCreateResponse{
VolumeName: "created-volume",
VolumeID: "vol-id-1",
HostPath: filepath.Join(volumesDirCreate, "vol-id-1"),
CapacityBytes: 5,
}
must.Eq(t, expectResp, resp)
stateDBs, err := memDB.GetDynamicHostVolumes()
must.NoError(t, err)
// should be saved to state
must.Len(t, 1, stateDBs)
must.Eq(t, "vol-id-1", stateDBs[0].ID)
must.Eq(t, "vol-id-1", stateDBs[0].CreateReq.ID)
// should be registered with node
must.MapContainsKey(t, node.vols, name, must.Sprintf("no %q in %+v", name, node.vols))
assertLocked(t, hvm, name)
// repeat create with same ID but different size may update the volume
req.RequestedCapacityMinBytes = 10
expectResp.CapacityBytes = 10
resp, err = hvm.Create(ctx, req)
must.NoError(t, err)
must.Eq(t, expectResp, resp)
// error saving state on restore/update should not run delete
hvm.stateMgr = errDB
resp, err = hvm.Create(ctx, req)
must.ErrorIs(t, err, cstate.ErrDBError)
must.Nil(t, resp)
must.Eq(t, "", plug.deleted)
plug.reset()
hvm.stateMgr = memDB
// duplicate create with the same vol name but different ID should fail
_, err = hvm.Create(ctx, &cstructs.ClientHostVolumeCreateRequest{
Name: name,
ID: "different-vol-id",
PluginID: "test-plugin",
})
must.ErrorIs(t, err, ErrVolumeNameExists)
})
t.Run("register", func(t *testing.T) {
name := "registered-volume"
req := &cstructs.ClientHostVolumeRegisterRequest{
ID: "vol-id-2",
Name: name,
HostPath: volumesDirRegister,
CapacityBytes: 1000,
}
err := hvm.Register(ctx, req)
must.NoError(t, err)
// should be saved to state and registered with node
stateDBs, err := memDB.GetDynamicHostVolumes()
must.NoError(t, err)
must.Len(t, 2, stateDBs)
sort.Slice(stateDBs, func(i, j int) bool { return stateDBs[i].ID < stateDBs[j].ID })
must.Eq(t, "vol-id-2", stateDBs[1].ID)
must.Eq(t, "vol-id-2", stateDBs[1].CreateReq.ID)
must.MapContainsKey(t, node.vols, name, must.Sprintf("no %q in %+v", name, node.vols))
assertLocked(t, hvm, name)
})
// despite being a subtest, this needs to run after "create" and "register"
t.Run("delete", func(t *testing.T) {
name := "created-volume"
// should be locked from "create" above
assertLocked(t, hvm, name)
// plugin doesn't exist
req := &cstructs.ClientHostVolumeDeleteRequest{
Name: name,
ID: "vol-id-1",
PluginID: "nope",
}
_, err := hvm.Delete(ctx, req)
must.ErrorIs(t, err, ErrPluginNotExists)
assertLocked(t, hvm, name)
// error from plugin
req.PluginID = "test-plugin"
plug.deleteErr = errors.New("sad delete")
_, err = hvm.Delete(ctx, req)
must.ErrorIs(t, err, plug.deleteErr)
assertLocked(t, hvm, name)
plug.reset()
// error saving state
hvm.stateMgr = errDB
_, err = hvm.Delete(ctx, req)
must.ErrorIs(t, err, cstate.ErrDBError)
assertLocked(t, hvm, name)
// happy path
hvm.stateMgr = memDB
// and delete it
resp, err := hvm.Delete(ctx, req)
must.NoError(t, err)
must.Eq(t, &cstructs.ClientHostVolumeDeleteResponse{
VolumeName: "created-volume",
VolumeID: "vol-id-1",
}, resp)
must.Eq(t, VolumeMap{
"registered-volume": &structs.ClientHostVolumeConfig{
Name: "registered-volume",
Path: volumesDirRegister,
ID: "vol-id-2",
},
}, node.vols, must.Sprint("created-volume should be deleted from node"))
stateVols, err := memDB.GetDynamicHostVolumes()
must.NoError(t, err)
must.Len(t, 1, stateVols, must.Sprint("only one volume should be deleted"))
assertNotLocked(t, hvm, name)
assertLocked(t, hvm, "registered-volume")
req.Name = "registered-volume"
req.ID = "vol-id-2"
req.PluginID = ""
resp, err = hvm.Delete(ctx, req)
must.NoError(t, err)
must.Eq(t, VolumeMap{}, node.vols, must.Sprint("all volumes should be deleted from node"))
stateVols, err = memDB.GetDynamicHostVolumes()
must.NoError(t, err)
must.Nil(t, stateVols, must.Sprint("all volumes should be deleted"))
assertNotLocked(t, hvm, "registered-volume")
})
}
type fakePlugin struct {
volsDir string
created string
deleted string
fingerprintErr error
createErr error
deleteErr error
}
func (p *fakePlugin) reset() {
p.deleted, p.fingerprintErr, p.createErr, p.deleteErr = "", nil, nil, nil
}
func (p *fakePlugin) Fingerprint(_ context.Context) (*PluginFingerprint, error) {
if p.fingerprintErr != nil {
return nil, p.fingerprintErr
}
v, err := version.NewVersion("0.0.1")
return &PluginFingerprint{
Version: v,
}, err
}
func (p *fakePlugin) Create(_ context.Context, req *cstructs.ClientHostVolumeCreateRequest) (*HostVolumePluginCreateResponse, error) {
if p.createErr != nil {
return nil, p.createErr
}
p.created = req.ID
return &HostVolumePluginCreateResponse{
Path: filepath.Join(p.volsDir, req.ID),
SizeBytes: req.RequestedCapacityMinBytes,
}, nil
}
func (p *fakePlugin) Delete(_ context.Context, req *cstructs.ClientHostVolumeDeleteRequest) error {
if p.deleteErr != nil {
return p.deleteErr
}
p.deleted = req.ID
return nil
}
func assertLocked(t *testing.T, hvm *HostVolumeManager, name string) {
t.Helper()
must.True(t, hvm.locker.isLocked(name), must.Sprintf("vol name %q should be locked", name))
}
func assertNotLocked(t *testing.T, hvm *HostVolumeManager, name string) {
t.Helper()
must.False(t, hvm.locker.isLocked(name), must.Sprintf("vol name %q should not be locked", name))
}
func TestHostVolumeManager_restoreFromState(t *testing.T) {
log := testlog.HCLogger(t)
hostPath := t.TempDir()
vol1 := &cstructs.HostVolumeState{
ID: "test-vol-id-1",
CreateReq: &cstructs.ClientHostVolumeCreateRequest{
Name: "created-volume",
ID: "test-vol-id-1",
PluginID: "mkdir",
},
}
vol2 := &cstructs.HostVolumeState{
ID: "test-vol-id-2",
HostPath: hostPath,
CreateReq: &cstructs.ClientHostVolumeCreateRequest{
Name: "registered-volume",
ID: "test-vol-id-2",
PluginID: "", // this signifies a Register operation
},
}
node := newFakeNode(t)
t.Run("no vols", func(t *testing.T) {
state := cstate.NewMemDB(log)
hvm := NewHostVolumeManager(log, Config{
StateMgr: state,
// no other fields are necessary when there are zero volumes
})
vols, err := hvm.restoreFromState(timeout(t))
must.NoError(t, err)
must.Eq(t, VolumeMap{}, vols)
})
t.Run("happy", func(t *testing.T) {
// put our volume in state
state := cstate.NewMemDB(log)
must.NoError(t, state.PutDynamicHostVolume(vol1))
must.NoError(t, state.PutDynamicHostVolume(vol2))
// new volume manager should load it from state and run Create,
// resulting in a volume directory in this mountDir.
volsDir := t.TempDir()
volPath := filepath.Join(volsDir, vol1.ID)
hvm := NewHostVolumeManager(log, Config{
StateMgr: state,
UpdateNodeVols: node.updateVol,
PluginDir: "/wherever",
VolumesDir: volsDir,
})
vols, err := hvm.restoreFromState(timeout(t))
must.NoError(t, err)
expect := map[string]*structs.ClientHostVolumeConfig{
"created-volume": {
Name: "created-volume",
ID: "test-vol-id-1",
Path: volPath,
ReadOnly: false,
},
"registered-volume": {
Name: "registered-volume",
ID: "test-vol-id-2",
Path: hostPath,
ReadOnly: false,
},
}
must.Eq(t, expect, vols)
must.DirExists(t, volPath)
assertLocked(t, hvm, "created-volume")
assertLocked(t, hvm, "registered-volume")
})
t.Run("state error", func(t *testing.T) {
state := &cstate.ErrDB{}
hvm := NewHostVolumeManager(log, Config{StateMgr: state})
vols, err := hvm.restoreFromState(timeout(t))
must.ErrorIs(t, err, cstate.ErrDBError)
must.Nil(t, vols)
})
t.Run("plugin missing", func(t *testing.T) {
state := cstate.NewMemDB(log)
vol := &cstructs.HostVolumeState{
CreateReq: &cstructs.ClientHostVolumeCreateRequest{
PluginID: "nonexistent-plugin",
},
}
must.NoError(t, state.PutDynamicHostVolume(vol))
hvm := NewHostVolumeManager(log, Config{StateMgr: state})
vols, err := hvm.restoreFromState(timeout(t))
must.ErrorIs(t, err, ErrPluginNotExists)
must.MapEmpty(t, vols)
})
t.Run("plugin error", func(t *testing.T) {
state := cstate.NewMemDB(log)
vol := &cstructs.HostVolumeState{
ID: "test-volume",
CreateReq: &cstructs.ClientHostVolumeCreateRequest{
PluginID: "test-plugin",
},
}
must.NoError(t, state.PutDynamicHostVolume(vol))
log, getLogs := logRecorder(t)
hvm := NewHostVolumeManager(log, Config{StateMgr: state})
plug := &fakePlugin{
createErr: errors.New("sad create"),
}
hvm.builtIns["test-plugin"] = plug
vols, err := hvm.restoreFromState(timeout(t))
// error during restore should not halt the whole client
must.NoError(t, err)
must.NotNil(t, vols)
// but it should log
logs := getLogs()
must.StrContains(t, logs, "[ERROR]")
must.StrContains(t, logs, `failed to restore: plugin_id=test-plugin volume_id=test-volume error="sad create"`)
})
}
type fakeNode struct {
vols VolumeMap
log hclog.Logger
}
func (n *fakeNode) updateVol(name string, volume *structs.ClientHostVolumeConfig) {
UpdateVolumeMap(n.log, n.vols, name, volume)
}
func newFakeNode(t *testing.T) *fakeNode {
return &fakeNode{
vols: make(VolumeMap),
log: testlog.HCLogger(t),
}
}
// timeout provides a context that times out in 1 second
func timeout(t *testing.T) context.Context {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
t.Cleanup(cancel)
return ctx
}
// logRecorder is here so we can assert that stdout/stderr appear in logs
func logRecorder(t *testing.T) (hclog.Logger, func() string) {
t.Helper()
buf := &bytes.Buffer{}
logger := hclog.New(&hclog.LoggerOptions{
Name: "log-recorder",
Output: buf,
Level: hclog.Debug,
IncludeLocation: true,
DisableTime: true,
})
return logger, func() string {
bts, err := io.ReadAll(buf)
test.NoError(t, err)
buf.Reset()
return string(bts)
}
}
@@ -0,0 +1,50 @@
#!/usr/bin/env bash
# Copyright IBM Corp. 2015, 2025
# SPDX-License-Identifier: BUSL-1.1
# plugin for host_volume_plugin_test.go
set -xeuo pipefail
env 1>&2
test "$1" == "$DHV_OPERATION"
echo 'all operations should ignore stderr' 1>&2
case $1 in
fingerprint)
echo '{"version": "0.0.2"}' ;;
create)
test "$DHV_VOLUME_NAME" == 'test-vol-name'
test "$DHV_VOLUME_ID" == 'test-vol-id'
test "$DHV_NAMESPACE" == 'test-namespace'
test "$DHV_CAPACITY_MIN_BYTES" -eq 5
test "$DHV_CAPACITY_MAX_BYTES" -eq 10
test "$DHV_NODE_ID" == 'test-node'
test "$DHV_NODE_POOL" == 'test-node-pool'
test "$DHV_PARAMETERS" == '{"key":"val"}'
test "$DHV_PLUGIN_DIR" == './test_fixtures'
test -d "$DHV_VOLUMES_DIR"
target="$DHV_VOLUMES_DIR/$DHV_VOLUME_ID"
test "$target" != '/'
mkdir -p "$target"
printf '{"path": "%s", "bytes": 5}' "$target"
;;
delete)
test "$DHV_NODE_ID" == 'test-node'
test "$DHV_NODE_POOL" == 'test-node-pool'
test "$DHV_NAMESPACE" == 'test-namespace'
test "$DHV_VOLUME_NAME" == 'test-vol-name'
test "$DHV_VOLUME_ID" == 'test-vol-id'
test "$DHV_PARAMETERS" == '{"key":"val"}'
test "$DHV_PLUGIN_DIR" == './test_fixtures'
test -d "$DHV_VOLUMES_DIR"
target="$DHV_VOLUMES_DIR/$DHV_VOLUME_ID"
test "$target" != '/'
test "$DHV_CREATED_PATH" == "$target"
rm -rfv "$target"
;;
*)
echo "unknown operation $1"
exit 1 ;;
esac
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
# Copyright IBM Corp. 2015, 2025
# SPDX-License-Identifier: BUSL-1.1
printf '{"error": "%s: sad plugin is sad"}' $1
echo "$1: it tells you all about it in stderr" 1>&2
exit 1
@@ -0,0 +1,73 @@
// Copyright IBM Corp. 2015, 2025
// SPDX-License-Identifier: BUSL-1.1
package hostvolumemanager
import (
"context"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/nomad/nomad/structs"
)
// this file is for fingerprinting *volumes*
// *plugins* are detected in client/fingerprint/dynamic_host_volumes.go
// HostVolumeNodeUpdater is used to add or remove volumes from the Node.
type HostVolumeNodeUpdater func(name string, volume *structs.ClientHostVolumeConfig)
// VolumeMap keys are volume `name`s, identical to Node.HostVolumes.
type VolumeMap map[string]*structs.ClientHostVolumeConfig
// UpdateVolumeMap returns true if it changes the provided `volumes` map.
// If `vol` is nil, key `name` will be removed from the map, if present.
// If it is not nil, `name: vol` will be set on the map, if different.
//
// Since it may mutate the map, the caller should make a copy
// or acquire a lock as appropriate for their context.
func UpdateVolumeMap(log hclog.Logger, volumes VolumeMap, name string, vol *structs.ClientHostVolumeConfig) (changed bool) {
current, exists := volumes[name]
if vol == nil {
if exists {
delete(volumes, name)
changed = true
}
} else {
// if the volume already exists with no ID, it will be because it was
// added to client agent config after having been previously created
// as a dynamic vol. dynamic takes precedence, but log a warning.
if exists && current.ID == "" {
log.Warn("overriding static host volume with dynamic", "name", name, "id", vol.ID)
}
if !exists || !vol.Equal(current) {
volumes[name] = vol
changed = true
}
}
return changed
}
// WaitForFirstFingerprint implements client.FingerprintingPluginManager
// so any existing volumes are added to the client node on agent start.
func (hvm *HostVolumeManager) WaitForFirstFingerprint(ctx context.Context) <-chan struct{} {
// the fingerprint manager puts batchFirstFingerprintsTimeout (50 seconds)
// on the context that it sends to us here so we don't need another
// timeout. we just need to cancel to report when we are done.
ctx, cancel := context.WithCancel(ctx)
defer cancel()
volumes, err := hvm.restoreFromState(ctx)
if err != nil {
hvm.log.Error("failed to restore state", "error", err)
return ctx.Done()
}
for name, vol := range volumes {
hvm.updateNodeVols(name, vol) // => batchNodeUpdates.updateNodeFromHostVolume()
}
return ctx.Done()
}
func (hvm *HostVolumeManager) Run() {}
func (hvm *HostVolumeManager) Shutdown() {}
func (hvm *HostVolumeManager) PluginType() string {
// "Plugin"Type is misleading, because this is for *volumes* but ok.
return "dynamic_host_volume"
}
@@ -0,0 +1,144 @@
// Copyright IBM Corp. 2015, 2025
// SPDX-License-Identifier: BUSL-1.1
package hostvolumemanager
import (
"path/filepath"
"testing"
"github.com/hashicorp/nomad/client/state"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper/testlog"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/shoenig/test/must"
)
func TestUpdateVolumeMap(t *testing.T) {
cases := []struct {
name string
vols VolumeMap
volName string
vol *structs.ClientHostVolumeConfig
expectMap VolumeMap
expectChange bool
expectLog string
}{
{
name: "delete absent",
vols: VolumeMap{},
volName: "anything",
vol: nil,
expectMap: VolumeMap{},
expectChange: false,
},
{
name: "delete present",
vols: VolumeMap{"deleteme": {}},
volName: "deleteme",
vol: nil,
expectMap: VolumeMap{},
expectChange: true,
},
{
name: "add absent",
vols: VolumeMap{},
volName: "addme",
vol: &structs.ClientHostVolumeConfig{},
expectMap: VolumeMap{"addme": {}},
expectChange: true,
},
{
name: "add present",
vols: VolumeMap{"ignoreme": {}},
volName: "ignoreme",
vol: &structs.ClientHostVolumeConfig{},
expectMap: VolumeMap{"ignoreme": {}},
expectChange: false,
},
{
// this should not happen with dynamic vols, but test anyway
name: "change present",
vols: VolumeMap{"changeme": {ID: "before"}},
volName: "changeme",
vol: &structs.ClientHostVolumeConfig{ID: "after"},
expectMap: VolumeMap{"changeme": {ID: "after"}},
expectChange: true,
},
{
// this should only happen during agent start, if a static vol has
// been added to config after a previous dynamic vol was created
// with the same name.
name: "override static",
vols: VolumeMap{"overrideme": {ID: ""}}, // static vols have no ID
volName: "overrideme",
vol: &structs.ClientHostVolumeConfig{ID: "dynamic-vol-id"},
expectMap: VolumeMap{"overrideme": {ID: "dynamic-vol-id"}},
expectChange: true,
expectLog: "overriding static host volume with dynamic: name=overrideme id=dynamic-vol-id",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
log, getLogs := logRecorder(t)
changed := UpdateVolumeMap(log, tc.vols, tc.volName, tc.vol)
must.Eq(t, tc.expectMap, tc.vols)
if tc.expectChange {
must.True(t, changed, must.Sprint("expect volume to have been changed"))
} else {
must.False(t, changed, must.Sprint("expect volume not to have been changed"))
}
must.StrContains(t, getLogs(), tc.expectLog)
})
}
}
func TestWaitForFirstFingerprint(t *testing.T) {
log := testlog.HCLogger(t)
tmp := t.TempDir()
memDB := state.NewMemDB(log)
node := newFakeNode(t)
hvm := NewHostVolumeManager(log, Config{
PluginDir: "",
VolumesDir: tmp,
StateMgr: memDB,
UpdateNodeVols: node.updateVol,
})
plug := &fakePlugin{volsDir: tmp}
hvm.builtIns = map[string]HostVolumePlugin{
"test-plugin": plug,
}
must.NoError(t, memDB.PutDynamicHostVolume(&cstructs.HostVolumeState{
ID: "vol-id",
CreateReq: &cstructs.ClientHostVolumeCreateRequest{
ID: "vol-id",
Name: "vol-name",
PluginID: "test-plugin",
},
}))
ctx := timeout(t)
done := hvm.WaitForFirstFingerprint(ctx)
select {
case <-ctx.Done():
t.Fatal("fingerprint timed out")
case <-done:
}
must.Eq(t, "vol-id", plug.created)
must.Eq(t, VolumeMap{
"vol-name": &structs.ClientHostVolumeConfig{
Name: "vol-name",
ID: "vol-id",
Path: filepath.Join(tmp, "vol-id"),
ReadOnly: false,
},
}, node.vols)
}