T3MP3ST public launch build
This commit is contained in:
1 parent
972247b881
commit
184c7d4e59
1589 files changed
+57150
-506
No files matched your search
+466
@@ -0,0 +1,466 @@
|
||||
// Copyright IBM Corp. 2015, 2025
|
||||
// SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
package hostvolumemanager
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/hashicorp/go-version"
|
||||
cstructs "github.com/hashicorp/nomad/client/structs"
|
||||
"github.com/hashicorp/nomad/helper"
|
||||
)
|
||||
|
||||
const (
|
||||
// environment variables for external plugins
|
||||
|
||||
EnvOperation = "DHV_OPERATION"
|
||||
EnvVolumesDir = "DHV_VOLUMES_DIR"
|
||||
EnvPluginDir = "DHV_PLUGIN_DIR"
|
||||
EnvCreatedPath = "DHV_CREATED_PATH"
|
||||
EnvNamespace = "DHV_NAMESPACE"
|
||||
EnvVolumeName = "DHV_VOLUME_NAME"
|
||||
EnvVolumeID = "DHV_VOLUME_ID"
|
||||
EnvNodeID = "DHV_NODE_ID"
|
||||
EnvNodePool = "DHV_NODE_POOL"
|
||||
EnvCapacityMin = "DHV_CAPACITY_MIN_BYTES"
|
||||
EnvCapacityMax = "DHV_CAPACITY_MAX_BYTES"
|
||||
EnvParameters = "DHV_PARAMETERS"
|
||||
)
|
||||
|
||||
// HostVolumePlugin manages the lifecycle of volumes.
|
||||
type HostVolumePlugin interface {
|
||||
Fingerprint(ctx context.Context) (*PluginFingerprint, error)
|
||||
Create(ctx context.Context, req *cstructs.ClientHostVolumeCreateRequest) (*HostVolumePluginCreateResponse, error)
|
||||
Delete(ctx context.Context, req *cstructs.ClientHostVolumeDeleteRequest) error
|
||||
}
|
||||
|
||||
// PluginFingerprint gets set on the node for volume scheduling.
|
||||
// Plugins are expected to respond to 'fingerprint' calls with json that
|
||||
// unmarshals to this struct.
|
||||
type PluginFingerprint struct {
|
||||
Version *version.Version `json:"version"`
|
||||
}
|
||||
|
||||
// HostVolumePluginCreateResponse returns values to the server that may be shown
|
||||
// to the user or stored in server state. Plugins are expected to respond to
|
||||
// 'create' calls with json that unmarshals to this struct.
|
||||
type HostVolumePluginCreateResponse struct {
|
||||
Path string `json:"path"`
|
||||
SizeBytes int64 `json:"bytes"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// HostVolumePluginDeleteResponse returns values to the server that may be shown
|
||||
// to the user or stored in server state. Plugins are expected to respond to
|
||||
// 'delete' calls with json that unmarshals to this struct.
|
||||
type HostVolumePluginDeleteResponse struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
const HostVolumePluginMkdirID = "mkdir"
|
||||
const HostVolumePluginMkdirVersion = "0.0.1"
|
||||
|
||||
// HostVolumePluginMkdirParams represents the parameters{} that the "mkdir"
|
||||
// plugin will accept.
|
||||
type HostVolumePluginMkdirParams struct {
|
||||
Uid int
|
||||
Gid int
|
||||
Mode os.FileMode
|
||||
}
|
||||
|
||||
var _ HostVolumePlugin = &HostVolumePluginMkdir{}
|
||||
|
||||
// HostVolumePluginMkdir is a plugin that creates a directory within the
|
||||
// specified VolumesDir. It is built-in to Nomad, so is always available.
|
||||
type HostVolumePluginMkdir struct {
|
||||
ID string
|
||||
VolumesDir string
|
||||
|
||||
log hclog.Logger
|
||||
}
|
||||
|
||||
func (p *HostVolumePluginMkdir) Fingerprint(_ context.Context) (*PluginFingerprint, error) {
|
||||
v, err := version.NewVersion(HostVolumePluginMkdirVersion)
|
||||
return &PluginFingerprint{
|
||||
Version: v,
|
||||
}, err
|
||||
}
|
||||
|
||||
func (p *HostVolumePluginMkdir) Create(_ context.Context,
|
||||
req *cstructs.ClientHostVolumeCreateRequest) (*HostVolumePluginCreateResponse, error) {
|
||||
|
||||
path := filepath.Join(p.VolumesDir, req.ID)
|
||||
log := p.log.With(
|
||||
"operation", "create",
|
||||
"volume_id", req.ID,
|
||||
"path", path)
|
||||
log.Debug("running plugin")
|
||||
|
||||
resp := &HostVolumePluginCreateResponse{
|
||||
Path: path,
|
||||
// "mkdir" volumes, being simple directories, have unrestricted size
|
||||
SizeBytes: 0,
|
||||
}
|
||||
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
// already exists
|
||||
return resp, nil
|
||||
} else if !os.IsNotExist(err) {
|
||||
// doesn't exist, but some other path error
|
||||
log.Error("error with path", "error", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
params, err := decodeMkdirParams(req.Parameters)
|
||||
if err != nil {
|
||||
log.Error("error with parameters", "error", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = os.MkdirAll(path, params.Mode)
|
||||
if err != nil {
|
||||
log.Error("error creating directory", "error", err)
|
||||
return nil, fmt.Errorf("error creating directory: %w", err)
|
||||
}
|
||||
|
||||
// os.MkdirAll perms are applied after umask, so the new directory may not
|
||||
// have the exact permissions requested.
|
||||
err = os.Chmod(path, params.Mode)
|
||||
if err != nil {
|
||||
log.Error("error setting directory permission mode", "error", err)
|
||||
return nil, fmt.Errorf("error setting directory permission mode: %w", err)
|
||||
}
|
||||
|
||||
if runtime.GOOS != "windows" {
|
||||
// Chown note: A uid or gid of -1 means to not change that value.
|
||||
if err = os.Chown(path, params.Uid, params.Gid); err != nil {
|
||||
log.Error("error changing owner/group", "error", err, "uid", params.Uid, "gid", params.Gid)
|
||||
|
||||
// Failing to change ownership is fatal for this plugin. Since we have
|
||||
// already created the directory, we should attempt to clean it.
|
||||
// Otherwise, the operator must do this manually.
|
||||
if err := os.RemoveAll(path); err != nil {
|
||||
log.Error("failed to remove directory after create failure",
|
||||
"error", err)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("error changing owner/group: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Debug("plugin ran successfully")
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func decodeMkdirParams(in map[string]string) (HostVolumePluginMkdirParams, error) {
|
||||
// default values if their associated keys are not in the input map
|
||||
out := HostVolumePluginMkdirParams{
|
||||
Mode: 0o700, // "0700"
|
||||
Uid: -1, // this default translates to "do not set" in os.Chown
|
||||
Gid: -1, // ditto
|
||||
}
|
||||
var err error
|
||||
|
||||
for param, val := range in {
|
||||
switch param {
|
||||
case "mode":
|
||||
// mode needs special treatment - it's octal. note that this does
|
||||
// not check whether it's a *reasonable* mode for a directory.
|
||||
// that will be discovered during MkdirAll and subsequent usage
|
||||
// by workloads (which we cannot predict).
|
||||
var number uint64
|
||||
number, err = strconv.ParseUint(val, 8, 32)
|
||||
out.Mode = os.FileMode(number)
|
||||
case "uid":
|
||||
out.Uid, err = strconv.Atoi(val)
|
||||
case "gid":
|
||||
out.Gid, err = strconv.Atoi(val)
|
||||
default:
|
||||
err = fmt.Errorf("unknown mkdir parameter: %q", param)
|
||||
}
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("invalid value for %q: %w", param, err)
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (p *HostVolumePluginMkdir) Delete(_ context.Context, req *cstructs.ClientHostVolumeDeleteRequest) error {
|
||||
path := filepath.Join(p.VolumesDir, req.ID)
|
||||
log := p.log.With(
|
||||
"operation", "delete",
|
||||
"volume_id", req.ID,
|
||||
"path", path)
|
||||
log.Debug("running plugin")
|
||||
|
||||
err := os.RemoveAll(path)
|
||||
if err != nil {
|
||||
log.Error("error with plugin", "error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug("plugin ran successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ HostVolumePlugin = &HostVolumePluginExternal{}
|
||||
|
||||
// NewHostVolumePluginExternal returns an external host volume plugin
|
||||
// if the specified executable exists on disk.
|
||||
func NewHostVolumePluginExternal(log hclog.Logger,
|
||||
pluginDir, filename, volumesDir, nodePool string) (*HostVolumePluginExternal, error) {
|
||||
// this should only be called with already-detected executables,
|
||||
// but we'll double-check it anyway, so we can provide a tidy error message
|
||||
// if it has changed between fingerprinting and execution.
|
||||
executable := filepath.Join(pluginDir, filename)
|
||||
f, err := os.Stat(executable)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("%w: %q", ErrPluginNotExists, filename)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if !helper.IsExecutable(f) {
|
||||
return nil, fmt.Errorf("%w: %q", ErrPluginNotExecutable, filename)
|
||||
}
|
||||
return &HostVolumePluginExternal{
|
||||
ID: filename,
|
||||
Executable: executable,
|
||||
VolumesDir: volumesDir,
|
||||
PluginDir: pluginDir,
|
||||
NodePool: nodePool,
|
||||
log: log,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// HostVolumePluginExternal calls an executable on disk. All operations
|
||||
// *must* be idempotent, and safe to be called concurrently per volume.
|
||||
// For each call, the executable's stdout and stderr may be logged, so plugin
|
||||
// authors should not include any sensitive information in their plugin outputs.
|
||||
type HostVolumePluginExternal struct {
|
||||
ID string
|
||||
Executable string
|
||||
VolumesDir string
|
||||
PluginDir string
|
||||
NodePool string
|
||||
|
||||
log hclog.Logger
|
||||
}
|
||||
|
||||
// Fingerprint calls the executable with the following parameters:
|
||||
// arguments: $1=fingerprint
|
||||
// environment:
|
||||
// - DHV_OPERATION=fingerprint
|
||||
//
|
||||
// Response should be valid JSON on stdout, with a "version" key, e.g.:
|
||||
// {"version": "0.0.1"}
|
||||
// The version value should be a valid version number as allowed by
|
||||
// version.NewVersion()
|
||||
//
|
||||
// Must complete within 5 seconds
|
||||
func (p *HostVolumePluginExternal) Fingerprint(ctx context.Context) (*PluginFingerprint, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, p.Executable, "fingerprint")
|
||||
cmd.Env = []string{EnvOperation + "=fingerprint"}
|
||||
stdout, stderr, err := runCommand(cmd)
|
||||
log := p.log.With(
|
||||
"operation", "fingerprint",
|
||||
"stdout", string(stdout),
|
||||
"stderr", string(stderr),
|
||||
)
|
||||
if err != nil {
|
||||
log.Error("error running plugin", "error", err)
|
||||
return nil, fmt.Errorf("error fingerprinting plugin %q: %w", p.ID, err)
|
||||
}
|
||||
fprint := &PluginFingerprint{}
|
||||
if err := json.Unmarshal(stdout, fprint); err != nil {
|
||||
err = fmt.Errorf("error parsing fingerprint output as json: %w", err)
|
||||
log.Error("error parsing plugin output", "error", err)
|
||||
return nil, err
|
||||
}
|
||||
return fprint, nil
|
||||
}
|
||||
|
||||
// Create calls the executable with the following parameters:
|
||||
// arguments: $1=create
|
||||
// environment:
|
||||
// - DHV_OPERATION=create
|
||||
// - DHV_VOLUMES_DIR={directory to put the volume in}
|
||||
// - DHV_PLUGIN_DIR={path to directory containing plugins}
|
||||
// - DHV_NAMESPACE={volume namespace}
|
||||
// - DHV_VOLUME_NAME={name from the volume specification}
|
||||
// - DHV_VOLUME_ID={volume ID generated by Nomad}
|
||||
// - DHV_NODE_ID={Nomad node ID}
|
||||
// - DHV_NODE_POOL={Nomad node pool}
|
||||
// - DHV_CAPACITY_MIN_BYTES={capacity_min from the volume spec, expressed in bytes}
|
||||
// - DHV_CAPACITY_MAX_BYTES={capacity_max from the volume spec, expressed in bytes}
|
||||
// - DHV_PARAMETERS={stringified json of parameters from the volume spec}
|
||||
//
|
||||
// Response should be valid JSON on stdout with "path" and "bytes", e.g.:
|
||||
// {"path": "/path/that/was/created", "bytes": 50000000}
|
||||
// "path" must be provided to confirm the requested path is what was
|
||||
// created by the plugin. "bytes" is the actual size of the volume created
|
||||
// by the plugin; if excluded, it will default to 0.
|
||||
//
|
||||
// Must complete within 60 seconds (timeout on RPC)
|
||||
func (p *HostVolumePluginExternal) Create(ctx context.Context,
|
||||
req *cstructs.ClientHostVolumeCreateRequest) (*HostVolumePluginCreateResponse, error) {
|
||||
|
||||
params, err := json.Marshal(req.Parameters)
|
||||
if err != nil {
|
||||
// should never happen; req.Parameters is a simple map[string]string
|
||||
return nil, fmt.Errorf("error marshaling volume pramaters: %w", err)
|
||||
}
|
||||
envVars := []string{
|
||||
fmt.Sprintf("%s=%s", EnvOperation, "create"),
|
||||
fmt.Sprintf("%s=%s", EnvVolumesDir, p.VolumesDir),
|
||||
fmt.Sprintf("%s=%s", EnvPluginDir, p.PluginDir),
|
||||
fmt.Sprintf("%s=%s", EnvNodePool, p.NodePool),
|
||||
// values from volume spec
|
||||
fmt.Sprintf("%s=%s", EnvNamespace, req.Namespace),
|
||||
fmt.Sprintf("%s=%s", EnvVolumeName, req.Name),
|
||||
fmt.Sprintf("%s=%s", EnvVolumeID, req.ID),
|
||||
fmt.Sprintf("%s=%d", EnvCapacityMin, req.RequestedCapacityMinBytes),
|
||||
fmt.Sprintf("%s=%d", EnvCapacityMax, req.RequestedCapacityMaxBytes),
|
||||
fmt.Sprintf("%s=%s", EnvNodeID, req.NodeID),
|
||||
fmt.Sprintf("%s=%s", EnvParameters, params),
|
||||
}
|
||||
|
||||
var pluginResp HostVolumePluginCreateResponse
|
||||
log := p.log.With("volume_name", req.Name, "volume_id", req.ID)
|
||||
stdout, _, err := p.runPlugin(ctx, log, "create", envVars)
|
||||
if err != nil {
|
||||
jsonErr := json.Unmarshal(stdout, &pluginResp)
|
||||
if jsonErr != nil {
|
||||
// if we got an error, we can't actually count on getting JSON, so
|
||||
// optimistically look for it and return the original error
|
||||
// otherwise
|
||||
return nil, fmt.Errorf(
|
||||
"error creating volume %q with plugin %q: %w", req.ID, p.ID, err)
|
||||
}
|
||||
return nil, fmt.Errorf("error creating volume %q with plugin %q: %w: %s",
|
||||
req.ID, p.ID, err, pluginResp.Error)
|
||||
}
|
||||
err = json.Unmarshal(stdout, &pluginResp)
|
||||
if err != nil {
|
||||
// note: if a plugin does not return valid json, a volume may be
|
||||
// created without any respective state in Nomad, since we return
|
||||
// an error here after the plugin has done who-knows-what.
|
||||
return nil, err
|
||||
}
|
||||
return &pluginResp, nil
|
||||
}
|
||||
|
||||
// Delete calls the executable with the following parameters:
|
||||
// arguments: $1=delete
|
||||
// environment:
|
||||
// - DHV_OPERATION=delete
|
||||
// - DHV_CREATED_PATH={path that `create` returned}
|
||||
// - DHV_VOLUMES_DIR={directory that volumes should be put in}
|
||||
// - DHV_PLUGIN_DIR={path to directory containing plugins}
|
||||
// - DHV_NAMESPACE={volume namespace}
|
||||
// - DHV_VOLUME_NAME={name from the volume specification}
|
||||
// - DHV_VOLUME_ID={volume ID generated by Nomad}
|
||||
// - DHV_NODE_ID={Nomad node ID}
|
||||
// - DHV_NODE_POOL={Nomad node pool}
|
||||
// - DHV_PARAMETERS={stringified json of parameters from the volume spec}
|
||||
//
|
||||
// Response on stdout is discarded.
|
||||
//
|
||||
// Must complete within 60 seconds (timeout on RPC)
|
||||
func (p *HostVolumePluginExternal) Delete(ctx context.Context,
|
||||
req *cstructs.ClientHostVolumeDeleteRequest) error {
|
||||
|
||||
params, err := json.Marshal(req.Parameters)
|
||||
if err != nil {
|
||||
// should never happen; req.Parameters is a simple map[string]string
|
||||
return fmt.Errorf("error marshaling volume pramaters: %w", err)
|
||||
}
|
||||
envVars := []string{
|
||||
fmt.Sprintf("%s=%s", EnvOperation, "delete"),
|
||||
fmt.Sprintf("%s=%s", EnvVolumesDir, p.VolumesDir),
|
||||
fmt.Sprintf("%s=%s", EnvPluginDir, p.PluginDir),
|
||||
fmt.Sprintf("%s=%s", EnvNodePool, p.NodePool),
|
||||
// from create response
|
||||
fmt.Sprintf("%s=%s", EnvCreatedPath, req.HostPath),
|
||||
// values from volume spec
|
||||
fmt.Sprintf("%s=%s", EnvNamespace, req.Namespace),
|
||||
fmt.Sprintf("%s=%s", EnvVolumeName, req.Name),
|
||||
fmt.Sprintf("%s=%s", EnvVolumeID, req.ID),
|
||||
fmt.Sprintf("%s=%s", EnvNodeID, req.NodeID),
|
||||
fmt.Sprintf("%s=%s", EnvParameters, params),
|
||||
}
|
||||
|
||||
log := p.log.With("volume_name", req.Name, "volume_id", req.ID)
|
||||
stdout, _, err := p.runPlugin(ctx, log, "delete", envVars)
|
||||
if err != nil {
|
||||
var pluginResp HostVolumePluginDeleteResponse
|
||||
jsonErr := json.Unmarshal(stdout, &pluginResp)
|
||||
if jsonErr != nil {
|
||||
// if we got an error, we can't actually count on getting JSON, so
|
||||
// optimistically look for it and return the original error
|
||||
// otherwise
|
||||
return fmt.Errorf("error reading plugin response when deleting volume %q with plugin %q: original error: %w", req.ID, p.ID, err)
|
||||
}
|
||||
return fmt.Errorf("error deleting volume %q with plugin %q: %w: %s",
|
||||
req.ID, p.ID, err, pluginResp.Error)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// runPlugin executes the... executable
|
||||
func (p *HostVolumePluginExternal) runPlugin(ctx context.Context, log hclog.Logger,
|
||||
op string, env []string) (stdout, stderr []byte, err error) {
|
||||
|
||||
log = log.With("operation", op)
|
||||
log.Debug("running plugin")
|
||||
|
||||
// set up plugin execution
|
||||
cmd := exec.CommandContext(ctx, p.Executable, op)
|
||||
cmd.Env = env
|
||||
|
||||
stdout, stderr, err = runCommand(cmd)
|
||||
|
||||
log = log.With(
|
||||
"stdout", string(stdout),
|
||||
"stderr", string(stderr),
|
||||
)
|
||||
if err != nil {
|
||||
log.Error("error with plugin", "error", err)
|
||||
return stdout, stderr, err
|
||||
}
|
||||
log.Debug("plugin ran successfully")
|
||||
return stdout, stderr, nil
|
||||
}
|
||||
|
||||
// runCommand executes the provided Cmd and captures stdout and stderr.
|
||||
func runCommand(cmd *exec.Cmd) (stdout, stderr []byte, err error) {
|
||||
var errBuf bytes.Buffer
|
||||
cmd.Stderr = io.Writer(&errBuf)
|
||||
mErr := &multierror.Error{}
|
||||
stdout, err = cmd.Output()
|
||||
if err != nil {
|
||||
mErr = multierror.Append(mErr, err)
|
||||
}
|
||||
stderr, err = io.ReadAll(&errBuf)
|
||||
if err != nil {
|
||||
mErr = multierror.Append(mErr, err)
|
||||
}
|
||||
return stdout, stderr, helper.FlattenMultierror(mErr.ErrorOrNil())
|
||||
}
|
||||
+346
@@ -0,0 +1,346 @@
|
||||
// Copyright IBM Corp. 2015, 2025
|
||||
// SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
package hostvolumemanager
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/hashicorp/go-version"
|
||||
"github.com/hashicorp/nomad/ci"
|
||||
cstructs "github.com/hashicorp/nomad/client/structs"
|
||||
"github.com/hashicorp/nomad/helper/testlog"
|
||||
"github.com/shoenig/test/must"
|
||||
)
|
||||
|
||||
func TestHostVolumePluginMkdir(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
tmp := t.TempDir()
|
||||
|
||||
plug := &HostVolumePluginMkdir{
|
||||
ID: "test-mkdir-plugin",
|
||||
VolumesDir: tmp,
|
||||
log: testlog.HCLogger(t),
|
||||
}
|
||||
|
||||
// contexts don't matter here, since they're thrown away by this plugin,
|
||||
// but sending timeout contexts anyway, in case the plugin changes later.
|
||||
_, err := plug.Fingerprint(timeout(t))
|
||||
must.NoError(t, err)
|
||||
|
||||
t.Run("happy", func(t *testing.T) {
|
||||
volID := "happy"
|
||||
target := filepath.Join(tmp, volID)
|
||||
// run multiple times, should be idempotent
|
||||
for range 2 {
|
||||
resp, err := plug.Create(timeout(t),
|
||||
&cstructs.ClientHostVolumeCreateRequest{
|
||||
ID: volID, // minimum required by this plugin
|
||||
})
|
||||
must.NoError(t, err)
|
||||
must.Eq(t, &HostVolumePluginCreateResponse{
|
||||
Path: target,
|
||||
SizeBytes: 0,
|
||||
}, resp)
|
||||
must.DirExists(t, target)
|
||||
must.DirMode(t, target, 0o700+os.ModeDir)
|
||||
}
|
||||
|
||||
// delete should be idempotent, too
|
||||
for range 2 {
|
||||
err = plug.Delete(timeout(t),
|
||||
&cstructs.ClientHostVolumeDeleteRequest{
|
||||
ID: volID,
|
||||
})
|
||||
must.NoError(t, err)
|
||||
must.DirNotExists(t, target)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("sad", func(t *testing.T) {
|
||||
volID := "sad"
|
||||
// can't mkdir inside a file
|
||||
plug.VolumesDir = "host_volume_plugin_test.go"
|
||||
t.Cleanup(func() {
|
||||
plug.VolumesDir = tmp
|
||||
})
|
||||
|
||||
resp, err := plug.Create(timeout(t),
|
||||
&cstructs.ClientHostVolumeCreateRequest{
|
||||
ID: volID, // minimum required by this plugin
|
||||
})
|
||||
must.ErrorContains(t, err, "host_volume_plugin_test.go/sad: not a directory")
|
||||
must.Nil(t, resp)
|
||||
|
||||
err = plug.Delete(timeout(t),
|
||||
&cstructs.ClientHostVolumeDeleteRequest{
|
||||
ID: volID,
|
||||
})
|
||||
must.ErrorContains(t, err, "host_volume_plugin_test.go/sad: not a directory")
|
||||
})
|
||||
|
||||
t.Run("happy params", func(t *testing.T) {
|
||||
volID := "happy_params"
|
||||
target := filepath.Join(tmp, volID)
|
||||
currentUser, err := user.Current()
|
||||
must.NoError(t, err)
|
||||
// run multiple times, should be idempotent
|
||||
for range 2 {
|
||||
_, err := plug.Create(timeout(t),
|
||||
&cstructs.ClientHostVolumeCreateRequest{
|
||||
ID: volID,
|
||||
Parameters: map[string]string{
|
||||
"uid": currentUser.Uid,
|
||||
"gid": currentUser.Gid,
|
||||
"mode": "0777",
|
||||
},
|
||||
})
|
||||
must.NoError(t, err)
|
||||
must.DirExists(t, target)
|
||||
must.DirMode(t, target, 0o777+os.ModeDir)
|
||||
}
|
||||
|
||||
err = plug.Delete(timeout(t),
|
||||
&cstructs.ClientHostVolumeDeleteRequest{
|
||||
ID: volID,
|
||||
})
|
||||
must.NoError(t, err)
|
||||
must.DirNotExists(t, target)
|
||||
})
|
||||
|
||||
t.Run("sad params", func(t *testing.T) {
|
||||
volID := "sad_params"
|
||||
// test one representative error; decodeMkdirParams has its own tests
|
||||
resp, err := plug.Create(timeout(t),
|
||||
&cstructs.ClientHostVolumeCreateRequest{
|
||||
ID: volID,
|
||||
Parameters: map[string]string{
|
||||
"mode": "invalid",
|
||||
},
|
||||
})
|
||||
must.ErrorContains(t, err, "invalid value")
|
||||
must.Nil(t, resp)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDecodeMkdirParams(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
params map[string]string
|
||||
expect HostVolumePluginMkdirParams
|
||||
err string
|
||||
}{
|
||||
{
|
||||
name: "none ok",
|
||||
params: nil,
|
||||
expect: HostVolumePluginMkdirParams{
|
||||
Uid: -1,
|
||||
Gid: -1,
|
||||
Mode: os.FileMode(0700),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "all ok",
|
||||
params: map[string]string{
|
||||
"uid": "1",
|
||||
"gid": "2",
|
||||
"mode": "0444",
|
||||
},
|
||||
expect: HostVolumePluginMkdirParams{
|
||||
Uid: 1,
|
||||
Gid: 2,
|
||||
Mode: os.FileMode(0444),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid mode: bad number",
|
||||
params: map[string]string{
|
||||
// this is what happens if you put mode=0700 instead of
|
||||
// mode="0700" in the HCL spec.
|
||||
"mode": "493",
|
||||
},
|
||||
err: `invalid value for "mode"`,
|
||||
},
|
||||
{
|
||||
name: "invalid mode: string",
|
||||
params: map[string]string{
|
||||
"mode": "consider the lobster",
|
||||
},
|
||||
err: `invalid value for "mode"`,
|
||||
},
|
||||
{
|
||||
name: "invalid uid",
|
||||
params: map[string]string{
|
||||
"uid": "a supposedly fun thing i'll never do again",
|
||||
},
|
||||
err: `invalid value for "uid"`,
|
||||
},
|
||||
{
|
||||
name: "invalid gid",
|
||||
params: map[string]string{
|
||||
"gid": "surely you jest",
|
||||
},
|
||||
err: `invalid value for "gid"`,
|
||||
},
|
||||
{
|
||||
name: "unknown param",
|
||||
params: map[string]string{
|
||||
"what": "the hell is water?",
|
||||
},
|
||||
err: `unknown mkdir parameter: "what"`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := decodeMkdirParams(tc.params)
|
||||
if tc.err != "" {
|
||||
must.ErrorContains(t, err, tc.err)
|
||||
} else {
|
||||
must.NoError(t, err)
|
||||
must.Eq(t, tc.expect, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewHostVolumePluginExternal(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
log := testlog.HCLogger(t)
|
||||
var err error
|
||||
|
||||
_, err = NewHostVolumePluginExternal(log, ".", "non-existent", "target", "")
|
||||
must.ErrorIs(t, err, ErrPluginNotExists)
|
||||
|
||||
_, err = NewHostVolumePluginExternal(log, ".", "host_volume_plugin_test.go", "target", "")
|
||||
must.ErrorIs(t, err, ErrPluginNotExecutable)
|
||||
|
||||
t.Run("unix", func(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("skipped because windows") // db TODO(1.10.0)
|
||||
}
|
||||
p, err := NewHostVolumePluginExternal(log,
|
||||
"./test_fixtures", "test_plugin.sh", "test-target", "test-pool")
|
||||
must.NoError(t, err)
|
||||
must.Eq(t, &HostVolumePluginExternal{
|
||||
ID: "test_plugin.sh",
|
||||
Executable: "test_fixtures/test_plugin.sh",
|
||||
VolumesDir: "test-target",
|
||||
PluginDir: "./test_fixtures",
|
||||
NodePool: "test-pool",
|
||||
log: log,
|
||||
}, p)
|
||||
})
|
||||
}
|
||||
|
||||
func TestHostVolumePluginExternal(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("skipped because windows") // db TODO(1.10.0)
|
||||
}
|
||||
|
||||
volID := "test-vol-id"
|
||||
tmp := t.TempDir()
|
||||
target := filepath.Join(tmp, volID)
|
||||
|
||||
expectVersion, err := version.NewVersion("0.0.2")
|
||||
must.NoError(t, err)
|
||||
|
||||
t.Run("happy", func(t *testing.T) {
|
||||
|
||||
log, getLogs := logRecorder(t)
|
||||
plug, err := NewHostVolumePluginExternal(log,
|
||||
"./test_fixtures", "test_plugin.sh", tmp, "test-node-pool")
|
||||
must.NoError(t, err)
|
||||
|
||||
// fingerprint
|
||||
v, err := plug.Fingerprint(timeout(t))
|
||||
logged := getLogs()
|
||||
must.NoError(t, err, must.Sprintf("logs: %s", logged))
|
||||
must.Eq(t, expectVersion, v.Version, must.Sprintf("logs: %s", logged))
|
||||
|
||||
// create
|
||||
resp, err := plug.Create(timeout(t),
|
||||
&cstructs.ClientHostVolumeCreateRequest{
|
||||
Name: "test-vol-name",
|
||||
ID: volID,
|
||||
Namespace: "test-namespace",
|
||||
NodeID: "test-node",
|
||||
RequestedCapacityMinBytes: 5,
|
||||
RequestedCapacityMaxBytes: 10,
|
||||
Parameters: map[string]string{"key": "val"},
|
||||
})
|
||||
logged = getLogs()
|
||||
must.NoError(t, err, must.Sprintf("logs: %s", logged))
|
||||
|
||||
must.Eq(t, &HostVolumePluginCreateResponse{
|
||||
Path: target,
|
||||
SizeBytes: 5,
|
||||
}, resp)
|
||||
must.DirExists(t, target)
|
||||
must.StrContains(t, logged, "OPERATION=create") // stderr from `env`
|
||||
must.StrContains(t, logged, `stdout="{`) // stdout from printf
|
||||
|
||||
// delete
|
||||
err = plug.Delete(timeout(t),
|
||||
&cstructs.ClientHostVolumeDeleteRequest{
|
||||
Name: "test-vol-name",
|
||||
ID: volID,
|
||||
HostPath: resp.Path,
|
||||
Namespace: "test-namespace",
|
||||
NodeID: "test-node",
|
||||
Parameters: map[string]string{"key": "val"},
|
||||
})
|
||||
logged = getLogs()
|
||||
must.NoError(t, err, must.Sprintf("logs: %s", logged))
|
||||
must.DirNotExists(t, target)
|
||||
must.StrContains(t, logged, "OPERATION=delete") // stderr from `env`
|
||||
must.StrContains(t, logged, "removed directory") // stdout from `rm -v`
|
||||
})
|
||||
|
||||
t.Run("sad", func(t *testing.T) {
|
||||
|
||||
log, getLogs := logRecorder(t)
|
||||
plug, err := NewHostVolumePluginExternal(log, "./test_fixtures", "test_plugin_sad.sh", tmp, "")
|
||||
must.NoError(t, err)
|
||||
|
||||
v, err := plug.Fingerprint(timeout(t))
|
||||
must.EqError(t, err, `error fingerprinting plugin "test_plugin_sad.sh": exit status 1`)
|
||||
must.Nil(t, v)
|
||||
logged := getLogs()
|
||||
must.StrContains(t, logged, "fingerprint: sad plugin is sad")
|
||||
must.StrContains(t, logged, "fingerprint: it tells you all about it in stderr")
|
||||
|
||||
// reset logger
|
||||
log, getLogs = logRecorder(t)
|
||||
plug.log = log
|
||||
|
||||
resp, err := plug.Create(timeout(t),
|
||||
&cstructs.ClientHostVolumeCreateRequest{
|
||||
ID: volID,
|
||||
})
|
||||
must.EqError(t, err, `error creating volume "test-vol-id" with plugin "test_plugin_sad.sh": exit status 1: create: sad plugin is sad`)
|
||||
must.Nil(t, resp)
|
||||
logged = getLogs()
|
||||
must.StrContains(t, logged, "create: sad plugin is sad")
|
||||
must.StrContains(t, logged, "create: it tells you all about it in stderr")
|
||||
|
||||
log, getLogs = logRecorder(t)
|
||||
plug.log = log
|
||||
|
||||
err = plug.Delete(timeout(t),
|
||||
&cstructs.ClientHostVolumeDeleteRequest{
|
||||
ID: volID,
|
||||
})
|
||||
must.EqError(t, err, `error deleting volume "test-vol-id" with plugin "test_plugin_sad.sh": exit status 1: delete: sad plugin is sad`)
|
||||
logged = getLogs()
|
||||
must.StrContains(t, logged, "delete: sad plugin is sad")
|
||||
must.StrContains(t, logged, "delete: it tells you all about it in stderr")
|
||||
})
|
||||
}
|
||||
+377
@@ -0,0 +1,377 @@
|
||||
// Copyright IBM Corp. 2015, 2025
|
||||
// SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
package hostvolumemanager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
cstructs "github.com/hashicorp/nomad/client/structs"
|
||||
"github.com/hashicorp/nomad/helper"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrPluginNotExists = errors.New("no such plugin")
|
||||
ErrPluginNotExecutable = errors.New("plugin not executable")
|
||||
ErrVolumeNameExists = errors.New("volume name already exists on this node")
|
||||
)
|
||||
|
||||
// HostVolumeStateManager manages the lifecycle of volumes in client state.
|
||||
type HostVolumeStateManager interface {
|
||||
PutDynamicHostVolume(*cstructs.HostVolumeState) error
|
||||
GetDynamicHostVolumes() ([]*cstructs.HostVolumeState, error)
|
||||
DeleteDynamicHostVolume(string) error
|
||||
}
|
||||
|
||||
// Config is used to configure a HostVolumeManager.
|
||||
type Config struct {
|
||||
// PluginDir is where external plugins may be found.
|
||||
PluginDir string
|
||||
|
||||
// VolumesDir is where plugins should place the directory
|
||||
// that will later become a volume's HostPath
|
||||
VolumesDir string
|
||||
|
||||
// NodePool is passed into external plugin execution environment.
|
||||
NodePool string
|
||||
|
||||
// StateMgr manages client state to restore on agent restarts.
|
||||
StateMgr HostVolumeStateManager
|
||||
|
||||
// UpdateNodeVols is run to update the node when a volume is created
|
||||
// or deleted.
|
||||
UpdateNodeVols HostVolumeNodeUpdater
|
||||
}
|
||||
|
||||
// HostVolumeManager executes plugins, manages volume metadata in client state,
|
||||
// and registers volumes with the client node.
|
||||
type HostVolumeManager struct {
|
||||
pluginDir string
|
||||
volumesDir string
|
||||
nodePool string
|
||||
stateMgr HostVolumeStateManager
|
||||
updateNodeVols HostVolumeNodeUpdater
|
||||
builtIns map[string]HostVolumePlugin
|
||||
locker *volLocker
|
||||
log hclog.Logger
|
||||
}
|
||||
|
||||
// NewHostVolumeManager includes default builtin plugins.
|
||||
func NewHostVolumeManager(logger hclog.Logger, config Config) *HostVolumeManager {
|
||||
logger = logger.Named("host_volume_manager")
|
||||
return &HostVolumeManager{
|
||||
pluginDir: config.PluginDir,
|
||||
volumesDir: config.VolumesDir,
|
||||
nodePool: config.NodePool,
|
||||
stateMgr: config.StateMgr,
|
||||
updateNodeVols: config.UpdateNodeVols,
|
||||
builtIns: map[string]HostVolumePlugin{
|
||||
HostVolumePluginMkdirID: &HostVolumePluginMkdir{
|
||||
ID: HostVolumePluginMkdirID,
|
||||
VolumesDir: config.VolumesDir,
|
||||
log: logger.With("plugin_id", HostVolumePluginMkdirID),
|
||||
},
|
||||
},
|
||||
locker: &volLocker{},
|
||||
log: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// Create runs the appropriate plugin for the given request, saves the request
|
||||
// to state, and updates the node with the volume.
|
||||
func (hvm *HostVolumeManager) Create(ctx context.Context,
|
||||
req *cstructs.ClientHostVolumeCreateRequest) (*cstructs.ClientHostVolumeCreateResponse, error) {
|
||||
|
||||
log := hvm.log.With("volume_name", req.Name, "volume_id", req.ID)
|
||||
|
||||
plug, err := hvm.getPlugin(req.PluginID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// can't have two of the same volume name w/ different IDs per client node
|
||||
isNewVolume, err := hvm.locker.lock(req.Name, req.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pluginResp, err := plug.Create(ctx, req)
|
||||
if err != nil {
|
||||
hvm.locker.release(req.Name)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
volState := &cstructs.HostVolumeState{
|
||||
ID: req.ID,
|
||||
HostPath: pluginResp.Path,
|
||||
CreateReq: req,
|
||||
}
|
||||
if err := hvm.stateMgr.PutDynamicHostVolume(volState); err != nil {
|
||||
// if we fail to write to state on initial create,
|
||||
// delete the volume so it isn't left lying around
|
||||
// without Nomad knowing about it.
|
||||
log.Error("failed to save volume in client state", "error", err)
|
||||
if isNewVolume {
|
||||
log.Error("initial create detected, running delete")
|
||||
delErr := plug.Delete(ctx, &cstructs.ClientHostVolumeDeleteRequest{
|
||||
ID: req.ID,
|
||||
PluginID: req.PluginID,
|
||||
NodeID: req.NodeID,
|
||||
HostPath: hvm.volumesDir,
|
||||
Parameters: req.Parameters,
|
||||
})
|
||||
if delErr != nil {
|
||||
log.Warn("error deleting volume after state store failure", "error", delErr)
|
||||
err = multierror.Append(err, delErr)
|
||||
}
|
||||
// free up the volume name whether delete succeeded or not.
|
||||
hvm.locker.release(req.Name)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
hvm.updateNodeVols(req.Name, genVolConfig(req, pluginResp.Path))
|
||||
|
||||
resp := &cstructs.ClientHostVolumeCreateResponse{
|
||||
VolumeName: req.Name,
|
||||
VolumeID: req.ID,
|
||||
HostPath: pluginResp.Path,
|
||||
CapacityBytes: pluginResp.SizeBytes,
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Register saves the request to state, and updates the node with the volume.
|
||||
func (hvm *HostVolumeManager) Register(ctx context.Context,
|
||||
req *cstructs.ClientHostVolumeRegisterRequest) error {
|
||||
|
||||
log := hvm.log.With("volume_name", req.Name, "volume_id", req.ID)
|
||||
|
||||
// can't have two of the same volume name w/ different IDs per client node
|
||||
if _, err := hvm.locker.lock(req.Name, req.ID); err != nil {
|
||||
log.Error("duplicate volume name", "error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
_, err := os.Stat(req.HostPath)
|
||||
if err != nil {
|
||||
hvm.locker.release(req.Name)
|
||||
err = fmt.Errorf("could not verify host path for %q: %w", req.Name, err)
|
||||
log.Error("error registering volume", "error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// generate a stub create request and plugin response for the fingerprint
|
||||
// and client state
|
||||
creq := &cstructs.ClientHostVolumeCreateRequest{
|
||||
ID: req.ID,
|
||||
Name: req.Name,
|
||||
NodeID: req.NodeID,
|
||||
Parameters: req.Parameters,
|
||||
}
|
||||
volState := &cstructs.HostVolumeState{
|
||||
ID: req.ID,
|
||||
CreateReq: creq,
|
||||
HostPath: req.HostPath,
|
||||
}
|
||||
if err := hvm.stateMgr.PutDynamicHostVolume(volState); err != nil {
|
||||
log.Error("failed to save volume in state", "error", err)
|
||||
hvm.locker.release(req.Name)
|
||||
return err
|
||||
}
|
||||
|
||||
hvm.updateNodeVols(req.Name, genVolConfig(creq, req.HostPath))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete runs the appropriate plugin for the given request, removes it from
|
||||
// state, and updates the node to remove the volume.
|
||||
func (hvm *HostVolumeManager) Delete(ctx context.Context,
|
||||
req *cstructs.ClientHostVolumeDeleteRequest) (*cstructs.ClientHostVolumeDeleteResponse, error) {
|
||||
|
||||
if req.PluginID != "" {
|
||||
plug, err := hvm.getPlugin(req.PluginID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = plug.Delete(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := hvm.stateMgr.DeleteDynamicHostVolume(req.ID); err != nil {
|
||||
hvm.log.Error("failed to delete volume in state", "volume_id", req.ID, "error", err)
|
||||
return nil, err // bail so a user may retry
|
||||
}
|
||||
|
||||
// free up volume name for reuse
|
||||
hvm.locker.release(req.Name)
|
||||
|
||||
hvm.updateNodeVols(req.Name, nil)
|
||||
|
||||
resp := &cstructs.ClientHostVolumeDeleteResponse{
|
||||
VolumeName: req.Name,
|
||||
VolumeID: req.ID,
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// getPlugin finds either a built-in plugin or an external plugin.
|
||||
func (hvm *HostVolumeManager) getPlugin(id string) (HostVolumePlugin, error) {
|
||||
if plug, ok := hvm.builtIns[id]; ok {
|
||||
return plug, nil
|
||||
}
|
||||
log := hvm.log.With("plugin_id", id)
|
||||
return NewHostVolumePluginExternal(log, hvm.pluginDir, id, hvm.volumesDir, hvm.nodePool)
|
||||
}
|
||||
|
||||
// restoreFromState loads all volumes from client state and runs Create for
|
||||
// each one, so volumes are restored upon agent restart or host reboot.
|
||||
func (hvm *HostVolumeManager) restoreFromState(ctx context.Context) (VolumeMap, error) {
|
||||
vols, err := hvm.stateMgr.GetDynamicHostVolumes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
volumes := make(VolumeMap)
|
||||
var mut sync.Mutex
|
||||
|
||||
if len(vols) == 0 {
|
||||
return volumes, nil // nothing to do
|
||||
}
|
||||
|
||||
group := multierror.Group{}
|
||||
for _, vol := range vols {
|
||||
group.Go(func() error {
|
||||
var volCfg *structs.ClientHostVolumeConfig
|
||||
var err error
|
||||
if vol.CreateReq.PluginID == "" {
|
||||
volCfg, err = hvm.restoreForRegister(vol)
|
||||
} else {
|
||||
volCfg, err = hvm.restoreForCreate(ctx, vol)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mut.Lock()
|
||||
volumes[vol.CreateReq.Name] = volCfg
|
||||
mut.Unlock()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
mErr := group.Wait()
|
||||
return volumes, helper.FlattenMultierror(mErr.ErrorOrNil())
|
||||
}
|
||||
|
||||
// restoreForCreate restores a single volume that was previously created by
|
||||
// Create, by "recreating" the volumes. Plugins have the best knowledge of their
|
||||
// side effects, and they must be idempotent.
|
||||
func (hvm *HostVolumeManager) restoreForCreate(ctx context.Context, vol *cstructs.HostVolumeState) (*structs.ClientHostVolumeConfig, error) {
|
||||
// missing plugins with associated volumes in state are considered
|
||||
// client-stopping errors. they need to be fixed by cluster admins.
|
||||
plug, err := hvm.getPlugin(vol.CreateReq.PluginID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// lock the name so future creates can't produce duplicates.
|
||||
_, err = hvm.locker.lock(vol.CreateReq.Name, vol.CreateReq.ID)
|
||||
// state should never have duplicate vol names, and restore happens
|
||||
// prior to node registration, so new creates shouldn't come in
|
||||
// concurrently, but check for error just in case.
|
||||
if err != nil {
|
||||
hvm.log.Error("error during restore",
|
||||
"volume_name", vol.CreateReq.Name,
|
||||
"volume_id", vol.CreateReq.ID,
|
||||
"error", err)
|
||||
// don't stop the world if it does happen, because an admin
|
||||
// couldn't do anything about it short of wiping client state.
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
resp, err := plug.Create(ctx, vol.CreateReq)
|
||||
if err != nil {
|
||||
// plugin execution errors are only logged
|
||||
hvm.log.Error("failed to restore",
|
||||
"plugin_id", vol.CreateReq.PluginID, "volume_id", vol.ID, "error", err)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return genVolConfig(vol.CreateReq, resp.Path), nil
|
||||
}
|
||||
|
||||
// restoreForRegister restores a single volume that was previously created by
|
||||
// Register, by converting the stored struct. It otherwise behaves the same as
|
||||
// restoreForCreate.
|
||||
func (hvm *HostVolumeManager) restoreForRegister(vol *cstructs.HostVolumeState) (*structs.ClientHostVolumeConfig, error) {
|
||||
_, err := hvm.locker.lock(vol.CreateReq.Name, vol.CreateReq.ID)
|
||||
if err != nil {
|
||||
hvm.log.Error("error during restore",
|
||||
"volume_name", vol.CreateReq.Name,
|
||||
"volume_id", vol.CreateReq.ID,
|
||||
"error", err)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
_, err = os.Stat(vol.HostPath)
|
||||
if err != nil {
|
||||
hvm.log.Error("failed to restore: could not verify host path",
|
||||
"volume_id", vol.ID, "error", err, "path", vol.HostPath)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return genVolConfig(vol.CreateReq, vol.HostPath), nil
|
||||
}
|
||||
|
||||
// genVolConfig generates the host volume config for the node to report as
|
||||
// available to the servers for job scheduling.
|
||||
func genVolConfig(req *cstructs.ClientHostVolumeCreateRequest, hostPath string) *structs.ClientHostVolumeConfig {
|
||||
return &structs.ClientHostVolumeConfig{
|
||||
Name: req.Name,
|
||||
ID: req.ID,
|
||||
Path: hostPath,
|
||||
|
||||
// dynamic volumes, like CSI, have more robust `capabilities`,
|
||||
// so we always set ReadOnly to false, and let the scheduler
|
||||
// decide when to ignore this and check capabilities instead.
|
||||
ReadOnly: false,
|
||||
}
|
||||
}
|
||||
|
||||
// volLocker is used to ensure that volumes on each node are unique by name.
|
||||
// The volume scheduler will prevent this too, but only after node fingerprint,
|
||||
// so we need to protect against concurrent duplicate creates.
|
||||
type volLocker struct {
|
||||
locks sync.Map
|
||||
}
|
||||
|
||||
// lock the provided name, return true if it was not already locked,
|
||||
// and error if it was already locked with a different ID.
|
||||
func (l *volLocker) lock(name, id string) (bool, error) {
|
||||
current, exists := l.locks.LoadOrStore(name, id)
|
||||
if exists && id != current.(string) {
|
||||
return false, ErrVolumeNameExists
|
||||
}
|
||||
return !exists, nil
|
||||
}
|
||||
|
||||
func (l *volLocker) release(name string) {
|
||||
l.locks.Delete(name)
|
||||
}
|
||||
|
||||
func (l *volLocker) isLocked(name string) bool {
|
||||
_, locked := l.locks.Load(name)
|
||||
return locked
|
||||
}
|
||||
+441
@@ -0,0 +1,441 @@
|
||||
// Copyright IBM Corp. 2015, 2025
|
||||
// SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
package hostvolumemanager
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/go-version"
|
||||
cstate "github.com/hashicorp/nomad/client/state"
|
||||
cstructs "github.com/hashicorp/nomad/client/structs"
|
||||
"github.com/hashicorp/nomad/helper/testlog"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
"github.com/shoenig/test"
|
||||
"github.com/shoenig/test/must"
|
||||
)
|
||||
|
||||
func TestHostVolumeManager(t *testing.T) {
|
||||
log := testlog.HCLogger(t)
|
||||
errDB := &cstate.ErrDB{}
|
||||
memDB := cstate.NewMemDB(log)
|
||||
node := newFakeNode(t)
|
||||
|
||||
volumesDirCreate := t.TempDir()
|
||||
volumesDirRegister := t.TempDir()
|
||||
|
||||
hvm := NewHostVolumeManager(log, Config{
|
||||
PluginDir: "./test_fixtures",
|
||||
VolumesDir: volumesDirCreate,
|
||||
StateMgr: errDB,
|
||||
UpdateNodeVols: node.updateVol,
|
||||
})
|
||||
|
||||
plug := &fakePlugin{volsDir: volumesDirCreate}
|
||||
hvm.builtIns["test-plugin"] = plug
|
||||
|
||||
ctx := timeout(t)
|
||||
|
||||
t.Run("create", func(t *testing.T) {
|
||||
// plugin doesn't exist
|
||||
name := "created-volume"
|
||||
req := &cstructs.ClientHostVolumeCreateRequest{
|
||||
Name: name,
|
||||
ID: "vol-id-1",
|
||||
PluginID: "nope",
|
||||
|
||||
RequestedCapacityMinBytes: 5,
|
||||
}
|
||||
_, err := hvm.Create(ctx, req)
|
||||
must.ErrorIs(t, err, ErrPluginNotExists)
|
||||
|
||||
// error from plugin
|
||||
req.PluginID = "test-plugin"
|
||||
plug.createErr = errors.New("sad create")
|
||||
_, err = hvm.Create(ctx, req)
|
||||
must.ErrorIs(t, err, plug.createErr)
|
||||
assertNotLocked(t, hvm, name)
|
||||
plug.reset()
|
||||
|
||||
// error saving state, then error from cleanup attempt
|
||||
plug.deleteErr = errors.New("sad delete")
|
||||
_, err = hvm.Create(ctx, req)
|
||||
must.ErrorIs(t, err, cstate.ErrDBError)
|
||||
must.ErrorIs(t, err, plug.deleteErr)
|
||||
assertNotLocked(t, hvm, name)
|
||||
plug.reset()
|
||||
|
||||
// error saving state, successful cleanup
|
||||
_, err = hvm.Create(ctx, req)
|
||||
must.ErrorIs(t, err, cstate.ErrDBError)
|
||||
must.Eq(t, "vol-id-1", plug.deleted)
|
||||
assertNotLocked(t, hvm, name)
|
||||
plug.reset()
|
||||
|
||||
// happy path
|
||||
hvm.stateMgr = memDB
|
||||
resp, err := hvm.Create(ctx, req)
|
||||
must.NoError(t, err)
|
||||
expectResp := &cstructs.ClientHostVolumeCreateResponse{
|
||||
VolumeName: "created-volume",
|
||||
VolumeID: "vol-id-1",
|
||||
HostPath: filepath.Join(volumesDirCreate, "vol-id-1"),
|
||||
CapacityBytes: 5,
|
||||
}
|
||||
must.Eq(t, expectResp, resp)
|
||||
stateDBs, err := memDB.GetDynamicHostVolumes()
|
||||
must.NoError(t, err)
|
||||
// should be saved to state
|
||||
must.Len(t, 1, stateDBs)
|
||||
must.Eq(t, "vol-id-1", stateDBs[0].ID)
|
||||
must.Eq(t, "vol-id-1", stateDBs[0].CreateReq.ID)
|
||||
// should be registered with node
|
||||
must.MapContainsKey(t, node.vols, name, must.Sprintf("no %q in %+v", name, node.vols))
|
||||
assertLocked(t, hvm, name)
|
||||
|
||||
// repeat create with same ID but different size may update the volume
|
||||
req.RequestedCapacityMinBytes = 10
|
||||
expectResp.CapacityBytes = 10
|
||||
resp, err = hvm.Create(ctx, req)
|
||||
must.NoError(t, err)
|
||||
must.Eq(t, expectResp, resp)
|
||||
|
||||
// error saving state on restore/update should not run delete
|
||||
hvm.stateMgr = errDB
|
||||
resp, err = hvm.Create(ctx, req)
|
||||
must.ErrorIs(t, err, cstate.ErrDBError)
|
||||
must.Nil(t, resp)
|
||||
must.Eq(t, "", plug.deleted)
|
||||
plug.reset()
|
||||
hvm.stateMgr = memDB
|
||||
|
||||
// duplicate create with the same vol name but different ID should fail
|
||||
_, err = hvm.Create(ctx, &cstructs.ClientHostVolumeCreateRequest{
|
||||
Name: name,
|
||||
ID: "different-vol-id",
|
||||
PluginID: "test-plugin",
|
||||
})
|
||||
must.ErrorIs(t, err, ErrVolumeNameExists)
|
||||
})
|
||||
|
||||
t.Run("register", func(t *testing.T) {
|
||||
name := "registered-volume"
|
||||
req := &cstructs.ClientHostVolumeRegisterRequest{
|
||||
ID: "vol-id-2",
|
||||
Name: name,
|
||||
HostPath: volumesDirRegister,
|
||||
CapacityBytes: 1000,
|
||||
}
|
||||
err := hvm.Register(ctx, req)
|
||||
must.NoError(t, err)
|
||||
|
||||
// should be saved to state and registered with node
|
||||
stateDBs, err := memDB.GetDynamicHostVolumes()
|
||||
must.NoError(t, err)
|
||||
must.Len(t, 2, stateDBs)
|
||||
sort.Slice(stateDBs, func(i, j int) bool { return stateDBs[i].ID < stateDBs[j].ID })
|
||||
must.Eq(t, "vol-id-2", stateDBs[1].ID)
|
||||
must.Eq(t, "vol-id-2", stateDBs[1].CreateReq.ID)
|
||||
must.MapContainsKey(t, node.vols, name, must.Sprintf("no %q in %+v", name, node.vols))
|
||||
assertLocked(t, hvm, name)
|
||||
})
|
||||
|
||||
// despite being a subtest, this needs to run after "create" and "register"
|
||||
t.Run("delete", func(t *testing.T) {
|
||||
name := "created-volume"
|
||||
// should be locked from "create" above
|
||||
assertLocked(t, hvm, name)
|
||||
|
||||
// plugin doesn't exist
|
||||
req := &cstructs.ClientHostVolumeDeleteRequest{
|
||||
Name: name,
|
||||
ID: "vol-id-1",
|
||||
PluginID: "nope",
|
||||
}
|
||||
_, err := hvm.Delete(ctx, req)
|
||||
must.ErrorIs(t, err, ErrPluginNotExists)
|
||||
assertLocked(t, hvm, name)
|
||||
|
||||
// error from plugin
|
||||
req.PluginID = "test-plugin"
|
||||
plug.deleteErr = errors.New("sad delete")
|
||||
_, err = hvm.Delete(ctx, req)
|
||||
must.ErrorIs(t, err, plug.deleteErr)
|
||||
assertLocked(t, hvm, name)
|
||||
plug.reset()
|
||||
|
||||
// error saving state
|
||||
hvm.stateMgr = errDB
|
||||
_, err = hvm.Delete(ctx, req)
|
||||
must.ErrorIs(t, err, cstate.ErrDBError)
|
||||
assertLocked(t, hvm, name)
|
||||
|
||||
// happy path
|
||||
hvm.stateMgr = memDB
|
||||
|
||||
// and delete it
|
||||
resp, err := hvm.Delete(ctx, req)
|
||||
must.NoError(t, err)
|
||||
must.Eq(t, &cstructs.ClientHostVolumeDeleteResponse{
|
||||
VolumeName: "created-volume",
|
||||
VolumeID: "vol-id-1",
|
||||
}, resp)
|
||||
must.Eq(t, VolumeMap{
|
||||
"registered-volume": &structs.ClientHostVolumeConfig{
|
||||
Name: "registered-volume",
|
||||
Path: volumesDirRegister,
|
||||
ID: "vol-id-2",
|
||||
},
|
||||
}, node.vols, must.Sprint("created-volume should be deleted from node"))
|
||||
stateVols, err := memDB.GetDynamicHostVolumes()
|
||||
must.NoError(t, err)
|
||||
must.Len(t, 1, stateVols, must.Sprint("only one volume should be deleted"))
|
||||
|
||||
assertNotLocked(t, hvm, name)
|
||||
assertLocked(t, hvm, "registered-volume")
|
||||
|
||||
req.Name = "registered-volume"
|
||||
req.ID = "vol-id-2"
|
||||
req.PluginID = ""
|
||||
resp, err = hvm.Delete(ctx, req)
|
||||
must.NoError(t, err)
|
||||
|
||||
must.Eq(t, VolumeMap{}, node.vols, must.Sprint("all volumes should be deleted from node"))
|
||||
stateVols, err = memDB.GetDynamicHostVolumes()
|
||||
must.NoError(t, err)
|
||||
must.Nil(t, stateVols, must.Sprint("all volumes should be deleted"))
|
||||
assertNotLocked(t, hvm, "registered-volume")
|
||||
})
|
||||
}
|
||||
|
||||
type fakePlugin struct {
|
||||
volsDir string
|
||||
created string
|
||||
deleted string
|
||||
fingerprintErr error
|
||||
createErr error
|
||||
deleteErr error
|
||||
}
|
||||
|
||||
func (p *fakePlugin) reset() {
|
||||
p.deleted, p.fingerprintErr, p.createErr, p.deleteErr = "", nil, nil, nil
|
||||
}
|
||||
|
||||
func (p *fakePlugin) Fingerprint(_ context.Context) (*PluginFingerprint, error) {
|
||||
if p.fingerprintErr != nil {
|
||||
return nil, p.fingerprintErr
|
||||
}
|
||||
v, err := version.NewVersion("0.0.1")
|
||||
return &PluginFingerprint{
|
||||
Version: v,
|
||||
}, err
|
||||
}
|
||||
|
||||
func (p *fakePlugin) Create(_ context.Context, req *cstructs.ClientHostVolumeCreateRequest) (*HostVolumePluginCreateResponse, error) {
|
||||
if p.createErr != nil {
|
||||
return nil, p.createErr
|
||||
}
|
||||
p.created = req.ID
|
||||
return &HostVolumePluginCreateResponse{
|
||||
Path: filepath.Join(p.volsDir, req.ID),
|
||||
SizeBytes: req.RequestedCapacityMinBytes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *fakePlugin) Delete(_ context.Context, req *cstructs.ClientHostVolumeDeleteRequest) error {
|
||||
if p.deleteErr != nil {
|
||||
return p.deleteErr
|
||||
}
|
||||
p.deleted = req.ID
|
||||
return nil
|
||||
}
|
||||
|
||||
func assertLocked(t *testing.T, hvm *HostVolumeManager, name string) {
|
||||
t.Helper()
|
||||
must.True(t, hvm.locker.isLocked(name), must.Sprintf("vol name %q should be locked", name))
|
||||
}
|
||||
|
||||
func assertNotLocked(t *testing.T, hvm *HostVolumeManager, name string) {
|
||||
t.Helper()
|
||||
must.False(t, hvm.locker.isLocked(name), must.Sprintf("vol name %q should not be locked", name))
|
||||
}
|
||||
|
||||
func TestHostVolumeManager_restoreFromState(t *testing.T) {
|
||||
log := testlog.HCLogger(t)
|
||||
hostPath := t.TempDir()
|
||||
|
||||
vol1 := &cstructs.HostVolumeState{
|
||||
ID: "test-vol-id-1",
|
||||
CreateReq: &cstructs.ClientHostVolumeCreateRequest{
|
||||
Name: "created-volume",
|
||||
ID: "test-vol-id-1",
|
||||
PluginID: "mkdir",
|
||||
},
|
||||
}
|
||||
vol2 := &cstructs.HostVolumeState{
|
||||
ID: "test-vol-id-2",
|
||||
HostPath: hostPath,
|
||||
CreateReq: &cstructs.ClientHostVolumeCreateRequest{
|
||||
Name: "registered-volume",
|
||||
ID: "test-vol-id-2",
|
||||
PluginID: "", // this signifies a Register operation
|
||||
},
|
||||
}
|
||||
|
||||
node := newFakeNode(t)
|
||||
|
||||
t.Run("no vols", func(t *testing.T) {
|
||||
state := cstate.NewMemDB(log)
|
||||
hvm := NewHostVolumeManager(log, Config{
|
||||
StateMgr: state,
|
||||
// no other fields are necessary when there are zero volumes
|
||||
})
|
||||
vols, err := hvm.restoreFromState(timeout(t))
|
||||
must.NoError(t, err)
|
||||
must.Eq(t, VolumeMap{}, vols)
|
||||
})
|
||||
|
||||
t.Run("happy", func(t *testing.T) {
|
||||
// put our volume in state
|
||||
state := cstate.NewMemDB(log)
|
||||
must.NoError(t, state.PutDynamicHostVolume(vol1))
|
||||
must.NoError(t, state.PutDynamicHostVolume(vol2))
|
||||
|
||||
// new volume manager should load it from state and run Create,
|
||||
// resulting in a volume directory in this mountDir.
|
||||
volsDir := t.TempDir()
|
||||
volPath := filepath.Join(volsDir, vol1.ID)
|
||||
|
||||
hvm := NewHostVolumeManager(log, Config{
|
||||
StateMgr: state,
|
||||
UpdateNodeVols: node.updateVol,
|
||||
PluginDir: "/wherever",
|
||||
VolumesDir: volsDir,
|
||||
})
|
||||
|
||||
vols, err := hvm.restoreFromState(timeout(t))
|
||||
must.NoError(t, err)
|
||||
|
||||
expect := map[string]*structs.ClientHostVolumeConfig{
|
||||
"created-volume": {
|
||||
Name: "created-volume",
|
||||
ID: "test-vol-id-1",
|
||||
Path: volPath,
|
||||
ReadOnly: false,
|
||||
},
|
||||
"registered-volume": {
|
||||
Name: "registered-volume",
|
||||
ID: "test-vol-id-2",
|
||||
Path: hostPath,
|
||||
ReadOnly: false,
|
||||
},
|
||||
}
|
||||
must.Eq(t, expect, vols)
|
||||
|
||||
must.DirExists(t, volPath)
|
||||
|
||||
assertLocked(t, hvm, "created-volume")
|
||||
assertLocked(t, hvm, "registered-volume")
|
||||
})
|
||||
|
||||
t.Run("state error", func(t *testing.T) {
|
||||
state := &cstate.ErrDB{}
|
||||
hvm := NewHostVolumeManager(log, Config{StateMgr: state})
|
||||
vols, err := hvm.restoreFromState(timeout(t))
|
||||
must.ErrorIs(t, err, cstate.ErrDBError)
|
||||
must.Nil(t, vols)
|
||||
})
|
||||
|
||||
t.Run("plugin missing", func(t *testing.T) {
|
||||
state := cstate.NewMemDB(log)
|
||||
vol := &cstructs.HostVolumeState{
|
||||
CreateReq: &cstructs.ClientHostVolumeCreateRequest{
|
||||
PluginID: "nonexistent-plugin",
|
||||
},
|
||||
}
|
||||
must.NoError(t, state.PutDynamicHostVolume(vol))
|
||||
|
||||
hvm := NewHostVolumeManager(log, Config{StateMgr: state})
|
||||
vols, err := hvm.restoreFromState(timeout(t))
|
||||
must.ErrorIs(t, err, ErrPluginNotExists)
|
||||
must.MapEmpty(t, vols)
|
||||
})
|
||||
|
||||
t.Run("plugin error", func(t *testing.T) {
|
||||
state := cstate.NewMemDB(log)
|
||||
vol := &cstructs.HostVolumeState{
|
||||
ID: "test-volume",
|
||||
CreateReq: &cstructs.ClientHostVolumeCreateRequest{
|
||||
PluginID: "test-plugin",
|
||||
},
|
||||
}
|
||||
must.NoError(t, state.PutDynamicHostVolume(vol))
|
||||
|
||||
log, getLogs := logRecorder(t)
|
||||
hvm := NewHostVolumeManager(log, Config{StateMgr: state})
|
||||
plug := &fakePlugin{
|
||||
createErr: errors.New("sad create"),
|
||||
}
|
||||
hvm.builtIns["test-plugin"] = plug
|
||||
|
||||
vols, err := hvm.restoreFromState(timeout(t))
|
||||
// error during restore should not halt the whole client
|
||||
must.NoError(t, err)
|
||||
must.NotNil(t, vols)
|
||||
// but it should log
|
||||
logs := getLogs()
|
||||
must.StrContains(t, logs, "[ERROR]")
|
||||
must.StrContains(t, logs, `failed to restore: plugin_id=test-plugin volume_id=test-volume error="sad create"`)
|
||||
})
|
||||
}
|
||||
|
||||
type fakeNode struct {
|
||||
vols VolumeMap
|
||||
log hclog.Logger
|
||||
}
|
||||
|
||||
func (n *fakeNode) updateVol(name string, volume *structs.ClientHostVolumeConfig) {
|
||||
UpdateVolumeMap(n.log, n.vols, name, volume)
|
||||
}
|
||||
|
||||
func newFakeNode(t *testing.T) *fakeNode {
|
||||
return &fakeNode{
|
||||
vols: make(VolumeMap),
|
||||
log: testlog.HCLogger(t),
|
||||
}
|
||||
}
|
||||
|
||||
// timeout provides a context that times out in 1 second
|
||||
func timeout(t *testing.T) context.Context {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
t.Cleanup(cancel)
|
||||
return ctx
|
||||
}
|
||||
|
||||
// logRecorder is here so we can assert that stdout/stderr appear in logs
|
||||
func logRecorder(t *testing.T) (hclog.Logger, func() string) {
|
||||
t.Helper()
|
||||
buf := &bytes.Buffer{}
|
||||
logger := hclog.New(&hclog.LoggerOptions{
|
||||
Name: "log-recorder",
|
||||
Output: buf,
|
||||
Level: hclog.Debug,
|
||||
IncludeLocation: true,
|
||||
DisableTime: true,
|
||||
})
|
||||
return logger, func() string {
|
||||
bts, err := io.ReadAll(buf)
|
||||
test.NoError(t, err)
|
||||
buf.Reset()
|
||||
return string(bts)
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright IBM Corp. 2015, 2025
|
||||
# SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
# plugin for host_volume_plugin_test.go
|
||||
set -xeuo pipefail
|
||||
|
||||
env 1>&2
|
||||
|
||||
test "$1" == "$DHV_OPERATION"
|
||||
|
||||
echo 'all operations should ignore stderr' 1>&2
|
||||
|
||||
case $1 in
|
||||
fingerprint)
|
||||
echo '{"version": "0.0.2"}' ;;
|
||||
create)
|
||||
test "$DHV_VOLUME_NAME" == 'test-vol-name'
|
||||
test "$DHV_VOLUME_ID" == 'test-vol-id'
|
||||
test "$DHV_NAMESPACE" == 'test-namespace'
|
||||
test "$DHV_CAPACITY_MIN_BYTES" -eq 5
|
||||
test "$DHV_CAPACITY_MAX_BYTES" -eq 10
|
||||
test "$DHV_NODE_ID" == 'test-node'
|
||||
test "$DHV_NODE_POOL" == 'test-node-pool'
|
||||
test "$DHV_PARAMETERS" == '{"key":"val"}'
|
||||
test "$DHV_PLUGIN_DIR" == './test_fixtures'
|
||||
test -d "$DHV_VOLUMES_DIR"
|
||||
target="$DHV_VOLUMES_DIR/$DHV_VOLUME_ID"
|
||||
test "$target" != '/'
|
||||
mkdir -p "$target"
|
||||
printf '{"path": "%s", "bytes": 5}' "$target"
|
||||
;;
|
||||
delete)
|
||||
test "$DHV_NODE_ID" == 'test-node'
|
||||
test "$DHV_NODE_POOL" == 'test-node-pool'
|
||||
test "$DHV_NAMESPACE" == 'test-namespace'
|
||||
test "$DHV_VOLUME_NAME" == 'test-vol-name'
|
||||
test "$DHV_VOLUME_ID" == 'test-vol-id'
|
||||
test "$DHV_PARAMETERS" == '{"key":"val"}'
|
||||
test "$DHV_PLUGIN_DIR" == './test_fixtures'
|
||||
test -d "$DHV_VOLUMES_DIR"
|
||||
target="$DHV_VOLUMES_DIR/$DHV_VOLUME_ID"
|
||||
test "$target" != '/'
|
||||
test "$DHV_CREATED_PATH" == "$target"
|
||||
rm -rfv "$target"
|
||||
;;
|
||||
*)
|
||||
echo "unknown operation $1"
|
||||
exit 1 ;;
|
||||
esac
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright IBM Corp. 2015, 2025
|
||||
# SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
printf '{"error": "%s: sad plugin is sad"}' $1
|
||||
echo "$1: it tells you all about it in stderr" 1>&2
|
||||
exit 1
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
// Copyright IBM Corp. 2015, 2025
|
||||
// SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
package hostvolumemanager
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
)
|
||||
|
||||
// this file is for fingerprinting *volumes*
|
||||
// *plugins* are detected in client/fingerprint/dynamic_host_volumes.go
|
||||
|
||||
// HostVolumeNodeUpdater is used to add or remove volumes from the Node.
|
||||
type HostVolumeNodeUpdater func(name string, volume *structs.ClientHostVolumeConfig)
|
||||
|
||||
// VolumeMap keys are volume `name`s, identical to Node.HostVolumes.
|
||||
type VolumeMap map[string]*structs.ClientHostVolumeConfig
|
||||
|
||||
// UpdateVolumeMap returns true if it changes the provided `volumes` map.
|
||||
// If `vol` is nil, key `name` will be removed from the map, if present.
|
||||
// If it is not nil, `name: vol` will be set on the map, if different.
|
||||
//
|
||||
// Since it may mutate the map, the caller should make a copy
|
||||
// or acquire a lock as appropriate for their context.
|
||||
func UpdateVolumeMap(log hclog.Logger, volumes VolumeMap, name string, vol *structs.ClientHostVolumeConfig) (changed bool) {
|
||||
current, exists := volumes[name]
|
||||
if vol == nil {
|
||||
if exists {
|
||||
delete(volumes, name)
|
||||
changed = true
|
||||
}
|
||||
} else {
|
||||
// if the volume already exists with no ID, it will be because it was
|
||||
// added to client agent config after having been previously created
|
||||
// as a dynamic vol. dynamic takes precedence, but log a warning.
|
||||
if exists && current.ID == "" {
|
||||
log.Warn("overriding static host volume with dynamic", "name", name, "id", vol.ID)
|
||||
}
|
||||
if !exists || !vol.Equal(current) {
|
||||
volumes[name] = vol
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// WaitForFirstFingerprint implements client.FingerprintingPluginManager
|
||||
// so any existing volumes are added to the client node on agent start.
|
||||
func (hvm *HostVolumeManager) WaitForFirstFingerprint(ctx context.Context) <-chan struct{} {
|
||||
// the fingerprint manager puts batchFirstFingerprintsTimeout (50 seconds)
|
||||
// on the context that it sends to us here so we don't need another
|
||||
// timeout. we just need to cancel to report when we are done.
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
volumes, err := hvm.restoreFromState(ctx)
|
||||
if err != nil {
|
||||
hvm.log.Error("failed to restore state", "error", err)
|
||||
return ctx.Done()
|
||||
}
|
||||
for name, vol := range volumes {
|
||||
hvm.updateNodeVols(name, vol) // => batchNodeUpdates.updateNodeFromHostVolume()
|
||||
}
|
||||
return ctx.Done()
|
||||
}
|
||||
func (hvm *HostVolumeManager) Run() {}
|
||||
func (hvm *HostVolumeManager) Shutdown() {}
|
||||
func (hvm *HostVolumeManager) PluginType() string {
|
||||
// "Plugin"Type is misleading, because this is for *volumes* but ok.
|
||||
return "dynamic_host_volume"
|
||||
}
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
// Copyright IBM Corp. 2015, 2025
|
||||
// SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
package hostvolumemanager
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/hashicorp/nomad/client/state"
|
||||
cstructs "github.com/hashicorp/nomad/client/structs"
|
||||
"github.com/hashicorp/nomad/helper/testlog"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
"github.com/shoenig/test/must"
|
||||
)
|
||||
|
||||
func TestUpdateVolumeMap(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
|
||||
vols VolumeMap
|
||||
volName string
|
||||
vol *structs.ClientHostVolumeConfig
|
||||
|
||||
expectMap VolumeMap
|
||||
expectChange bool
|
||||
|
||||
expectLog string
|
||||
}{
|
||||
{
|
||||
name: "delete absent",
|
||||
vols: VolumeMap{},
|
||||
volName: "anything",
|
||||
vol: nil,
|
||||
expectMap: VolumeMap{},
|
||||
expectChange: false,
|
||||
},
|
||||
{
|
||||
name: "delete present",
|
||||
vols: VolumeMap{"deleteme": {}},
|
||||
volName: "deleteme",
|
||||
vol: nil,
|
||||
expectMap: VolumeMap{},
|
||||
expectChange: true,
|
||||
},
|
||||
{
|
||||
name: "add absent",
|
||||
vols: VolumeMap{},
|
||||
volName: "addme",
|
||||
vol: &structs.ClientHostVolumeConfig{},
|
||||
expectMap: VolumeMap{"addme": {}},
|
||||
expectChange: true,
|
||||
},
|
||||
{
|
||||
name: "add present",
|
||||
vols: VolumeMap{"ignoreme": {}},
|
||||
volName: "ignoreme",
|
||||
vol: &structs.ClientHostVolumeConfig{},
|
||||
expectMap: VolumeMap{"ignoreme": {}},
|
||||
expectChange: false,
|
||||
},
|
||||
{
|
||||
// this should not happen with dynamic vols, but test anyway
|
||||
name: "change present",
|
||||
vols: VolumeMap{"changeme": {ID: "before"}},
|
||||
volName: "changeme",
|
||||
vol: &structs.ClientHostVolumeConfig{ID: "after"},
|
||||
expectMap: VolumeMap{"changeme": {ID: "after"}},
|
||||
expectChange: true,
|
||||
},
|
||||
{
|
||||
// this should only happen during agent start, if a static vol has
|
||||
// been added to config after a previous dynamic vol was created
|
||||
// with the same name.
|
||||
name: "override static",
|
||||
vols: VolumeMap{"overrideme": {ID: ""}}, // static vols have no ID
|
||||
volName: "overrideme",
|
||||
vol: &structs.ClientHostVolumeConfig{ID: "dynamic-vol-id"},
|
||||
expectMap: VolumeMap{"overrideme": {ID: "dynamic-vol-id"}},
|
||||
expectChange: true,
|
||||
expectLog: "overriding static host volume with dynamic: name=overrideme id=dynamic-vol-id",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
log, getLogs := logRecorder(t)
|
||||
|
||||
changed := UpdateVolumeMap(log, tc.vols, tc.volName, tc.vol)
|
||||
must.Eq(t, tc.expectMap, tc.vols)
|
||||
|
||||
if tc.expectChange {
|
||||
must.True(t, changed, must.Sprint("expect volume to have been changed"))
|
||||
} else {
|
||||
must.False(t, changed, must.Sprint("expect volume not to have been changed"))
|
||||
}
|
||||
|
||||
must.StrContains(t, getLogs(), tc.expectLog)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitForFirstFingerprint(t *testing.T) {
|
||||
log := testlog.HCLogger(t)
|
||||
tmp := t.TempDir()
|
||||
memDB := state.NewMemDB(log)
|
||||
node := newFakeNode(t)
|
||||
hvm := NewHostVolumeManager(log, Config{
|
||||
PluginDir: "",
|
||||
VolumesDir: tmp,
|
||||
StateMgr: memDB,
|
||||
UpdateNodeVols: node.updateVol,
|
||||
})
|
||||
plug := &fakePlugin{volsDir: tmp}
|
||||
hvm.builtIns = map[string]HostVolumePlugin{
|
||||
"test-plugin": plug,
|
||||
}
|
||||
must.NoError(t, memDB.PutDynamicHostVolume(&cstructs.HostVolumeState{
|
||||
ID: "vol-id",
|
||||
CreateReq: &cstructs.ClientHostVolumeCreateRequest{
|
||||
ID: "vol-id",
|
||||
Name: "vol-name",
|
||||
PluginID: "test-plugin",
|
||||
},
|
||||
}))
|
||||
|
||||
ctx := timeout(t)
|
||||
done := hvm.WaitForFirstFingerprint(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal("fingerprint timed out")
|
||||
case <-done:
|
||||
}
|
||||
|
||||
must.Eq(t, "vol-id", plug.created)
|
||||
must.Eq(t, VolumeMap{
|
||||
"vol-name": &structs.ClientHostVolumeConfig{
|
||||
Name: "vol-name",
|
||||
ID: "vol-id",
|
||||
Path: filepath.Join(tmp, "vol-id"),
|
||||
ReadOnly: false,
|
||||
},
|
||||
}, node.vols)
|
||||
}
|
||||
Reference in new issue
Block a user