DAOS-19450 control: enforce per-pool node cert at connect

When a pool has a CA installed, the engine expects the client to
present a node cert and proof-of-possession in the pool connect
RPC. If the client does not present a valid cert and/or PoP, the
connection is rejected. Pools without a CA accept clients as usual.

Patch: 5/8
Features: pool
Signed-off-by: Michael MacDonald <github@macdonald.cx>

Author: mjmac

src/control/lib/daos/drpc.go

@@ -1,6 +1,6 @@
 //
 // (C) Copyright 2019-2024 Intel Corporation.
-// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -290,6 +290,7 @@ func (m securityMethod) ID() int32 {
 func (m securityMethod) String() string {
 	if s, ok := map[securityMethod]string{
 		MethodValidateCredentials: "validate credentials",
+		MethodValidateNodeCert:    "validate node cert",
 	}[m]; ok {
 		return s
 	}
@@ -300,4 +301,6 @@ func (m securityMethod) String() string {
 const (
 	// MethodValidateCredentials is a ModuleSecurity method
 	MethodValidateCredentials securityMethod = C.DRPC_METHOD_SEC_VALIDATE_CREDS
+	// MethodValidateNodeCert is a ModuleSecurity method for per-pool node cert validation
+	MethodValidateNodeCert securityMethod = C.DRPC_METHOD_SEC_VALIDATE_NODE_CERT
 )

src/control/lib/daos/status.go

@@ -172,6 +172,8 @@ const (
 	ControlIncompatible Status = -C.DER_CONTROL_INCOMPAT
 	// NoCert indicates that one or more configured certificates could not be accessed.
 	NoCert Status = -C.DER_NO_CERT
+	// NoNodeCert indicates that a pool requires a node certificate but none is configured.
+	NoNodeCert Status = -C.DER_NO_NODE_CERT
 	// BadCert indicates that an invalid certificate was detected.
 	BadCert Status = -C.DER_BAD_CERT
 	// RedundancyFactorExceeded indicates that the maximum number of failed components was exceeded.

src/control/server/config/server.go

@@ -782,6 +782,12 @@ func (cfg *Server) Validate(log logging.Logger) (err error) {
 	}
 	log.Debug(msg)
 
+	if cfg.TransportConfig != nil {
+		if err = cfg.TransportConfig.Validate(); err != nil {
+			return errors.Wrap(err, "transport config validation failed")
+		}
+	}
+
 	// Append the user-friendly message to any error.
 	defer func() {
 		if err != nil && !fault.HasResolution(err) {

src/control/server/config/server_test.go

@@ -17,6 +17,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/google/go-cmp/cmp"
@@ -2391,3 +2392,48 @@ func TestConfig_HasPMem(t *testing.T) {
 		})
 	}
 }
+
+func TestServerConfig_TransportMaxClockSkew(t *testing.T) {
+	testDir, cleanup := test.CreateTestDir(t)
+	defer cleanup()
+	testFile := filepath.Join(testDir, sConfigUncomment)
+	uncommentServerConfig(t, testFile)
+
+	for name, tc := range map[string]struct {
+		skew   time.Duration
+		expErr error
+	}{
+		"zero":                 {0, nil},
+		"positive":             {time.Minute, nil},
+		"one second boundary":  {time.Second, nil},
+		"negative":             {-time.Second, errors.New("must not be negative")},
+		"sub-second (bare ns)": {300 * time.Nanosecond, errors.New("below 1s")},
+	} {
+		t.Run(name, func(t *testing.T) {
+			log, buf := logging.NewTestLogger(t.Name())
+			defer test.ShowBufferOnFailure(t, buf)
+
+			cfg := baseCfg(t, log, testFile)
+			cfg.TransportConfig.CertMaxClockSkew = tc.skew
+			test.CmpErr(t, tc.expErr, cfg.Validate(log))
+		})
+	}
+}
+
+// A transport_config block that omits cert_max_clock_skew keeps the default.
+func TestServerConfig_TransportMaxClockSkewDefault(t *testing.T) {
+	log, buf := logging.NewTestLogger(t.Name())
+	defer test.ShowBufferOnFailure(t, buf)
+	testDir, cleanup := test.CreateTestDir(t)
+	defer cleanup()
+
+	path := filepath.Join(testDir, "daos_server.yml")
+	if err := os.WriteFile(path, []byte("transport_config:\n  allow_insecure: true\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	cfg, err := mockConfigFromFile(t, log, path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	test.AssertEqual(t, security.DefaultCertMaxClockSkew, cfg.TransportConfig.CertMaxClockSkew, "cert_max_clock_skew")
+}

src/control/server/security_rpc.go

@@ -1,6 +1,6 @@
 //
 // (C) Copyright 2019-2022 Intel Corporation.
-// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -12,7 +12,10 @@ import (
 	"crypto"
 	"fmt"
 	"path/filepath"
+	"time"
 
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/daos-stack/daos/src/control/drpc"
@@ -84,11 +87,14 @@ func (m *SecurityModule) validateRespWithStatus(status daos.Status) ([]byte, err
 
 // HandleCall is the handler for calls to the SecurityModule
 func (m *SecurityModule) HandleCall(_ context.Context, session *drpc.Session, method drpc.Method, body []byte) ([]byte, error) {
-	if method != daos.MethodValidateCredentials {
+	switch method {
+	case daos.MethodValidateCredentials:
+		return m.processValidateCredentials(body)
+	case daos.MethodValidateNodeCert:
+		return m.processValidateNodeCert(body)
+	default:
 		return nil, drpc.UnknownMethodFailure()
 	}
-
-	return m.processValidateCredentials(body)
 }
 
 // ID will return Security module ID
@@ -105,7 +111,79 @@ func (m *SecurityModule) GetMethod(id int32) (drpc.Method, error) {
 	switch id {
 	case daos.MethodValidateCredentials.ID():
 		return daos.MethodValidateCredentials, nil
+	case daos.MethodValidateNodeCert.ID():
+		return daos.MethodValidateNodeCert, nil
 	default:
 		return nil, fmt.Errorf("invalid method ID %d for module %s", id, m.String())
 	}
 }
+
+func (m *SecurityModule) processValidateNodeCert(body []byte) ([]byte, error) {
+	req := &auth.ValidateNodeCertReq{}
+	if err := proto.Unmarshal(body, req); err != nil {
+		return nil, drpc.UnmarshalingPayloadFailure()
+	}
+
+	poolUUID, err := uuid.FromBytes(req.GetPoolUuid())
+	if err != nil {
+		return m.rejectNodeCert(req, daos.InvalidInput,
+			fmt.Sprintf("invalid pool UUID (%d bytes)", len(req.GetPoolUuid())))
+	}
+
+	daosCA, err := m.config.CACert()
+	if err != nil {
+		return m.rejectNodeCert(req, daos.NoCert,
+			fmt.Sprintf("failed to load DAOS CA: %v", err))
+	}
+	if daosCA == nil {
+		return m.rejectNodeCert(req, daos.NoCert, "no DAOS CA without transport security")
+	}
+
+	p := &security.NodeCertPresentation{
+		Root:        daosCA,
+		PoolCA:      req.PoolCa,
+		Cert:        req.NodeCert,
+		PoPSig:      req.PopSig,
+		PoPPayload:  req.PopPayload,
+		PoolUUID:    poolUUID,
+		MachineName: req.MachineName,
+		Watermarks:  req.CertWatermarks,
+		MaxSkew:     m.config.CertMaxClockSkew,
+		Now:         time.Now(),
+	}
+	payload, err := p.Validate()
+	if err != nil {
+		return m.rejectNodeCert(req, validationStatus(err), err.Error())
+	}
+
+	m.log.Debugf("node cert validated: pool=%s, handle=%s", poolUUID, payload.HandleID())
+
+	return drpc.Marshal(&auth.ValidateNodeCertResp{Status: 0})
+}
+
+// validationStatus maps the security package's sentinel errors to wire statuses.
+func validationStatus(err error) daos.Status {
+	switch {
+	case errors.Is(err, security.ErrInvalidInput):
+		return daos.InvalidInput
+	case errors.Is(err, security.ErrCertInvalid), errors.Is(err, security.ErrCertNotYetValid),
+		errors.Is(err, security.ErrCertRevoked):
+		return daos.BadCert
+	case errors.Is(err, security.ErrPoPInvalid), errors.Is(err, security.ErrPoPStale):
+		return daos.NoPermission
+	case errors.Is(err, security.ErrBadWatermarks):
+		return daos.IOError
+	default:
+		return daos.MiscError
+	}
+}
+
+// rejectNodeCert logs and returns a ValidateNodeCertResp carrying status + detail.
+func (m *SecurityModule) rejectNodeCert(req *auth.ValidateNodeCertReq, status daos.Status, detail string) ([]byte, error) {
+	m.log.Errorf("node cert rejected (pool=%x, status=%s): %s",
+		req.GetPoolUuid(), status, detail)
+	return drpc.Marshal(&auth.ValidateNodeCertResp{
+		Status: int32(status),
+		Detail: detail,
+	})
+}