From d4f04697f1aede24046fbb0ef232fd6bb839b4c5 Mon Sep 17 00:00:00 2001 From: Mishaoyue <934922813@qq.com> Date: Wed, 11 Nov 2026 03:04:07 +0000 Subject: [PATCH] fix(expander): do not copy a template node's GPU usage to the node it simulates To find a node type that fits a pending Pod, the expander copies an existing node and its GPUs and checks whether the Pod fits the copy. The GPU copies kept the template's usedBy and phase. When the template's GPUs were held by native Pods (usedBy=nvidia-device-plugin under progressive migration) or were not Running, PhaseFilter rejected every copy, and the expander reported "failed to satisfy the pending pod, no potential GPU nodes can fit" without creating a NodeClaim. The copy stands for a new, idle node: set usedBy to tensor-fusion, phase to Running and clear runningApps, as Available is already reset. --- internal/scheduler/expander/handler.go | 8 +++ .../scheduler/expander/prepare_node_test.go | 72 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 internal/scheduler/expander/prepare_node_test.go diff --git a/internal/scheduler/expander/handler.go b/internal/scheduler/expander/handler.go index 0be2e3b7..793f5bd1 100644 --- a/internal/scheduler/expander/handler.go +++ b/internal/scheduler/expander/handler.go @@ -805,7 +805,15 @@ func (e *NodeExpander) prepareNewNodesForScheduleAttempt( gpuCopy.Status.NodeSelector = make(map[string]string, 1) } gpuCopy.Status.NodeSelector[constants.KubernetesHostNameLabel] = newPreparedNode.Name + // The copy stands for a GPU of a node that does not exist yet: it is + // idle, healthy and not taken by a native device plugin, whatever the + // template GPU is. Copying the template's usedBy or phase made + // PhaseFilter reject every copy of a node whose GPUs native Pods hold, + // so expansion found no node that could fit. gpuCopy.Status.Available = gpuCopy.Status.Capacity.DeepCopy() + gpuCopy.Status.UsedBy = tfv1.UsedByTensorFusion + gpuCopy.Status.Phase = tfv1.TensorFusionGPUPhaseRunning + gpuCopy.Status.RunningApps = nil newPreparedGPUs = append(newPreparedGPUs, gpuCopy) } return newPreparedNode, newPreparedGPUs diff --git a/internal/scheduler/expander/prepare_node_test.go b/internal/scheduler/expander/prepare_node_test.go new file mode 100644 index 00000000..9577338e --- /dev/null +++ b/internal/scheduler/expander/prepare_node_test.go @@ -0,0 +1,72 @@ +package expander + +import ( + "context" + "testing" + + tfv1 "github.com/NexusGPU/tensor-fusion/api/v1" + "github.com/NexusGPU/tensor-fusion/internal/gpuallocator/filter" + "github.com/NexusGPU/tensor-fusion/internal/utils" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// A node whose GPUs native Pods hold, or that are not Running, is still a +// valid template: the node expansion would create is new and idle. The +// copies must not inherit the template GPUs' usedBy or phase, or PhaseFilter +// rejects every copy and expansion fails with "no potential GPU nodes can fit". +func TestPrepareNewNodesForScheduleAttemptResetsTemplateGPUState(t *testing.T) { + utils.SetProgressiveMigration(true) + defer utils.SetProgressiveMigration(false) + + capacity := &tfv1.Resource{Tflops: resource.MustParse("125"), Vram: resource.MustParse("24Gi")} + template := map[string]*tfv1.GPU{ + "native": { + ObjectMeta: metav1.ObjectMeta{Name: "gpu-native"}, + Status: tfv1.GPUStatus{ + Phase: tfv1.TensorFusionGPUPhaseRunning, + UsedBy: tfv1.UsedByNvidiaDevicePlugin, + Capacity: capacity, + Available: &tfv1.Resource{}, + RunningApps: []*tfv1.RunningAppDetail{{Name: "app", Namespace: "ns"}}, + }, + }, + "pending": { + ObjectMeta: metav1.ObjectMeta{Name: "gpu-pending"}, + Status: tfv1.GPUStatus{ + Phase: tfv1.TensorFusionGPUPhasePending, + UsedBy: tfv1.UsedByTensorFusion, + Capacity: capacity, + Available: &tfv1.Resource{}, + }, + }, + } + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "template", Labels: map[string]string{}}} + + newNode, gpus := (&NodeExpander{}).prepareNewNodesForScheduleAttempt(node, template) + + if len(gpus) != 2 { + t.Fatalf("got %d GPUs, want 2", len(gpus)) + } + for _, gpu := range gpus { + if gpu.Status.UsedBy != tfv1.UsedByTensorFusion || gpu.Status.Phase != tfv1.TensorFusionGPUPhaseRunning { + t.Errorf("GPU %s: usedBy=%q phase=%q, want tensor-fusion/Running", gpu.Name, gpu.Status.UsedBy, gpu.Status.Phase) + } + if gpu.Status.Available.Vram.Cmp(capacity.Vram) != 0 || len(gpu.Status.RunningApps) != 0 { + t.Errorf("GPU %s is not idle: available=%v runningApps=%d", gpu.Name, gpu.Status.Available, len(gpu.Status.RunningApps)) + } + } + passed, err := filter.NewPhaseFilter(tfv1.TensorFusionGPUPhaseRunning).Filter(context.Background(), tfv1.NameNamespace{}, gpus) + if err != nil || len(passed) != 2 { + t.Fatalf("PhaseFilter kept %d of 2 copies (err=%v)", len(passed), err) + } + + // The template GPUs are left untouched. + if template["native"].Status.UsedBy != tfv1.UsedByNvidiaDevicePlugin || template["pending"].Status.Phase != tfv1.TensorFusionGPUPhasePending { + t.Error("template GPUs were modified") + } + if newNode.Name == node.Name { + t.Error("the new node reuses the template node's name") + } +}