kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/release-1.9/installer/volcano-development.yaml

volcanosh/vc-controller-manager:v1.9.0
volcanosh/vc-scheduler:v1.9.0
volcanosh/vc-webhook-manager:v1.9.0

type PluginOption struct {
    //插件名称
    Name string `yaml:"name"`
    //是否启用jobOrderFn
    EnabledJobOrder *bool `yaml:"enableJobOrder"`
    //是否启用分层
    EnabledHierarchy *bool `yaml:"enableHierarchy"`
    //是否启用jobReadyFn
    EnabledJobReady *bool `yaml:"enableJobReady"`
    //是否启用jobPipelinedFn
    EnabledJobPipelined *bool `yaml:"enableJobPipelined"`
    //是否启用taskOrderFn
    EnabledTaskOrder *bool `yaml:"enableTaskOrder"`
    //是否启用preemptableFn
    EnabledPreemptable *bool `yaml:"enablePreemptable"`
    //是否启用reclaimableFn
    EnabledReclaimable *bool `yaml:"enableReclaimable"`
    //是否启用preemptiveFn
    EnablePreemptive *bool `yaml:"enablePreemptive"`
    //是否启用queueOrderFn
    EnabledQueueOrder *bool `yaml:"enableQueueOrder"`
    //是否启用clusterOrderFn
    EnabledClusterOrder *bool `yaml:"EnabledClusterOrder"`
    //是否启用predicateFn
    EnabledPredicate *bool `yaml:"enablePredicate"`
    //是否启用bestNodeFn
    EnabledBestNode *bool `yaml:"enableBestNode"`
    //是否启用nodeOrderFn
    EnabledNodeOrder *bool `yaml:"enableNodeOrder"`
    //是否启用targetJobFn
    EnabledTargetJob *bool `yaml:"enableTargetJob"`
    //是否启用reservedNodesFn
    EnabledReservedNodes *bool `yaml:"enableReservedNodes"`
    //是否启用jobEnqueuedFn
    EnabledJobEnqueued *bool `yaml:"enableJobEnqueued"`
    //是否启用victimsFn
    EnabledVictim *bool `yaml:"enabledVictim"`
    //是否启用jobStarvingFn
    EnabledJobStarving *bool `yaml:"enableJobStarving"`
    //是否启用overusedFn
    EnabledOverused *bool `yaml:"enabledOverused"`
    //是否启用allocatable
    EnabledAllocatable *bool `yaml:"enabledAllocatable"`
    //插件执行需要的参数
    Arguments map[string]interface{} `yaml:"arguments"`
}

type Tier struct {
    Plugins []PluginOption `yaml:"plugins"`
}

type Scheduler struct {
    cache schedcache.Cache //集群缓存信息
    schedulerConf string //配置文件路径
    fileWatcher filewatcher.FileWatcher //配置文件监听器
    schedulePeriod time.Duration //调度循环间隔，默认1s
    mutex sync.Mutex
    actions []framework.Action //Action列表
    plugins []conf.Tier //Plugin列表
    ...
}

type Session struct {
    UID types.UID //会话ID
    cache cache.Cache //集群缓存信息，包括节点、队列、任务等
    TotalResource *api.Resource //当前所有节点资源总量
    Jobs map[api.JobID]*api.JobInfo //当前所有任务信息
    Nodes map[string]*api.NodeInfo //当前所有节点信息
    Queues map[api.QueueID]*api.QueueInfo //当前所有队列信息
    NamespaceInfo map[api.NamespaceName]*api.NamespaceInfo //当前所有namespace信息
    plugins map[string]Plugin //当前所有插件信息
    jobOrderFns map[string]api.CompareFn //job对比函数，用于优先级排序
    queueOrderFns map[string]api.CompareFn //queue对比函数，用于优先级排序
    taskOrderFns map[string]api.CompareFn //task对比函数，用于优先级排序
    ...
}

//队列CRD类型
type Queue struct {
    metav1.TypeMeta
    metav1.ObjectMeta
    Spec QueueSpec
    Status QueueStatus
}
type QueueSpec struct {
    Weight     int32 //队列权重
    Capability v1.ResourceList //队列资源容量
    Reclaimable *bool //是否可被其他队列抢占资源
    ExtendClusters []Cluster //队列中job分发的集群列表
    Guarantee Guarantee //队列资源预留量
    Affinity *Affinity //设置队列的NodeGroup亲和性、反亲和性规则
    Type string //队列类型
    Parent string //父队列
    Deserved v1.ResourceList //队列应分配资源量，能被其他队列共享
}
type QueueStatus struct {
    State QueueState //队列状态
    Unknown int32 //Unknown状态PodGroup数量
    Pending int32 //Pending状态PodGroup数量
    Running int32 //Running状态PodGroup数量
    Inqueue int32 //Inqueue状态PodGroup数量
    Completed int32 //Completed状态PodGroup数量
    Reservation Reservation //队列预留资源量
    Allocated v1.ResourceList //队列已分配资源量
}
//队列Go数据结构
type QueueInfo struct {
    UID QueueID //队列ID
    Name string //队列名
    Weight int32 //队列权重，用于计算队列资源分配
    Weights string //分层队列权重，使用/分隔，在drf插件中使用
    Hierarchy string //分层队列名，使用/分隔，在drf插件中使用
    Queue *scheduling.Queue //队列 CRD类型
}

type NodeInfo struct {
    Name string //节点名称
    Node *v1.Node //对应k8s Node
    State NodeState //节点状态
    Releasing *Resource //节点上即将释放的资源
    Idle *Resource //节点空闲资源
    Used *Resource //节点已使用的资源，包括running、terminating pod
    Allocatable *Resource //可分配资源
    Capacity *Resource //资源总量
    Tasks map[TaskID]*TaskInfo //节点上任务列表
    ...
}

//调度上下文
type TransactionContext struct {
    NodeName string //节点名
    Status   TaskStatus //task状态
}
type TaskInfo struct {
    UID TaskID //task ID，对应pod uid
    Job JobID //job ID
    Name string //名称
    Namespace string //命名空间
    Resreq *Resource //task运行时使用资源量
    InitResreq *Resource //task启动时使用资源量
    TransactionContext //调度上下文
    LastTransaction *TransactionContext //上一轮调度上下文
    Priority int32 //task优先级
    Preemptable bool //task是否可被抢占
    BestEffort bool //未指定资源申请量，如未设置requests
    Pod *v1.Pod //对应k8s pod
    ...
}

type tasksMap map[TaskID]*TaskInfo

type JobInfo struct {
    UID JobID //任务ID，格式：PodGroup.Namespace/PodGroup.Name
    PgUID types.UID //对应PodGroup UID
    Name string //名称，等于PodGroup.Name
    Namespace string //命名空间，等于PodGroup.Namespace
    Queue QueueID //队列名称
    Priority int32 //优先级
    MinAvailable int32 //执行job所需最小task数
    WaitingTime *time.Duration //job入队最长等待时间，从创建时刻开始
    TaskStatusIndex map[TaskStatus]tasksMap //task状态索引，tasksMap中key为task uid
    Tasks tasksMap //task列表，tasksMap中key为task uid
    TaskMinAvailable map[TaskID]int32 //对应每个task的minAvailable，key为pod annotations中volcano.sh/task-spec的值
    TaskMinAvailableTotal int32 //所有task minAvailable总和
    Allocated *Resource //已分配资源量
    TotalRequest *Resource //总请求资源量
    PodGroup *PodGroup //对应PodGroup CRD
    Preemptable bool //job是否可被抢占
    ...
}

type PodGroupSpec struct {
    MinMember int32 //执行PodGroup所需的最小task数量，作为总体判断，如果资源不满足，PodGroup不会被调度
    MinTaskMember map[string]int32 //执行每个task所需的最小pod数量，如master 1个、worker 3个，如果资源不满足，PodGroup不会被调度
    Queue string //定义PodGroup所在queue，用于分配资源，如果queue不存在，PodGroup不会被调度
    PriorityClassName string //指定PodGroup的优先级
    MinResources *v1.ResourceList //执行PodGroup所需的最小资源，如果不满足，PodGroup不会被调度
}
type PodGroupStatus struct {
    Phase PodGroupPhase //当前PodGroup的状态
    Conditions []PodGroupCondition //PodGroup的condition
    Running int32 //运行中的pod数
    Succeeded int32 //成功的pod数
    Failed int32 //失败的pod数
}
//PodGroup CRD类型
type PodGroup struct {
    metav1.TypeMeta
    metav1.ObjectMeta
    Spec PodGroupSpec
    Status PodGroupStatus
}

type Resource struct {
    MilliCPU float64 //CPU资源
    Memory float64 //内存资源
    ScalarResources map[v1.ResourceName]float64 //其他资源，如GPU
    ...
}

type Action interface {
    Name() string //action名称
    Initialize() //初始化action
    Execute(ssn *Session) //执行action
    UnInitialize() //action后处理
}

type Plugin interface {
    Name() string //名称
    OnSessionOpen(ssn *Session) //启动会话执行
    OnSessionClose(ssn *Session) //关闭会话执行
}

type SchedulerCache struct {
    sync.Mutex
    kubeClient   kubernetes.Interface
    //资源监听对象
    podInformer                infov1.PodInformer
    nodeInformer               infov1.NodeInformer
    podGroupInformerV1beta1    vcinformerv1.PodGroupInformer
    queueInformerV1beta1       vcinformerv1.QueueInformer
    pvInformer                 infov1.PersistentVolumeInformer
    pvcInformer                infov1.PersistentVolumeClaimInformer
    scInformer                 storagev1.StorageClassInformer
    pcInformer                 schedv1.PriorityClassInformer
    quotaInformer              infov1.ResourceQuotaInformer
    csiNodeInformer            storagev1.CSINodeInformer
    csiDriverInformer          storagev1.CSIDriverInformer
    csiStorageCapacityInformer storagev1beta1.CSIStorageCapacityInformer
    cpuInformer                cpuinformerv1.NumatopologyInformer
    //资源缓存
    Jobs                 map[schedulingapi.JobID]*schedulingapi.JobInfo
    Nodes                map[string]*schedulingapi.NodeInfo
    Queues               map[schedulingapi.QueueID]*schedulingapi.QueueInfo
    PriorityClasses      map[string]*schedulingv1.PriorityClass
    NodeList             []string
    defaultPriorityClass *schedulingv1.PriorityClass
    defaultPriority      int32
    CSINodesStatus       map[string]*schedulingapi.CSINodeStatusInfo
    NamespaceCollection map[string]*schedulingapi.NamespaceCollection
    ...
}

type gangPlugin struct {
    pluginArguments framework.Arguments //插件参数
}

tiers:
- plugins:
  - name: gang
    enablePreemptable: false

type priorityWeight struct {
    BinPackingWeight    int //binpack插件影响node分值的权重
    BinPackingCPU       int //CPU权重
    BinPackingMemory    int //内存权重
    BinPackingResources map[v1.ResourceName]int //其他自定义资源权重
}
type binpackPlugin struct {
    weight priorityWeight //插件权重参数
}

tiers:
- plugins:
  - name: binpack
    arguments:
      binpack.weight: 10
      binpack.cpu: 5
      binpack.memory: 1
      binpack.resources: nvidia.com/gpu, example.com/foo
      binpack.resources.nvidia.com/gpu: 2
      binpack.resources.example.com/foo: 3

type priorityPlugin struct {
    pluginArguments framework.Arguments
}

tiers:
- plugins:
  - name: priority

//每个job的资源属性
type drfAttr struct {
    share            float64 //遍历所有资源，计算job.allocated/ssn.totalResource，获取比值最大者为share，即每个job的主导资源，用于job优先级排序，share值越小，表示job资源占用越少，优先级越高
    dominantResource string //job的主导资源名
    allocated        *api.Resource //job中所有已分配状态task的Resreq总量，已分配状态包括：Bound、Binding、Running、Allocated
}
type drfPlugin struct {
    totalResource  *api.Resource //集群资源总量
    totalAllocated *api.Resource //资源总分配量
    jobAttrs map[api.JobID]*drfAttr //所有任务资源属性
    hierarchicalRoot *hierarchicalNode //分层结构根节点
    pluginArguments framework.Arguments
}

tiers:
- plugins:
  - name: drf

//每个队列的资源属性
type queueAttr struct {
    queueID api.QueueID //queue uid
    name    string //queue名称
    weight  int32 //queue权重
    share   float64 //遍历queue中所有资源，计算allocated/deserved，获取比值最大者为share，用于queue优先级排序，值越小，表示资源占用越少，优先级越高
    deserved  *api.Resource //应分配资源量
    allocated *api.Resource //已分配资源量
    request   *api.Resource //请求资源量
    elastic *api.Resource //弹性资源量，每个job的弹性资源为job.allocated-job.minAvailable
    inqueue    *api.Resource //inqueue job的请求资源量
    capability *api.Resource //queue设置的资源容量
    realCapability *api.Resource //实际资源容量，集群总容量减去其他队列预留资源总量，并且不超过capability
    guarantee      *api.Resource //预留资源量
}
type proportionPlugin struct {
    totalResource  *api.Resource //集群资源总量
    totalGuarantee *api.Resource //资源预留总量
    queueOpts      map[api.QueueID]*queueAttr //各队列资源使用情况
    pluginArguments framework.Arguments
}

tiers:
- plugins:
  - name: proportion

6.6 predicates

6.6.1 数据结构

type baseResource struct {
    CPU    float64
    Memory float64
}
type predicateEnable struct {
    nodeAffinityEnable      bool
    nodePortEnable          bool
    taintTolerationEnable   bool
    podAffinityEnable       bool
    nodeVolumeLimitsEnable  bool
    volumeZoneEnable        bool
    podTopologySpreadEnable bool
    cacheEnable             bool
    //是否启用资源比率判断，启用的资源由predicate.resources设置，值为逗号分隔的资源名列表，如：nvidia.com/gpu
    proportionalEnable      bool
    //启用资源比率判断时，各资源对应的CPU、Memory值，如：ResourceName为nvidia.com/gpu，baseResource.CPU为4，baseResource.Memory为8时，表示每个GPU实例，至少需要分配4个CPU和至少8G内存。但实际源码中函数checkNodeResourceIsProportional计算cpuReserved、memoryReserved时需要分别乘1000才表示核数、GB，当前源码分别表示1/1000核数、MB
    proportional            map[v1.ResourceName]baseResource
}
type predicatesPlugin struct {
    pluginArguments framework.Arguments
}

6.6.2 插件初始化

6.6.3 回调函数

6.6.4 插件配置

tiers:
- plugins:
  - name: predicates
    arguments:
      predicate.NodeAffinityEnable: true
      predicate.NodePortsEnable: true
      predicate.TaintTolerationEnable: true
      predicate.PodAffinityEnable: true
      predicate.NodeVolumeLimitsEnable: true
      predicate.VolumeZoneEnable: true
      predicate.PodTopologySpreadEnable: true
      predicate.GPUSharingEnable: true
      predicate.GPUNumberEnable: true
      predicate.CacheEnable: true
      predicate.ProportionalEnable: true
      predicate.resources: nvidia.com/gpu
      predicate.resources.nvidia.com/gpu.cpu: 4
      predicate.resources.nvidia.com/gpu.memory: 8

6.7 nodeorder

6.7.1 数据结构

type priorityWeight struct {
    leastReqWeight          int
    mostReqWeight           int
    nodeAffinityWeight      int
    podAffinityWeight       int
    balancedResourceWeight  int
    taintTolerationWeight   int
    imageLocalityWeight     int
    podTopologySpreadWeight int
}
type nodeOrderPlugin struct {
    pluginArguments framework.Arguments
}

6.7.2 插件初始化

6.7.3 回调函数

6.7.4 插件配置

tiers:
- plugins:
  - name: nodeorder
    arguments:
      leastrequested.weight: 1
      mostrequested.weight: 0
      nodeaffinity.weight: 2
      podaffinity.weight: 2
      balancedresource.weight: 1
      tainttoleration.weight: 3
      imagelocality.weight: 1
      podtopologyspread.weight: 2

6.8 nodegroup

6.8.1 数据结构

type queueGroupAffinity struct {
    queueGroupAntiAffinityRequired  map[string]sets.Set[string]
    queueGroupAntiAffinityPreferred map[string]sets.Set[string]
    queueGroupAffinityRequired      map[string]sets.Set[string]
    queueGroupAffinityPreferred     map[string]sets.Set[string]
}
type nodeGroupPlugin struct {
    pluginArguments framework.Arguments
}

6.8.2 插件初始化

6.8.3 回调函数

6.8.4 插件配置

tiers:
- plugins:
  - name: nodegroup

6.9 capacity

6.10 conformance

6.10.1 数据结构

type conformancePlugin struct {
    pluginArguments framework.Arguments
}

6.10.2 插件初始化

6.10.3 回调函数

6.10.4 插件配置

tiers:
- plugins:
  - name: conformance

6.11 overcommit

6.11.1 数据结构

type overcommitPlugin struct {
    pluginArguments framework.Arguments //插件参数
    totalResource *api.Resource //总资源
    idleResource *api.Resource //空闲资源
    inqueueResource *api.Resource //inqueue资源
    overCommitFactor float64 //超用比率，必须大于1，默认为1.2
}

6.11.2 插件初始化

6.11.3 回调函数

6.11.4 插件配置

tiers:
- plugins:
  - name: overcommit
    arguments:
      overcommit-factor: 1.0

6.12 sla

6.12.1 数据结构

type slaPlugin struct {
    pluginArguments framework.Arguments //插件参数
    jobWaitingTime *time.Duration //job被inqueue的最大等待时间
}

6.12.2 插件初始化

6.12.3 回调函数

6.12.4 插件配置

tiers:
- plugins:
  - name: sla
    arguments:
      sla-waiting-time: 1h2m3s4ms5µs6ns

kind: Job
metadata:
  annotations:
    sla-waiting-time: 1h2m3s4ms5us6ns

7、功能扩展

7.1 Action扩展

package launch

import (
    "k8s.io/klog/v2"
    "volcano.sh/volcano/pkg/scheduler/api"
    "volcano.sh/volcano/pkg/scheduler/framework"
)

type Action struct {
    session *framework.Session
}

func New() *Action {
    return &Action{}
}

func (launch *Action) Name() string {
    return "launch"
}

func (launch *Action) Initialize() {}

func (launch *Action) Execute(ssn *framework.Session) {
    klog.V(5).Infof("Enter Launch ...")
    defer klog.V(5).Infof("Leaving Launch ...")

    stmt := framework.NewStatement(ssn)
    for _, job := range ssn.Jobs {
        if vr := ssn.JobValid(job); vr != nil && !vr.Pass {
            continue
        }
        for _, task := range job.TaskStatusIndex[api.Pending] {
            selectNode := launch.choseNode(ssn)
            if err := stmt.Allocate(task, selectNode); err != nil {
                _ = stmt.UnAllocate(task)
            }
        }
        if ssn.JobReady(job) {
            stmt.Commit()
        }
    }
}

// select node has minimum pod number
func (launch *Action) choseNode(ssn *framework.Session) *api.NodeInfo {
    var selectNode *api.NodeInfo
    taskNum := -1
    for _, node := range ssn.NodeList {
        if taskNum == -1 || len(node.Tasks) < taskNum {
            taskNum = len(node.Tasks)
            selectNode = node
        }
    }
    return selectNode
}

func (launch *Action) UnInitialize() {}

framework.RegisterAction(launch.New())

actions: "enqueue, launch"
tiers:
- plugins:
  - name: proportion

kubectl apply -f nginx-1.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-1
  namespace: default
  labels:
    app: nginx-1
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx-1
  template:
    metadata:
      labels:
        app: nginx-1
    spec:
      schedulerName: volcano
      containers:
      - name: nginx-1
        image: docker.io/library/nginx:1.25
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 81
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-1
  namespace: default
spec:
  type: NodePort
  selector:
    app: nginx-1
  ports:
  - port: 81
    targetPort: 81
    nodePort: 30201

kubectl apply -f nginx-2.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-2
  namespace: default
  labels:
    app: nginx-2
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx-2
  template:
    metadata:
      labels:
        app: nginx-2
    spec:
      schedulerName: volcano
      containers:
      - name: nginx-2
        image: docker.io/library/nginx:1.25
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 82
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-2
  namespace: default
spec:
  type: NodePort
  selector:
    app: nginx-2
  ports:
  - port: 82
    targetPort: 82
    nodePort: 30202