Per cluster member resource reservations

doc/metadata.txt

@@ -1,6 +1,26 @@
 // Code generated by lxd-metadata; DO NOT EDIT.
 
 <!-- config group cluster-cluster start -->
+```{config:option} limits.reserve.cpu cluster-cluster
+:shortdesc: "Number of CPUs to reserve for the LXD server"
+:type: "integer"
+Number of CPUs to reserve for the LXD server. This setting only limits
+the sum of instance `limits.cpu` that can be located on a cluster member.
+
+When this key is set, all instances on that member must have
+`limits.cpu` set.
+```
+
+```{config:option} limits.reserve.memory cluster-cluster
+:shortdesc: "Amount of memory to reserve for the LXD server"
+:type: "string"
+Amount of memory to reserve for the LXD server. This setting only limits
+the sum of instance `limits.memory` that can be located on a cluster member.
+
+When this key is set, all instances on that member must have
+`limits.memory` set.
+```
+
 ```{config:option} scheduler.instance cluster-cluster
 :defaultdesc: "`all`"
 :shortdesc: "Controls how instances are scheduled to run on this member"
@@ -4623,6 +4643,30 @@ When using custom automatic instance placement logic, this option stores the scr
 See {ref}`clustering-instance-placement-scriptlet` for more information.
 ```
 
+```{config:option} limits.reserve.cpu server-miscellaneous
+:scope: "global"
+:shortdesc: "Number CPUs to reserve for the LXD server"
+:type: "integer"
+Number of CPUs to reserve for the LXD server. This setting only limits
+the sum of instance `limits.cpu` that can be located on any cluster member.
+
+This value is overridden by the corresponding cluster member configuration key.
+
+When this key is set, all instances must have `limits.cpu` set.
+```
+
+```{config:option} limits.reserve.memory server-miscellaneous
+:scope: "global"
+:shortdesc: "Amount of memory to reserve for the LXD server"
+:type: "string"
+Amount of memory to reserve for the LXD server. This setting only limits
+the sum of instance `limits.memory` that can be located on any cluster member.
+
+This value is overridden by the corresponding cluster member configuration key.
+
+When this key is set, all instances must have `limits.memory` set.
+```
+
 ```{config:option} maas.api.key server-miscellaneous
 :scope: "global"
 :shortdesc: "API key to manage MAAS"

doc/rest-api.yaml

@@ -583,6 +583,10 @@ definitions:
                 format: uint64
                 type: integer
                 x-go-name: BufferRAM
+            cpu_threads:
+                format: uint64
+                type: integer
+                x-go-name: CPUThreads
             free_ram:
                 format: uint64
                 type: integer

lxd/api_1.0.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"os"
 	"slices"
+	"strings"
 
 	"github.com/canonical/lxd/client"
 	"github.com/canonical/lxd/lxd/auth"
@@ -21,6 +22,7 @@ import (
 	"github.com/canonical/lxd/lxd/instance/instancetype"
 	"github.com/canonical/lxd/lxd/lifecycle"
 	"github.com/canonical/lxd/lxd/node"
+	"github.com/canonical/lxd/lxd/project/limits"
 	"github.com/canonical/lxd/lxd/request"
 	"github.com/canonical/lxd/lxd/response"
 	scriptletLoad "github.com/canonical/lxd/lxd/scriptlet/load"
@@ -699,7 +701,38 @@ func doAPI10Update(d *Daemon, r *http.Request, req api.ServerPut, patch bool) re
 	// Then deal with cluster wide configuration
 	var clusterChanged map[string]string
 	var newClusterConfig *clusterConfig.Config
-	oldClusterConfig := make(map[string]any)
+	var oldClusterConfig map[string]any
+
+	// If req.Config doesn't contain a limits.reserve.* then we don't need to
+	// call AllowClusterUpdate at all; Removing a global limits.reserve.*
+	// doesn't change the effective limit on any cluster member.
+	changesReservation := false
+	for key := range req.Config {
+		if strings.Contains(key, "limits.reserve") {
+			changesReservation = true
+			break
+		}
+	}
+
+	var sysinfo map[string]api.ClusterMemberSysInfo
+	if changesReservation {
+		var members []db.NodeInfo
+
+		err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error {
+			members, err = tx.GetNodes(ctx)
+			return err
+		})
+		if err != nil {
+			return response.SmartError(err)
+		}
+
+		sysinfo, err = cluster.ClusterSysInfo(s, members)
+		if err == cluster.ErrorClusterUnavailable {
+			return response.Unavailable(fmt.Errorf("Cannot set limits.reserve.{cpu,memory} when cluster members are unreachble"))
+		} else if err != nil {
+			return response.SmartError(err)
+		}
+	}
 
 	err = s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error {
 		var err error
@@ -709,8 +742,23 @@ func doAPI10Update(d *Daemon, r *http.Request, req api.ServerPut, patch bool) re
 		}
 
 		// Keep old config around in case something goes wrong. In that case the config will be reverted.
-		for k, v := range newClusterConfig.Dump() {
-			oldClusterConfig[k] = v
+		oldClusterConfig = newClusterConfig.Dump()
+
+		if changesReservation {
+			// TODO Don't do the patch logic twice :facepalm:
+			validationConfig := newClusterConfig.Dump()
+			if patch {
+				for key, val := range req.Config {
+					validationConfig[key] = val
+				}
+			} else {
+				validationConfig = req.Config
+			}
+
+			err = limits.AllowClusterUpdate(ctx, tx, sysinfo, validationConfig)
+			if err != nil {
+				return err
+			}
 		}
 
 		if patch {

lxd/api_cluster.go

@@ -35,6 +35,7 @@ import (
 	"github.com/canonical/lxd/lxd/lifecycle"
 	"github.com/canonical/lxd/lxd/node"
 	"github.com/canonical/lxd/lxd/operations"
+	"github.com/canonical/lxd/lxd/project/limits"
 	"github.com/canonical/lxd/lxd/request"
 	"github.com/canonical/lxd/lxd/response"
 	"github.com/canonical/lxd/lxd/scriptlet"
@@ -1621,6 +1622,11 @@ func updateClusterNode(s *state.State, gateway *cluster.Gateway, r *http.Request
 		return response.SmartError(err)
 	}
 
+	resp := forwardedResponseToNode(s, r, name)
+	if resp != nil {
+		return resp
+	}
+
 	leaderAddress, err := gateway.LeaderAddress()
 	if err != nil {
 		return response.InternalError(err)
@@ -1715,6 +1721,16 @@ func updateClusterNode(s *state.State, gateway *cluster.Gateway, r *http.Request
 		newRoles = append(newRoles, db.ClusterRole(role))
 	}
 
+	sysinfo, err := cluster.LocalSysInfo()
+	if err != nil {
+		return response.InternalError(err)
+	}
+
+	var globalConfigDump map[string]any
+	if s.GlobalConfig != nil {
+		globalConfigDump = s.GlobalConfig.Dump()
+	}
+
 	// Update the database
 	err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
 		nodeInfo, err := tx.GetNodeByName(ctx, name)
@@ -1741,6 +1757,11 @@ func updateClusterNode(s *state.State, gateway *cluster.Gateway, r *http.Request
 			}
 		}
 
+		err = limits.AllowClusterMemberUpdate(ctx, tx, globalConfigDump, name, sysinfo, req.Config)
+		if err != nil {
+			return fmt.Errorf("Permission denied: %w", err)
+		}
+
 		// Update node config.
 		err = tx.UpdateNodeConfig(ctx, nodeInfo.ID, req.Config)
 		if err != nil {
@@ -1829,6 +1850,28 @@ func clusterValidateConfig(config map[string]string) error {
 		//  defaultdesc: `all`
 		//  shortdesc: Controls how instances are scheduled to run on this member
 		"scheduler.instance": validate.Optional(validate.IsOneOf("all", "group", "manual")),
+
+		// lxdmeta:generate(entities=cluster; group=cluster; key=limits.reserve.cpu)
+		// Number of CPUs to reserve for the LXD server. This setting only limits
+		// the sum of instance `limits.cpu` that can be located on a cluster member.
+		//
+		// When this key is set, all instances on that member must have
+		// `limits.cpu` set.
+		// ---
+		//  type: integer
+		//  shortdesc: Number of CPUs to reserve for the LXD server
+		"limits.reserve.cpu": validate.IsAny,
+
+		// lxdmeta:generate(entities=cluster; group=cluster; key=limits.reserve.memory)
+		// Amount of memory to reserve for the LXD server. This setting only limits
+		// the sum of instance `limits.memory` that can be located on a cluster member.
+		//
+		// When this key is set, all instances on that member must have
+		// `limits.memory` set.
+		// ---
+		//  type: string
+		//  shortdesc: Amount of memory to reserve for the LXD server
+		"limits.reserve.memory": validate.IsAny,
 	}
 
 	for k, v := range config {
@@ -2961,7 +3004,7 @@ func clusterNodeStateGet(d *Daemon, r *http.Request) response.Response {
 		return resp
 	}
 
-	memberState, err := cluster.MemberState(r.Context(), s, memberName)
+	memberState, err := cluster.MemberState(r.Context(), s)
 	if err != nil {
 		return response.SmartError(err)
 	}

lxd/cluster/config/config.go

@@ -734,6 +734,32 @@ var ConfigSchema = config.Schema{
 	//  defaultdesc: Content of `/etc/ovn/key_host` if present
 	//  shortdesc: OVN SSL client key
 	"network.ovn.client_key": {Default: ""},
+
+	// lxdmeta:generate(entities=server; group=miscellaneous; key=limits.reserve.cpu)
+	// Number of CPUs to reserve for the LXD server. This setting only limits
+	// the sum of instance `limits.cpu` that can be located on any cluster member.
+	//
+	// This value is overridden by the corresponding cluster member configuration key.
+	//
+	// When this key is set, all instances must have `limits.cpu` set.
+	// ---
+	//  type: integer
+	//  scope: global
+	//  shortdesc: Number CPUs to reserve for the LXD server
+	"limits.reserve.cpu": {Type: config.Int64},
+
+	// lxdmeta:generate(entities=server; group=miscellaneous; key=limits.reserve.memory)
+	// Amount of memory to reserve for the LXD server. This setting only limits
+	// the sum of instance `limits.memory` that can be located on any cluster member.
+	//
+	// This value is overridden by the corresponding cluster member configuration key.
+	//
+	// When this key is set, all instances must have `limits.memory` set.
+	// ---
+	//  type: string
+	//  scope: global
+	//  shortdesc: Amount of memory to reserve for the LXD server
+	"limits.reserve.memory": {Type: config.String},
 }
 
 func expiryValidator(value string) error {