Improved orchestrator timeouts

- Improved orchestrator timeouts
- Added system reserved data to the orchestrator

src/controllers/orchestrator.go

@@ -722,6 +722,7 @@ func GetOrchestratorOverviewHandler() restapi.ControllerHandler {
 
 		for _, value := range resources {
 			item := models.HostResourceOverviewResponse{}
+			item.SystemReserved = mappers.MapApiHostResourceItemFromHostResourceItem(value.SystemReserved)
 			item.Total = mappers.MapApiHostResourceItemFromHostResourceItem(value.Total)
 			item.TotalAvailable = mappers.MapApiHostResourceItemFromHostResourceItem(value.TotalAvailable)
 			item.TotalInUse = mappers.MapApiHostResourceItemFromHostResourceItem(value.TotalInUse)

src/data/models/host_resources.go

@@ -5,6 +5,7 @@ type HostResourceOverviewResponseItem struct {
 	CpuBrand       string            `json:"cpu_brand,omitempty"`
 	ReverseProxy   *HostReverseProxy `json:"reverse_proxy,omitempty"`
 	TotalAppleVms  int64             `json:"total_apple_vms,omitempty"`
+	SystemReserved HostResourceItem  `json:"system_reserved,omitempty"`
 	Total          HostResourceItem  `json:"total,omitempty"`
 	TotalAvailable HostResourceItem  `json:"total_available,omitempty"`
 	TotalInUse     HostResourceItem  `json:"total_in_use,omitempty"`
@@ -16,6 +17,7 @@ type HostResources struct {
 	CpuBrand       string            `json:"cpu_brand,omitempty"`
 	ReverseProxy   *HostReverseProxy `json:"reverse_proxy,omitempty"`
 	TotalAppleVms  int64             `json:"total_apple_vms,omitempty"`
+	SystemReserved HostResourceItem  `json:"system_reserved,omitempty"`
 	Total          HostResourceItem  `json:"total,omitempty"`
 	TotalAvailable HostResourceItem  `json:"total_available,omitempty"`
 	TotalInUse     HostResourceItem  `json:"total_in_use,omitempty"`
@@ -29,6 +31,19 @@ func (c *HostResources) Diff(source HostResources) bool {
 	if c.Total.Diff(source.Total) {
 		return true
 	}
+	if c.TotalAppleVms != source.TotalAppleVms {
+		return true
+	}
+	if c.ReverseProxy == nil && source.ReverseProxy != nil {
+		return true
+	}
+	if c.ReverseProxy != nil && source.ReverseProxy == nil {
+		return true
+	}
+	if c.ReverseProxy != nil && source.ReverseProxy != nil {
+		c.ReverseProxy.Diff(*source.ReverseProxy)
+	}
+
 	if c.TotalAvailable.Diff(source.TotalAvailable) {
 		return true
 	}
@@ -41,6 +56,9 @@ func (c *HostResources) Diff(source HostResources) bool {
 	if c.TotalAppleVms != source.TotalAppleVms {
 		return true
 	}
+	if c.SystemReserved.Diff(source.SystemReserved) {
+		return true
+	}
 
 	return false
 }
@@ -84,3 +102,26 @@ type HostReverseProxy struct {
 	Port    string             `json:"port,omitempty"`
 	Hosts   []ReverseProxyHost `json:"hosts,omitempty"`
 }
+
+func (c *HostReverseProxy) Diff(source HostReverseProxy) bool {
+	if c.Enabled != source.Enabled {
+		return true
+	}
+	if c.Host != source.Host {
+		return true
+	}
+	if c.Port != source.Port {
+		return true
+	}
+	if len(c.Hosts) != len(source.Hosts) {
+		return true
+	}
+
+	for i, host := range c.Hosts {
+		if host.Diff(source.Hosts[i]) {
+			return true
+		}
+	}
+
+	return false
+}

src/data/models/orchestrator_host.go

@@ -195,13 +195,31 @@ func (o *OrchestratorHost) Diff(source OrchestratorHost) bool {
 	}
 
 	for _, vm := range o.VirtualMachines {
+		found := false
 		for _, vm2 := range source.VirtualMachines {
 			if vm.ID == vm2.ID {
+				found = true
 				if vm.Diff(vm2) {
 					return true
 				}
+				break
 			}
 		}
+		if !found {
+			return true
+		}
+	}
+
+	if o.IsReverseProxyEnabled != source.IsReverseProxyEnabled {
+		return true
+	}
+
+	if o.ReverseProxy != nil && source.ReverseProxy == nil {
+		return true
+	}
+
+	if o.ReverseProxy == nil && source.ReverseProxy != nil {
+		return true
 	}
 
 	if o.ReverseProxy != nil && source.ReverseProxy != nil {

src/data/orchestrator.go

@@ -86,7 +86,7 @@ func (j *JsonDatabase) GetOrchestratorHost(ctx basecontext.ApiContext, idOrHost
 
 	for _, host := range hosts {
 		dbHost := host.GetHost()
-		ctx.LogDebugf("host: %s", dbHost)
+		ctx.LogDebugf("Processing Host: %s", dbHost)
 		if strings.EqualFold(host.ID, idOrHost) || strings.EqualFold(host.GetHost(), idOrHost) {
 			return &host, nil
 		}
@@ -173,8 +173,10 @@ func (j *JsonDatabase) UpdateOrchestratorHost(ctx basecontext.ApiContext, host *
 
 	for _, dbHost := range j.data.OrchestratorHosts {
 		if strings.EqualFold(dbHost.ID, host.ID) || strings.EqualFold(dbHost.Host, host.Host) {
+			ctx.LogDebugf("[Database] Host %s already exists with ID %s", host.Host, dbHost.ID)
 			index, err := GetRecordIndex(j.data.OrchestratorHosts, "id", host.ID)
 			if err != nil {
+				ctx.LogDebugf("[Database] Error getting host index: %v", err.Error())
 				return nil, err
 			}
 			if host.Diff(j.data.OrchestratorHosts[index]) {
@@ -210,6 +212,7 @@ func (j *JsonDatabase) UpdateOrchestratorHost(ctx basecontext.ApiContext, host *
 				j.data.OrchestratorHosts[index].ReverseProxyHosts = host.ReverseProxyHosts
 
 				_ = j.SaveNow(ctx)
+				ctx.LogDebugf("[Database] Host %s updated and saved", host.Host)
 				return &j.data.OrchestratorHosts[index], nil
 			} else {
 				ctx.LogDebugf("[Database] No changes detected for host %s", host.Host)
@@ -218,6 +221,8 @@ func (j *JsonDatabase) UpdateOrchestratorHost(ctx basecontext.ApiContext, host *
 		}
 	}
 
+	ctx.LogDebugf("[Database] Host %s not found, cannot update it", host.Host)
+
 	return nil, ErrOrchestratorHostNotFound
 }
 
@@ -288,6 +293,8 @@ func (j *JsonDatabase) GetOrchestratorAvailableResources(ctx basecontext.ApiCont
 				item.PhysicalCpuCount += host.Resources.TotalAvailable.PhysicalCpuCount
 				item.FreeDiskSize += host.Resources.TotalAvailable.FreeDiskSize
 				item.MemorySize += host.Resources.TotalAvailable.MemorySize
+				item.DiskSize += host.Resources.TotalAvailable.DiskSize
+				item.TotalAppleVms += host.Resources.TotalAvailable.TotalAppleVms
 				result[host.Resources.CpuType] = item
 			}
 		}
@@ -310,6 +317,8 @@ func (j *JsonDatabase) GetOrchestratorTotalResources(ctx basecontext.ApiContext)
 				item.PhysicalCpuCount += host.Resources.Total.PhysicalCpuCount
 				item.FreeDiskSize += host.Resources.Total.FreeDiskSize
 				item.MemorySize += host.Resources.Total.MemorySize
+				item.DiskSize += host.Resources.Total.DiskSize
+				item.TotalAppleVms += host.Resources.Total.TotalAppleVms
 				result[host.Resources.CpuType] = item
 			}
 		}
@@ -331,6 +340,8 @@ func (j *JsonDatabase) GetOrchestratorInUseResources(ctx basecontext.ApiContext)
 				item.LogicalCpuCount += host.Resources.TotalInUse.LogicalCpuCount
 				item.PhysicalCpuCount += host.Resources.TotalInUse.PhysicalCpuCount
 				item.FreeDiskSize += host.Resources.TotalInUse.FreeDiskSize
+				item.DiskSize += host.Resources.TotalInUse.DiskSize
+				item.TotalAppleVms += host.Resources.TotalInUse.TotalAppleVms
 				item.MemorySize += host.Resources.TotalInUse.MemorySize
 				result[host.Resources.CpuType] = item
 			}
@@ -355,6 +366,31 @@ func (j *JsonDatabase) GetOrchestratorReservedResources(ctx basecontext.ApiConte
 				item.PhysicalCpuCount += host.Resources.TotalReserved.PhysicalCpuCount
 				item.FreeDiskSize += host.Resources.TotalReserved.FreeDiskSize
 				item.MemorySize += host.Resources.TotalReserved.MemorySize
+				item.DiskSize += host.Resources.TotalReserved.DiskSize
+				item.TotalAppleVms += host.Resources.TotalReserved.TotalAppleVms
+				result[host.Resources.CpuType] = item
+			}
+		}
+	}
+
+	return result
+}
+
+func (j *JsonDatabase) GetOrchestratorSystemReservedResources(ctx basecontext.ApiContext) map[string]models.HostResourceItem {
+	result := make(map[string]models.HostResourceItem)
+
+	for _, host := range j.data.OrchestratorHosts {
+		if host.State == "healthy" && host.Enabled {
+			if host.Resources != nil {
+				if _, ok := result[host.Resources.CpuType]; !ok {
+					result[host.Resources.CpuType] = models.HostResourceItem{}
+				}
+				item := result[host.Resources.CpuType]
+				item.LogicalCpuCount += host.Resources.SystemReserved.LogicalCpuCount
+				item.PhysicalCpuCount += host.Resources.SystemReserved.PhysicalCpuCount
+				item.FreeDiskSize += host.Resources.SystemReserved.FreeDiskSize
+				item.DiskSize += host.Resources.SystemReserved.DiskSize
+				item.MemorySize += host.Resources.SystemReserved.MemorySize
 				result[host.Resources.CpuType] = item
 			}
 		}

src/mappers/system.go

@@ -9,6 +9,7 @@ func MapHostResourcesFromSystemUsageResponse(m models.SystemUsageResponse) data_
 	result := data_models.HostResources{
 		CpuType:        m.CpuType,
 		CpuBrand:       m.CpuBrand,
+		SystemReserved: MapHostResourceItemFromSystemUsageItem(m.SystemReserved),
 		Total:          MapHostResourceItemFromSystemUsageItem(m.Total),
 		TotalAvailable: MapHostResourceItemFromSystemUsageItem(m.TotalAvailable),
 		TotalInUse:     MapHostResourceItemFromSystemUsageItem(m.TotalInUse),
@@ -49,6 +50,7 @@ func MapSystemUsageResponseFromHostResources(m data_models.HostResources) *model
 	result := models.SystemUsageResponse{
 		CpuType:        m.CpuType,
 		CpuBrand:       m.CpuBrand,
+		SystemReserved: MapSystemUsageItemFromHostResourceItem(&m.SystemReserved),
 		Total:          MapSystemUsageItemFromHostResourceItem(&m.Total),
 		TotalAvailable: MapSystemUsageItemFromHostResourceItem(&m.TotalAvailable),
 		TotalInUse:     MapSystemUsageItemFromHostResourceItem(&m.TotalInUse),

src/models/orchestrator_host.go

@@ -12,7 +12,7 @@ type HostResourceItem struct {
 	TotalAppleVms    int64   `json:"total_apple_vms,omitempty"`
 	PhysicalCpuCount int64   `json:"physical_cpu_count,omitempty"`
 	LogicalCpuCount  int64   `json:"logical_cpu_count"`
-	MemorySize       float64 `json:"memory_size,omitempty"`
+	MemorySize       float64 `json:"memory_size"`
 	DiskSize         float64 `json:"disk_size,omitempty"`
 	FreeDiskSize     float64 `json:"free_disk_size,omitempty"`
 }

src/models/orchestrator_host_resource.go

@@ -3,6 +3,7 @@ package models
 type HostResourceOverviewResponse struct {
 	CpuType        string           `json:"cpu_type,omitempty"`
 	CpuBrand       string           `json:"cpu_brand,omitempty"`
+	SystemReserved HostResourceItem `json:"system_reserved"`
 	Total          HostResourceItem `json:"total"`
 	TotalAvailable HostResourceItem `json:"total_available"`
 	TotalInUse     HostResourceItem `json:"total_in_use"`

src/orchestrator/get_hosts.go

@@ -27,16 +27,15 @@ func (s *OrchestratorService) GetHosts(ctx basecontext.ApiContext, filter string
 
 	var wg sync.WaitGroup
 	mutex := sync.Mutex{}
-
+	wg.Add(len(dtoOrchestratorHosts))
 	for _, host := range dtoOrchestratorHosts {
 		starTime := time.Now()
-		wg.Add(1)
 		go func(host models.OrchestratorHost) {
-			ctx.LogDebugf("Processing Host: %v", host.Host)
 			defer wg.Done()
-
+			ctx.LogDebugf("Processing Host: %v", host.Host)
 			if host.Enabled {
 				host.State = s.GetHostHealthCheckState(&host)
+				ctx.LogDebugf("Host State: %v", host.State)
 			}
 
 			mutex.Lock()

src/orchestrator/get_orchestrator_resources.go

@@ -16,10 +16,12 @@ func (s *OrchestratorService) GetResources(ctx basecontext.ApiContext) ([]models
 	inUseResources := dbService.GetOrchestratorInUseResources(ctx)
 	availableResources := dbService.GetOrchestratorAvailableResources(ctx)
 	reservedResources := dbService.GetOrchestratorReservedResources(ctx)
+	systemReservedResources := dbService.GetOrchestratorSystemReservedResources(ctx)
 
 	result := make([]models.HostResourceOverviewResponseItem, 0)
 	for key, value := range totalResources {
 		item := models.HostResourceOverviewResponseItem{}
+		item.SystemReserved = systemReservedResources[key]
 		item.Total = value
 		item.TotalAvailable = availableResources[key]
 		item.TotalInUse = inUseResources[key]

src/orchestrator/main.go

@@ -9,6 +9,7 @@ import (
 	"github.com/Parallels/prl-devops-service/config"
 	"github.com/Parallels/prl-devops-service/data"
 	"github.com/Parallels/prl-devops-service/data/models"
+	"github.com/Parallels/prl-devops-service/helpers"
 	"github.com/Parallels/prl-devops-service/mappers"
 	apimodels "github.com/Parallels/prl-devops-service/models"
 	"github.com/Parallels/prl-devops-service/restapi"
@@ -32,8 +33,8 @@ func NewOrchestratorService(ctx basecontext.ApiContext) *OrchestratorService {
 	if globalOrchestratorService == nil {
 		globalOrchestratorService = &OrchestratorService{
 			ctx:                ctx,
-			timeout:            2 * time.Minute,
-			healthCheckTimeout: 10 * time.Second,
+			timeout:            5 * time.Minute,
+			healthCheckTimeout: 3 * time.Second,
 		}
 		cfg := config.Get()
 		globalOrchestratorService.refreshInterval = time.Duration(cfg.OrchestratorPullFrequency()) * time.Second
@@ -148,6 +149,7 @@ func (s *OrchestratorService) processHost(host models.OrchestratorHost) {
 		_ = s.persistHost(&host)
 		return
 	} else {
+		s.ctx.LogInfof("[Orchestrator] host %s is alive and well: %s", host.Host, healthCheck.Message)
 		host.SetHealthy()
 		host.HealthCheck = healthCheck
 	}
@@ -189,8 +191,6 @@ func (s *OrchestratorService) processHost(host models.OrchestratorHost) {
 		}
 	}
 
-	s.ctx.LogInfof("[Orchestrator] Host %s has %v CPU Cores and %v Mb of RAM", host.Host, host.Resources.Total.LogicalCpuCount, host.Resources.Total.MemorySize)
-
 	// Updating the Virtual Machines
 	vms, err := s.GetHostVirtualMachinesInfo(&host)
 	if err != nil {
@@ -216,13 +216,18 @@ func (s *OrchestratorService) processHost(host models.OrchestratorHost) {
 	}
 
 	host.Resources.TotalAppleVms = int64(totalAppleVms)
+	host.UpdatedAt = helpers.GetUtcCurrentDateTime()
+
+	s.ctx.LogInfof("[Orchestrator] Host %s has %v CPU Cores and %v Mb of RAM, contains %v VMs of which %v are MacVMs", host.Host, host.Resources.Total.LogicalCpuCount, host.Resources.Total.MemorySize, len(host.VirtualMachines), host.Resources.TotalAppleVms)
 
 	_ = s.persistHost(&host)
 
 	// Free up memory
 	host.HealthCheck = nil
 	host.Resources = nil
 	host.VirtualMachines = nil
+	host.ReverseProxy = nil
+	host.ReverseProxyHosts = nil
 }
 
 func (s *OrchestratorService) persistHost(host *models.OrchestratorHost) error {
@@ -235,20 +240,18 @@ func (s *OrchestratorService) persistHost(host *models.OrchestratorHost) error {
 		s.ctx.LogErrorf("[Orchestrator] Error getting host %s: %v", host.Host, err.Error())
 		return err
 	}
-	if oldHost.UpdatedAt != host.UpdatedAt {
-		hostToSave = *oldHost
-		hostToSave.HealthCheck = host.HealthCheck
-		hostToSave.Resources = host.Resources
-		hostToSave.VirtualMachines = host.VirtualMachines
-		hostToSave.ReverseProxy = host.ReverseProxy
-		hostToSave.ReverseProxyHosts = host.ReverseProxyHosts
-	}
-
-	if _, err := s.db.UpdateOrchestratorHost(s.ctx, &hostToSave); err != nil {
-		s.ctx.LogErrorf("[Orchestrator] Error saving host %s: %v", host.Host, err.Error())
-		return err
+	s.ctx.LogDebugf("[Orchestrator] oldHost: %v, updated at %s and new one %s updated at %v", oldHost.ID, oldHost.UpdatedAt, host.ID, host.UpdatedAt)
+	if oldHost.UpdatedAt > host.UpdatedAt {
+		s.ctx.LogDebugf("[Orchestrator] Host %s was updated by another process, skipping", host.Host)
+	} else {
+		s.ctx.LogDebugf("[Orchestrator] Saving host %s", host.Host)
+		if _, err := s.db.UpdateOrchestratorHost(s.ctx, &hostToSave); err != nil {
+			s.ctx.LogErrorf("[Orchestrator] Error saving host %s: %v", host.Host, err.Error())
+			return err
+		}
 	}
 
+	s.ctx.LogDebugf("[Orchestrator] Host %s saved, freeing up memory", host.Host)
 	// Free up memory
 	hostToSave.HealthCheck = nil
 	hostToSave.Resources = nil