Skip to content

Commit

Permalink
Initial work on the cache endpoints (#244)
Browse files Browse the repository at this point in the history 
- Added the ability to control cache from the orchestrator
- improved error handling
- Improved the cached cleaning process
  • Loading branch information
@cjlapao
cjlapao authored Nov 20, 2024
1 parent d58c626 commit db3fd29
Show file tree
Hide file tree
Showing 10 changed files with 796 additions and 80 deletions.
220 changes: 164 additions & 56 deletions src/catalog/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,52 @@ import (
"github.com/Parallels/prl-devops-service/basecontext"
"github.com/Parallels/prl-devops-service/catalog/models"
"github.com/Parallels/prl-devops-service/config"
"github.com/Parallels/prl-devops-service/errors"
"github.com/Parallels/prl-devops-service/serviceprovider/system"
"github.com/cjlapao/common-go/helper"
)

type CacheFile struct {
IsDir bool
Path string
}

func (s *CatalogManifestService) CleanAllCache(ctx basecontext.ApiContext) error {
cfg := config.Get()
cacheLocation, err := cfg.CatalogCacheFolder()
if err != nil {
return err
}

err = filepath.Walk(cacheLocation, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
return os.Remove(path)
}
return os.RemoveAll(path)
})
clearFiles := []CacheFile{}
entries, err := os.ReadDir(cacheLocation)
if err != nil {
return err
}

for _, entry := range entries {
clearFiles = append(clearFiles, CacheFile{
IsDir: entry.IsDir(),
Path: filepath.Join(cacheLocation, entry.Name()),
})
}

for _, file := range clearFiles {
if file.IsDir {
if err := os.RemoveAll(file.Path); err != nil {
return err
}
} else {
if err := os.Remove(file.Path); err != nil {
return err
}
}
}

return nil
}

func (s *CatalogManifestService) CleanCacheFile(ctx basecontext.ApiContext, catalogId string) error {
func (s *CatalogManifestService) CleanCacheFile(ctx basecontext.ApiContext, catalogId string, version string) error {
cfg := config.Get()
cacheLocation, err := cfg.CatalogCacheFolder()
if err != nil {
Expand All @@ -47,24 +66,38 @@ func (s *CatalogManifestService) CleanCacheFile(ctx basecontext.ApiContext, cata
return err
}

found := false
for _, cache := range allCache.Manifests {
if strings.EqualFold(cache.CatalogId, catalogId) {
if cache.CacheType == "folder" {
if err := os.RemoveAll(filepath.Join(cacheLocation, cache.CacheFileName)); err != nil {
return err
}
} else {
if err := os.Remove(cache.CacheLocalFullPath); err != nil {
return err
if strings.EqualFold(cache.Version, version) || version == "" {
found = true
if cache.CacheType == "folder" {
if err := os.RemoveAll(filepath.Join(cacheLocation, cache.CacheFileName)); err != nil {
return err
}
} else {
if err := os.Remove(cache.CacheLocalFullPath); err != nil {
return err
}
}
}

if err := os.Remove(filepath.Join(cacheLocation, cache.CacheMetadataName)); err != nil {
return err
if cache.CacheMetadataName != "" {
if _, err := os.Stat(filepath.Join(cacheLocation, cache.CacheMetadataName)); os.IsNotExist(err) {
continue
}

if err := os.Remove(filepath.Join(cacheLocation, cache.CacheMetadataName)); err != nil {
return err
}
}
}
}
}

if !found {
return errors.NewWithCodef(404, "Cache not found for catalog %s and version %s", catalogId, version)
}

return nil
}

Expand All @@ -84,54 +117,129 @@ func (s *CatalogManifestService) GetCacheItems(ctx basecontext.ApiContext) (mode
return err
}
if !info.IsDir() && filepath.Ext(path) == ".meta" {
var metaContent models.VirtualMachineCatalogManifest
manifestBytes, err := helper.ReadFromFile(path)
if err != nil {
if err := s.processMetadataCache(path, info, &response, &totalSize); err != nil {
return err
}
err = json.Unmarshal(manifestBytes, &metaContent)
if err != nil {
} else {
if err := s.processOldCache(ctx, path, info, &response, &totalSize); err != nil {
return err
}
cacheName := strings.TrimSuffix(path, filepath.Ext(path))
cacheInfo, err := os.Stat(cacheName)
}
return nil
})
if err != nil {
return response, err
}

response.TotalSize = totalSize
if response.Manifests == nil {
response.Manifests = make([]models.VirtualMachineCatalogManifest, 0)
}

return response, nil
}

func (s *CatalogManifestService) processMetadataCache(path string, info os.FileInfo, response *models.VirtualMachineCatalogManifestList, totalSize *int64) error {
var metaContent models.VirtualMachineCatalogManifest
manifestBytes, err := helper.ReadFromFile(path)
if err != nil {
return err
}
err = json.Unmarshal(manifestBytes, &metaContent)
if err != nil {
return err
}
cacheName := strings.TrimSuffix(path, filepath.Ext(path))
cacheInfo, err := os.Stat(cacheName)
if err != nil {
return err
}
if cacheInfo.IsDir() {
metaContent.CacheType = "folder"
var folderSize int64
err = filepath.Walk(cacheName, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if cacheInfo.IsDir() {
metaContent.CacheType = "folder"
var totalSize int64
err = filepath.Walk(cacheName, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
totalSize += info.Size()
}
return nil
})
if !info.IsDir() {
folderSize += info.Size()
}
return nil
})
if err != nil {
return err
}
metaContent.CacheSize = int64(float64(folderSize) / (1024 * 1024))
} else {
metaContent.CacheType = "file"
metaContent.CacheSize = int64(float64(cacheInfo.Size()) / (1024 * 1024))
}

metaContent.CacheLocalFullPath = cacheName
metaContent.CacheFileName = filepath.Base(cacheName)
metaContent.CacheMetadataName = filepath.Base(path)
metaContent.CacheDate = info.ModTime().Format("2006-01-02 15:04:05")
response.Manifests = append(response.Manifests, metaContent)
*totalSize += metaContent.CacheSize
return nil
}

func (s *CatalogManifestService) processOldCache(ctx basecontext.ApiContext, path string, info os.FileInfo, response *models.VirtualMachineCatalogManifestList, totalSize *int64) error {
if filepath.Ext(path) == ".pvm" || filepath.Ext(path) == ".macvm" {
metaPath := path + ".meta"
if _, err := os.Stat(metaPath); err == nil {
return nil
}

srvCtl := system.Get()
arch, err := srvCtl.GetArchitecture(ctx)
if err != nil {
arch = "unknown"
}

cacheSize := info.Size() / 1024 / 1024
cacheType := "file"
if info.IsDir() {
cacheType = "folder"
var folderSize int64
err = filepath.Walk(path, func(p string, i os.FileInfo, err error) error {
if err != nil {
return err
}
metaContent.CacheSize = totalSize
} else {
metaContent.CacheType = "file"
metaContent.CacheSize = cacheInfo.Size()
if !i.IsDir() {
folderSize += i.Size()
}
return nil
})
if err != nil {
return err
}

metaContent.CacheLocalFullPath = cacheName
metaContent.CacheFileName = filepath.Base(cacheName)
metaContent.CacheMetadataName = filepath.Base(path)
metaContent.CacheDate = info.ModTime().Format("2006-01-02 15:04:05")
response.Manifests = append(response.Manifests, metaContent)
totalSize += metaContent.CacheSize
cacheSize = folderSize / 1024 / 1024
}
return nil
})
if err != nil {
return response, err
}

response.TotalSize = totalSize
return response, nil
oldCacheManifest := models.VirtualMachineCatalogManifest{
ID: filepath.Base(path),
CatalogId: filepath.Base(path),
Version: "unknown",
Architecture: arch,
CacheType: cacheType,
CacheSize: cacheSize,
CacheLocalFullPath: path,
CacheFileName: filepath.Base(path),
CacheMetadataName: filepath.Base(path),
CacheDate: info.ModTime().Format("2006-01-02 15:04:05"),
IsCompressed: false,
Size: cacheSize,
}
if filepath.Ext(path) == ".pvm" {
oldCacheManifest.Type = "pvm"
} else {
oldCacheManifest.Type = "macvm"
}

response.Manifests = append(response.Manifests, oldCacheManifest)
*totalSize += cacheSize
}
return nil
}
69 changes: 64 additions & 5 deletions src/controllers/catalog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,14 @@ func registerCatalogManifestHandlers(ctx basecontext.ApiContext, version string)
WithHandler(DeleteCatalogCacheItemHandler()).
Register()

restapi.NewController().
WithMethod(restapi.DELETE).
WithVersion(version).
WithPath("/catalog/cache/{catalogId}/{version}").
WithRequiredClaim(constants.SUPER_USER_ROLE).
WithHandler(DeleteCatalogCacheItemVersionHandler()).
Register()

restapi.NewController().
WithMethod(restapi.GET).
WithVersion(version).
Expand Down Expand Up @@ -1880,10 +1888,10 @@ func GetCatalogCacheHandler() restapi.ControllerHandler {
return
}

// responseManifests := mappers.DtoCatalogManifestsToApi(manifestsDto)
responseManifests := mappers.BaseVirtualMachineCatalogManifestListToApi(items)

w.WriteHeader(http.StatusOK)
_ = json.NewEncoder(w).Encode(items)
_ = json.NewEncoder(w).Encode(responseManifests)
ctx.LogInfof("Manifests cached items returned: %v", len(items.Manifests))
}
}
Expand Down Expand Up @@ -1919,8 +1927,8 @@ func DeleteCatalogCacheHandler() restapi.ControllerHandler {
}
}

// @Summary Deletes catalog cache item
// @Description This endpoint returns all the remote catalog cache if any
// @Summary Deletes catalog cache item and all its versions
// @Description This endpoint returns all the remote catalog cache if any and all its versions
// @Tags Catalogs
// @Produce json
// @Param catalogId path string true "Catalog ID"
Expand All @@ -1947,7 +1955,58 @@ func DeleteCatalogCacheItemHandler() restapi.ControllerHandler {
}

catalogSvc := catalog.NewManifestService(ctx)
err := catalogSvc.CleanCacheFile(ctx, catalogId)
err := catalogSvc.CleanCacheFile(ctx, catalogId, "")
if err != nil {
ReturnApiError(ctx, w, models.NewFromError(err))
return
}

// responseManifests := mappers.DtoCatalogManifestsToApi(manifestsDto)

w.WriteHeader(http.StatusAccepted)
ctx.LogInfof("Manifests cached item %v removed", len(catalogId))
}
}

// @Summary Deletes catalog cache version item
// @Description This endpoint deletes a version of a cache ite,
// @Tags Catalogs
// @Produce json
// @Param catalogId path string true "Catalog ID"
// @Param version path string true "Version"
// @Success 202
// @Failure 400 {object} models.ApiErrorResponse
// @Failure 401 {object} models.OAuthErrorResponse
// @Security ApiKeyAuth
// @Security BearerAuth
// @Router /v1/catalog/cache/{catalogId}/{version} [delete]
func DeleteCatalogCacheItemVersionHandler() restapi.ControllerHandler {
return func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
ctx := GetBaseContext(r)
defer Recover(ctx, r, w)

vars := mux.Vars(r)
catalogId := vars["catalogId"]
version := vars["version"]
if catalogId == "" {
ReturnApiError(ctx, w, models.ApiErrorResponse{
Message: "Catalog ID is required",
Code: http.StatusBadRequest,
})
return
}

if version == "" {
ReturnApiError(ctx, w, models.ApiErrorResponse{
Message: "Version is required",
Code: http.StatusBadRequest,
})
return
}

catalogSvc := catalog.NewManifestService(ctx)
err := catalogSvc.CleanCacheFile(ctx, catalogId, version)
if err != nil {
ReturnApiError(ctx, w, models.NewFromError(err))
return
Expand Down
Loading

0 comments on commit db3fd29

Please sign in to comment.