Cache /api/-/search results

eclipse · Mar 30, 2023 · 45f9cca · 45f9cca
1 parent cdf9e01
commit 45f9cca
Show file tree

Hide file tree

Showing 11 changed files with 224 additions and 81 deletions.
diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionService.java b/server/src/main/java/org/eclipse/openvsx/ExtensionService.java
@@ -117,6 +117,7 @@ public void updateExtension(Extension extension) {
         cache.evictNamespaceDetails(extension);
         cache.evictLatestExtensionVersion(extension);
         cache.evictExtensionJsons(extension);
+        cache.evictSearchEntryJsons(extension);
 
         if (extension.getVersions().stream().anyMatch(ExtensionVersion::isActive)) {
             // There is at least one active version => activate the extension

diff --git a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java
@@ -29,7 +29,6 @@
 import org.eclipse.openvsx.entities.*;
 import org.eclipse.openvsx.json.*;
 import org.eclipse.openvsx.repositories.RepositoryService;
-import org.eclipse.openvsx.search.ExtensionSearch;
 import org.eclipse.openvsx.search.ISearchService;
 import org.eclipse.openvsx.search.SearchUtilService;
 import org.eclipse.openvsx.storage.StorageUtilService;
@@ -39,8 +38,6 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.cache.annotation.Cacheable;
 import org.springframework.dao.DataIntegrityViolationException;
-import org.springframework.data.elasticsearch.core.SearchHit;
-import org.springframework.data.elasticsearch.core.SearchHits;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.retry.annotation.Retryable;
@@ -82,6 +79,9 @@ public class LocalRegistryService implements IExtensionRegistry {
     @Autowired
     CacheService cache;
 
+    @Autowired
+    SearchEntryService searchEntries;
+
     @Override
     public NamespaceJson getNamespace(String namespaceName) {
         var namespace = repositories.findNamespace(namespaceName);
@@ -211,7 +211,22 @@ public SearchResultJson search(ISearchService.Options options) {
         }
 
         var searchHits = search.search(options);
-        json.extensions = toSearchEntries(searchHits, options);
+        var extensions = new ArrayList<SearchEntryJson>();
+        for (var searchHit : searchHits) {
+            var searchEntry = searchEntries.toJson(searchHit, options.includeAllVersions);
+            if(searchEntry != null) {
+                // use averageRating, reviewCount and downloadCount from ElasticSearch response,
+                // so that cached SearchEntryJson doesn't have to be evicted every time
+                // averageRating, reviewCount or downloadCount are updated.
+                var extensionSearch = searchHit.getContent();
+                searchEntry.averageRating = extensionSearch.averageRating;
+                searchEntry.reviewCount = extensionSearch.reviewCount;
+                searchEntry.downloadCount = extensionSearch.downloadCount;
+                extensions.add(searchEntry);
+            }
+        }
+
+        json.extensions = extensions;
         json.offset = options.requestedOffset;
         json.totalSize = (int) searchHits.getTotalHits();
         return json;
@@ -741,81 +756,6 @@ public ResultJson deleteReview(String namespace, String extensionName) {
         return ResultJson.success("Deleted review for " + extension.getNamespace().getName() + "." + extension.getName());
     }
 
-    private Extension getExtension(SearchHit<ExtensionSearch> searchHit) {
-        var searchItem = searchHit.getContent();
-        var extension = entityManager.find(Extension.class, searchItem.id);
-        if (extension == null || !extension.isActive()) {
-            extension = new Extension();
-            extension.setId(searchItem.id);
-            search.removeSearchEntry(extension);
-            return null;
-        }
-
-        return extension;
-    }
-
-    private List<SearchEntryJson> toSearchEntries(SearchHits<ExtensionSearch> searchHits, ISearchService.Options options) {
-        var serverUrl = UrlUtil.getBaseUrl();
-        var extensions = searchHits.stream()
-                .map(this::getExtension)
-                .filter(Objects::nonNull)
-                .collect(Collectors.toList());
-
-        var latestVersions = extensions.stream()
-                .map(e -> {
-                    var latest = versions.getLatestTrxn(e, null, false, true);
-                    return new AbstractMap.SimpleEntry<>(e.getId(), latest);
-                })
-                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
-
-        var searchEntries = latestVersions.entrySet().stream()
-                .map(e -> {
-                    var entry = e.getValue().toSearchEntryJson();
-                    entry.url = createApiUrl(serverUrl, "api", entry.namespace, entry.name);
-                    return new AbstractMap.SimpleEntry<>(e.getKey(), entry);
-                })
-                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
-
-        var fileUrls = storageUtil.getFileUrls(latestVersions.values(), serverUrl, DOWNLOAD, ICON);
-        searchEntries.forEach((extensionId, searchEntry) -> searchEntry.files = fileUrls.get(latestVersions.get(extensionId).getId()));
-        if (options.includeAllVersions) {
-            var allActiveVersions = repositories.findActiveVersions(extensions).stream()
-                    .sorted(ExtensionVersion.SORT_COMPARATOR)
-                    .collect(Collectors.toList());
-
-            var activeVersionsByExtensionId = allActiveVersions.stream()
-                    .collect(Collectors.groupingBy(ev -> ev.getExtension().getId()));
-
-            var versionUrls = storageUtil.getFileUrls(allActiveVersions, serverUrl, DOWNLOAD);
-            for(var extension : extensions) {
-                var activeVersions = activeVersionsByExtensionId.get(extension.getId());
-                var searchEntry = searchEntries.get(extension.getId());
-                searchEntry.allVersions = getAllVersionReferences(activeVersions, versionUrls, serverUrl);
-            }
-        }
-
-        return extensions.stream()
-                .map(Extension::getId)
-                .map(searchEntries::get)
-                .collect(Collectors.toList());
-    }
-
-    private List<SearchEntryJson.VersionReference> getAllVersionReferences(
-            List<ExtensionVersion> extVersions,
-            Map<Long, Map<String, String>> versionUrls,
-            String serverUrl
-    ) {
-        Collections.sort(extVersions, ExtensionVersion.SORT_COMPARATOR);
-        return extVersions.stream().map(extVersion -> {
-            var ref = new SearchEntryJson.VersionReference();
-            ref.version = extVersion.getVersion();
-            ref.engines = extVersion.getEnginesMap();
-            ref.url = UrlUtil.createApiVersionUrl(serverUrl, extVersion);
-            ref.files = versionUrls.get(extVersion.getId());
-            return ref;
-        }).collect(Collectors.toList());
-    }
-
     public ExtensionJson toExtensionVersionJson(ExtensionVersion extVersion, String targetPlatform, boolean onlyActive, boolean inTransaction) {
         var extension = extVersion.getExtension();
         var latest = inTransaction

diff --git a/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java b/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java
@@ -12,11 +12,14 @@
 import org.eclipse.openvsx.entities.Extension;
 import org.eclipse.openvsx.entities.ExtensionVersion;
 import org.eclipse.openvsx.entities.UserData;
+import org.eclipse.openvsx.json.SearchEntryJson;
 import org.eclipse.openvsx.repositories.RepositoryService;
+import org.eclipse.openvsx.search.ExtensionSearch;
 import org.eclipse.openvsx.util.TargetPlatform;
 import org.eclipse.openvsx.util.VersionAlias;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.cache.CacheManager;
+import org.springframework.data.elasticsearch.core.SearchHit;
 import org.springframework.stereotype.Component;
 
 import java.util.ArrayList;
@@ -27,6 +30,7 @@ public class CacheService {
 
     public static final String CACHE_DATABASE_SEARCH = "database.search";
     public static final String CACHE_EXTENSION_JSON = "extension.json";
+    public static final String CACHE_SEARCH_ENTRY_JSON = "search.entry.json";
     public static final String CACHE_LATEST_EXTENSION_VERSION = "latest.extension.version";
     public static final String CACHE_NAMESPACE_DETAILS_JSON = "namespace.details.json";
     public static final String CACHE_AVERAGE_REVIEW_RATING = "average.review.rating";
@@ -43,6 +47,9 @@ public class CacheService {
     @Autowired
     ExtensionJsonCacheKeyGenerator extensionJsonCacheKey;
 
+    @Autowired
+    SearchEntryJsonCacheKeyGenerator searchEntryJsonCacheKeyGenerator;
+
     @Autowired
     LatestExtensionVersionCacheKeyGenerator latestExtensionVersionCacheKey;
 
@@ -92,6 +99,33 @@ public void evictExtensionJsons(Extension extension) {
         }
     }
 
+    public SearchEntryJson getSearchEntryJson(SearchHit<ExtensionSearch> searchHit, boolean includeAllVersions) {
+        var cache = cacheManager.getCache(CACHE_SEARCH_ENTRY_JSON);
+        return cache != null
+                ? cache.get(searchEntryJsonCacheKeyGenerator.generate(searchHit.getContent().id, includeAllVersions), SearchEntryJson.class)
+                : null;
+    }
+
+    public void putSearchEntryJson(SearchEntryJson searchEntry, SearchHit<ExtensionSearch> searchHit, boolean includeAllVersions) {
+        var cache = cacheManager.getCache(CACHE_SEARCH_ENTRY_JSON);
+        if(cache != null) {
+            cache.put(searchEntryJsonCacheKeyGenerator.generate(searchHit.getContent().id, includeAllVersions), searchEntry);
+        }
+    }
+
+    public void evictSearchEntryJsons(Extension extension) {
+        var cache = cacheManager.getCache(CACHE_SEARCH_ENTRY_JSON);
+        if(cache == null) {
+            return; // cache is not created
+        }
+
+        var includeAllVersionsList = List.of(true, false);
+        for(var includeAllVersions : includeAllVersionsList) {
+            var key = searchEntryJsonCacheKeyGenerator.generate(extension.getId(), includeAllVersions);
+            cache.evictIfPresent(key);
+        }
+    }
+
     public void evictLatestExtensionVersion(Extension extension) {
         var cache = cacheManager.getCache(CACHE_LATEST_EXTENSION_VERSION);
         if(cache != null) {

diff --git a/server/src/main/java/org/eclipse/openvsx/cache/SearchEntryJsonCacheKeyGenerator.java b/server/src/main/java/org/eclipse/openvsx/cache/SearchEntryJsonCacheKeyGenerator.java
@@ -0,0 +1,20 @@
+/** ******************************************************************************
+ * Copyright (c) 2022 Precies. Software Ltd and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ * ****************************************************************************** */
+package org.eclipse.openvsx.cache;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class SearchEntryJsonCacheKeyGenerator {
+
+    public Object generate(long extensionId, boolean includeAllVersions) {
+        return "extensionId=" + extensionId + ",includeAllVersions=" + includeAllVersions;
+    }
+}
diff --git a/server/src/main/java/org/eclipse/openvsx/entities/Extension.java b/server/src/main/java/org/eclipse/openvsx/entities/Extension.java
@@ -62,6 +62,7 @@ public ExtensionSearch toSearch(ExtensionVersion latest) {
         search.name = this.getName();
         search.namespace = this.getNamespace().getName();
         search.extensionId = search.namespace + "." + search.name;
+        search.averageRating = this.getAverageRating();
         search.downloadCount = this.getDownloadCount();
         search.targetPlatforms = this.getVersions().stream()
                 .map(ExtensionVersion::getTargetPlatform)

diff --git a/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java b/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java
@@ -78,7 +78,7 @@ public class SearchEntryJson implements Serializable {
         name = "VersionReference",
         description = "Essential metadata of an extension version"
     )
-    public static class VersionReference {
+    public static class VersionReference implements Serializable {
 
         @Schema(description = "URL to get the full metadata of this version")
         public String url;

diff --git a/server/src/main/java/org/eclipse/openvsx/json/SearchEntryService.java b/server/src/main/java/org/eclipse/openvsx/json/SearchEntryService.java
@@ -0,0 +1,118 @@
+/** ******************************************************************************
+ * Copyright (c) 2022 Precies. Software Ltd and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ * ****************************************************************************** */
+package org.eclipse.openvsx.json;
+
+import org.eclipse.openvsx.cache.CacheService;
+import org.eclipse.openvsx.entities.Extension;
+import org.eclipse.openvsx.entities.ExtensionVersion;
+import org.eclipse.openvsx.repositories.RepositoryService;
+import org.eclipse.openvsx.search.ExtensionSearch;
+import org.eclipse.openvsx.search.SearchUtilService;
+import org.eclipse.openvsx.storage.StorageUtilService;
+import org.eclipse.openvsx.util.UrlUtil;
+import org.eclipse.openvsx.util.VersionService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.elasticsearch.core.SearchHit;
+import org.springframework.stereotype.Component;
+
+import javax.persistence.EntityManager;
+import javax.transaction.Transactional;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.eclipse.openvsx.entities.FileResource.DOWNLOAD;
+import static org.eclipse.openvsx.entities.FileResource.ICON;
+import static org.eclipse.openvsx.util.UrlUtil.createApiUrl;
+
+@Component
+public class SearchEntryService {
+
+    @Autowired
+    EntityManager entityManager;
+
+    @Autowired
+    VersionService versions;
+
+    @Autowired
+    StorageUtilService storageUtil;
+
+    @Autowired
+    SearchUtilService search;
+
+    @Autowired
+    RepositoryService repositories;
+
+    @Autowired
+    CacheService cache;
+
+    @Transactional
+    public SearchEntryJson toJson(SearchHit<ExtensionSearch> searchHit, boolean includeAllVersions) {
+        var searchEntry = cache.getSearchEntryJson(searchHit, includeAllVersions);
+        if(searchEntry != null) {
+            return searchEntry;
+        }
+
+        var extension = getExtension(searchHit);
+        if(extension == null) {
+            return null;
+        }
+
+        var serverUrl = UrlUtil.getBaseUrl();
+        if(includeAllVersions && cache != null) {
+            searchEntry = cache.getSearchEntryJson(searchHit, false);
+        }
+        if(searchEntry == null) {
+            var latest = versions.getLatest(extension, null, false, true);
+            searchEntry = latest.toSearchEntryJson();
+            searchEntry.url = createApiUrl(serverUrl, "api", searchEntry.namespace, searchEntry.name);
+            searchEntry.files = storageUtil.getFileUrls(latest, serverUrl, DOWNLOAD, ICON);
+            cache.putSearchEntryJson(searchEntry, searchHit, false);
+        }
+        if (includeAllVersions) {
+            var activeVersions = repositories.findActiveVersions(extension).toList();
+            var versionUrls = storageUtil.getFileUrls(activeVersions, serverUrl, DOWNLOAD);
+            searchEntry.allVersions = getAllVersionReferences(activeVersions, versionUrls, serverUrl);
+            cache.putSearchEntryJson(searchEntry, searchHit, true);
+        }
+
+        return searchEntry;
+    }
+
+    private Extension getExtension(SearchHit<ExtensionSearch> searchHit) {
+        var searchItem = searchHit.getContent();
+        var extension = entityManager.find(Extension.class, searchItem.id);
+        if (extension == null || !extension.isActive()) {
+            extension = new Extension();
+            extension.setId(searchItem.id);
+            search.removeSearchEntry(extension);
+            return null;
+        }
+
+        return extension;
+    }
+
+    private List<SearchEntryJson.VersionReference> getAllVersionReferences(
+            List<ExtensionVersion> extVersions,
+            Map<Long, Map<String, String>> versionUrls,
+            String serverUrl
+    ) {
+        return extVersions.stream()
+                .sorted(ExtensionVersion.SORT_COMPARATOR)
+                .map(extVersion -> {
+                    var ref = new SearchEntryJson.VersionReference();
+                    ref.version = extVersion.getVersion();
+                    ref.engines = extVersion.getEnginesMap();
+                    ref.url = UrlUtil.createApiVersionUrl(serverUrl, extVersion);
+                    ref.files = versionUrls.get(extVersion.getId());
+                    return ref;
+                }).collect(Collectors.toList());
+    }
+}
diff --git a/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java b/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java
@@ -44,6 +44,14 @@ public class ExtensionSearch implements Serializable {
     @Field(index = false)
     public long timestamp;
 
+    @Nullable
+    @Field(index = false, type = FieldType.Float)
+    public Double averageRating;
+
+    @Nullable
+    @Field(index = false, type = FieldType.Float)
+    public Long reviewCount;
+
     @Nullable
     @Field(index = false, type = FieldType.Float)
     public Double rating;

diff --git a/server/src/main/resources/ehcache.xml b/server/src/main/resources/ehcache.xml
@@ -30,6 +30,16 @@
             <heap unit="entries">1024</heap>
         </resources>
     </cache>
+    <cache alias="search.entry.json">
+        <expiry>
+            <ttl unit="seconds">3600</ttl>
+        </expiry>
+        <resources>
+            <heap unit="entries">1024</heap>
+            <offheap unit="MB">32</offheap>
+            <disk unit="MB">128</disk>
+        </resources>
+    </cache>
     <cache alias="extension.json">
         <expiry>
             <ttl unit="seconds">3600</ttl>