Skip to content

Commit

Permalink
Cache /api/-/search results
Browse files Browse the repository at this point in the history
# Conflicts:
#	server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java
#	server/src/main/resources/ehcache.xml
#	server/src/test/java/org/eclipse/openvsx/AdminAPITest.java
#	server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java
  • Loading branch information
amvanbaren committed Mar 16, 2023
1 parent a874a24 commit 11d525a
Show file tree
Hide file tree
Showing 11 changed files with 209 additions and 81 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ public void updateExtension(Extension extension) {
cache.evictNamespaceDetails(extension);
cache.evictLatestExtensionVersion(extension);
cache.evictExtensionJsons(extension);
cache.evictSearchEntryJsons(extension);

if (extension.getVersions().stream().anyMatch(ExtensionVersion::isActive)) {
// There is at least one active version => activate the extension
Expand Down
97 changes: 18 additions & 79 deletions server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import org.eclipse.openvsx.entities.*;
import org.eclipse.openvsx.json.*;
import org.eclipse.openvsx.repositories.RepositoryService;
import org.eclipse.openvsx.search.ExtensionSearch;
import org.eclipse.openvsx.search.ISearchService;
import org.eclipse.openvsx.search.SearchUtilService;
import org.eclipse.openvsx.storage.StorageUtilService;
Expand All @@ -39,8 +38,6 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.dao.DataIntegrityViolationException;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.retry.annotation.Retryable;
Expand Down Expand Up @@ -82,6 +79,9 @@ public class LocalRegistryService implements IExtensionRegistry {
@Autowired
CacheService cache;

@Autowired
SearchEntryService searchEntries;

@Override
public NamespaceJson getNamespace(String namespaceName) {
var namespace = repositories.findNamespace(namespaceName);
Expand Down Expand Up @@ -211,7 +211,21 @@ public SearchResultJson search(ISearchService.Options options) {
}

var searchHits = search.search(options);
json.extensions = toSearchEntries(searchHits, options);
var extensions = new ArrayList<SearchEntryJson>();
for (var searchHit : searchHits) {
var searchEntry = searchEntries.toJson(searchHit, options.includeAllVersions);
if(searchEntry != null) {
// use averageRating and downloadCount from ElasticSearch response,
// so that cached SearchEntryJson doesn't have to be evicted every time
// averageRating or downloadCount are updated.
var extensionSearch = searchHit.getContent();
searchEntry.averageRating = extensionSearch.averageRating;
searchEntry.downloadCount = extensionSearch.downloadCount;
extensions.add(searchEntry);
}
}

json.extensions = extensions;
json.offset = options.requestedOffset;
json.totalSize = (int) searchHits.getTotalHits();
return json;
Expand Down Expand Up @@ -741,81 +755,6 @@ public ResultJson deleteReview(String namespace, String extensionName) {
return ResultJson.success("Deleted review for " + extension.getNamespace().getName() + "." + extension.getName());
}

private Extension getExtension(SearchHit<ExtensionSearch> searchHit) {
var searchItem = searchHit.getContent();
var extension = entityManager.find(Extension.class, searchItem.id);
if (extension == null || !extension.isActive()) {
extension = new Extension();
extension.setId(searchItem.id);
search.removeSearchEntry(extension);
return null;
}

return extension;
}

private List<SearchEntryJson> toSearchEntries(SearchHits<ExtensionSearch> searchHits, ISearchService.Options options) {
var serverUrl = UrlUtil.getBaseUrl();
var extensions = searchHits.stream()
.map(this::getExtension)
.filter(Objects::nonNull)
.collect(Collectors.toList());

var latestVersions = extensions.stream()
.map(e -> {
var latest = versions.getLatestTrxn(e, null, false, true);
return new AbstractMap.SimpleEntry<>(e.getId(), latest);
})
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

var searchEntries = latestVersions.entrySet().stream()
.map(e -> {
var entry = e.getValue().toSearchEntryJson();
entry.url = createApiUrl(serverUrl, "api", entry.namespace, entry.name);
return new AbstractMap.SimpleEntry<>(e.getKey(), entry);
})
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

var fileUrls = storageUtil.getFileUrls(latestVersions.values(), serverUrl, DOWNLOAD, ICON);
searchEntries.forEach((extensionId, searchEntry) -> searchEntry.files = fileUrls.get(latestVersions.get(extensionId).getId()));
if (options.includeAllVersions) {
var allActiveVersions = repositories.findActiveVersions(extensions).stream()
.sorted(ExtensionVersion.SORT_COMPARATOR)
.collect(Collectors.toList());

var activeVersionsByExtensionId = allActiveVersions.stream()
.collect(Collectors.groupingBy(ev -> ev.getExtension().getId()));

var versionUrls = storageUtil.getFileUrls(allActiveVersions, serverUrl, DOWNLOAD);
for(var extension : extensions) {
var activeVersions = activeVersionsByExtensionId.get(extension.getId());
var searchEntry = searchEntries.get(extension.getId());
searchEntry.allVersions = getAllVersionReferences(activeVersions, versionUrls, serverUrl);
}
}

return extensions.stream()
.map(Extension::getId)
.map(searchEntries::get)
.collect(Collectors.toList());
}

private List<SearchEntryJson.VersionReference> getAllVersionReferences(
List<ExtensionVersion> extVersions,
Map<Long, Map<String, String>> versionUrls,
String serverUrl
) {
Collections.sort(extVersions, ExtensionVersion.SORT_COMPARATOR);
return extVersions.stream().map(extVersion -> {
var ref = new SearchEntryJson.VersionReference();
ref.version = extVersion.getVersion();
ref.engines = extVersion.getEnginesMap();
ref.url = UrlUtil.createApiVersionUrl(serverUrl, extVersion);
ref.files = versionUrls.get(extVersion.getId());
return ref;
}).collect(Collectors.toList());
}

public ExtensionJson toExtensionVersionJson(ExtensionVersion extVersion, String targetPlatform, boolean onlyActive, boolean inTransaction) {
var extension = extVersion.getExtension();
var latest = inTransaction
Expand Down
18 changes: 18 additions & 0 deletions server/src/main/java/org/eclipse/openvsx/cache/CacheService.java
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,13 @@ public class CacheService {

public static final String CACHE_DATABASE_SEARCH = "database.search";
public static final String CACHE_EXTENSION_JSON = "extension.json";
public static final String CACHE_SEARCH_ENTRY_JSON = "search.entry.json";
public static final String CACHE_LATEST_EXTENSION_VERSION = "latest.extension.version";
public static final String CACHE_NAMESPACE_DETAILS_JSON = "namespace.details.json";
public static final String CACHE_AVERAGE_REVIEW_RATING = "average.review.rating";

public static final String GENERATOR_EXTENSION_JSON = "extensionJsonCacheKeyGenerator";
public static final String GENERATOR_SEARCH_ENTRY_JSON = "searchEntryJsonCacheKeyGenerator";
public static final String GENERATOR_LATEST_EXTENSION_VERSION = "latestExtensionVersionCacheKeyGenerator";

@Autowired
Expand All @@ -43,6 +45,9 @@ public class CacheService {
@Autowired
ExtensionJsonCacheKeyGenerator extensionJsonCacheKey;

@Autowired
SearchEntryJsonCacheKeyGenerator searchEntryJsonCacheKeyGenerator;

@Autowired
LatestExtensionVersionCacheKeyGenerator latestExtensionVersionCacheKey;

Expand Down Expand Up @@ -92,6 +97,19 @@ public void evictExtensionJsons(Extension extension) {
}
}

public void evictSearchEntryJsons(Extension extension) {
var cache = cacheManager.getCache(CACHE_SEARCH_ENTRY_JSON);
if(cache == null) {
return; // cache is not created
}

var includeAllVersionsList = List.of(true, false);
for(var includeAllVersions : includeAllVersionsList) {
var key = searchEntryJsonCacheKeyGenerator.generate(extension.getId(), includeAllVersions);
cache.evictIfPresent(key);
}
}

public void evictLatestExtensionVersion(Extension extension) {
var cache = cacheManager.getCache(CACHE_LATEST_EXTENSION_VERSION);
if(cache != null) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/** ******************************************************************************
* Copyright (c) 2022 Precies. Software Ltd and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License v. 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* SPDX-License-Identifier: EPL-2.0
* ****************************************************************************** */
package org.eclipse.openvsx.cache;

import org.eclipse.openvsx.search.ExtensionSearch;
import org.springframework.cache.interceptor.KeyGenerator;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.stereotype.Component;

import java.lang.reflect.Method;

@Component
public class SearchEntryJsonCacheKeyGenerator implements KeyGenerator {

@Override
public Object generate(Object target, Method method, Object... params) {
var searchHit = (SearchHit<?>) params[0];
var includeAllVersions = (boolean) params[1];
if(searchHit.getContent() instanceof ExtensionSearch) {
var extensionSearch = (ExtensionSearch) searchHit.getContent();
return generate(extensionSearch.id, includeAllVersions);
}

return null;
}

public Object generate(long extensionId, boolean includeAllVersions) {
return "extensionId=" + extensionId + ",includeAllVersions=" + includeAllVersions;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ public ExtensionSearch toSearch(ExtensionVersion latest) {
search.name = this.getName();
search.namespace = this.getNamespace().getName();
search.extensionId = search.namespace + "." + search.name;
search.averageRating = this.getAverageRating();
search.downloadCount = this.getDownloadCount();
search.targetPlatforms = this.getVersions().stream()
.map(ExtensionVersion::getTargetPlatform)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public class SearchEntryJson implements Serializable {
name = "VersionReference",
description = "Essential metadata of an extension version"
)
public static class VersionReference {
public static class VersionReference implements Serializable {

@Schema(description = "URL to get the full metadata of this version")
public String url;
Expand Down
106 changes: 106 additions & 0 deletions server/src/main/java/org/eclipse/openvsx/json/SearchEntryService.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
/** ******************************************************************************
* Copyright (c) 2022 Precies. Software Ltd and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License v. 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* SPDX-License-Identifier: EPL-2.0
* ****************************************************************************** */
package org.eclipse.openvsx.json;

import org.eclipse.openvsx.entities.Extension;
import org.eclipse.openvsx.entities.ExtensionVersion;
import org.eclipse.openvsx.repositories.RepositoryService;
import org.eclipse.openvsx.search.ExtensionSearch;
import org.eclipse.openvsx.search.SearchUtilService;
import org.eclipse.openvsx.storage.StorageUtilService;
import org.eclipse.openvsx.util.UrlUtil;
import org.eclipse.openvsx.util.VersionService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.stereotype.Component;

import javax.persistence.EntityManager;
import javax.transaction.Transactional;
import java.util.*;
import java.util.stream.Collectors;

import static org.eclipse.openvsx.cache.CacheService.CACHE_SEARCH_ENTRY_JSON;
import static org.eclipse.openvsx.cache.CacheService.GENERATOR_SEARCH_ENTRY_JSON;
import static org.eclipse.openvsx.entities.FileResource.DOWNLOAD;
import static org.eclipse.openvsx.entities.FileResource.ICON;
import static org.eclipse.openvsx.util.UrlUtil.createApiUrl;

@Component
public class SearchEntryService {

@Autowired
EntityManager entityManager;

@Autowired
VersionService versions;

@Autowired
StorageUtilService storageUtil;

@Autowired
SearchUtilService search;

@Autowired
RepositoryService repositories;

@Transactional
@Cacheable(value = CACHE_SEARCH_ENTRY_JSON, keyGenerator = GENERATOR_SEARCH_ENTRY_JSON)
public SearchEntryJson toJson(SearchHit<ExtensionSearch> searchHit, boolean includeAllVersions) {
var serverUrl = UrlUtil.getBaseUrl();
var extension = getExtension(searchHit);
if(extension == null) {
return null;
}

var latest = versions.getLatest(extension, null, false, true);
var searchEntry = latest.toSearchEntryJson();
searchEntry.url = createApiUrl(serverUrl, "api", searchEntry.namespace, searchEntry.name);
searchEntry.files = storageUtil.getFileUrls(latest, serverUrl, DOWNLOAD, ICON);

if (includeAllVersions) {
var activeVersions = repositories.findActiveVersions(extension).toList();
var versionUrls = storageUtil.getFileUrls(activeVersions, serverUrl, DOWNLOAD);
searchEntry.allVersions = getAllVersionReferences(activeVersions, versionUrls, serverUrl);
}

return searchEntry;
}

private Extension getExtension(SearchHit<ExtensionSearch> searchHit) {
var searchItem = searchHit.getContent();
var extension = entityManager.find(Extension.class, searchItem.id);
if (extension == null || !extension.isActive()) {
extension = new Extension();
extension.setId(searchItem.id);
search.removeSearchEntry(extension);
return null;
}

return extension;
}

private List<SearchEntryJson.VersionReference> getAllVersionReferences(
List<ExtensionVersion> extVersions,
Map<Long, Map<String, String>> versionUrls,
String serverUrl
) {
return extVersions.stream()
.sorted(ExtensionVersion.SORT_COMPARATOR)
.map(extVersion -> {
var ref = new SearchEntryJson.VersionReference();
ref.version = extVersion.getVersion();
ref.engines = extVersion.getEnginesMap();
ref.url = UrlUtil.createApiVersionUrl(serverUrl, extVersion);
ref.files = versionUrls.get(extVersion.getId());
return ref;
}).collect(Collectors.toList());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@ public class ExtensionSearch implements Serializable {
@Field(index = false)
public long timestamp;

@Nullable
@Field(index = false, type = FieldType.Float)
public Double averageRating;

@Nullable
@Field(index = false, type = FieldType.Float)
public Double rating;
Expand Down
10 changes: 10 additions & 0 deletions server/src/main/resources/ehcache.xml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,16 @@
<heap unit="entries">1024</heap>
</resources>
</cache>
<cache alias="search.entry.json">
<expiry>
<ttl unit="seconds">3600</ttl>
</expiry>
<resources>
<heap unit="entries">1024</heap>
<offheap unit="MB">32</offheap>
<disk unit="MB">128</disk>
</resources>
</cache>
<cache alias="extension.json">
<expiry>
<ttl unit="seconds">3600</ttl>
Expand Down
3 changes: 2 additions & 1 deletion server/src/test/java/org/eclipse/openvsx/AdminAPITest.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.eclipse.openvsx.storage.StorageUtilService;
import org.eclipse.openvsx.util.TargetPlatform;
import org.eclipse.openvsx.util.VersionService;
import org.jobrunr.scheduling.JobRequestScheduler;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -67,7 +68,7 @@
ClientRegistrationRepository.class, UpstreamRegistryService.class, GoogleCloudStorageService.class,
AzureBlobStorageService.class, VSCodeIdService.class, AzureDownloadCountService.class,
CacheService.class, PublishExtensionVersionHandler.class, SearchUtilService.class,
EclipseService.class, SimpleMeterRegistry.class
EclipseService.class, SimpleMeterRegistry.class, SearchEntryService.class
})
public class AdminAPITest {

Expand Down
Loading

0 comments on commit 11d525a

Please sign in to comment.