[backend/frontend] Handle Endpoint page filtering (#1553)

openbas-api/src/main/java/io/openbas/rest/asset/endpoint/EndpointApi.java

@@ -1,5 +1,6 @@
 package io.openbas.rest.asset.endpoint;
 
+import io.openbas.aop.LogExecutionTime;
 import io.openbas.asset.EndpointService;
 import io.openbas.database.model.AssetAgentJob;
 import io.openbas.database.model.Endpoint;
@@ -10,18 +11,18 @@
 import io.openbas.database.specification.AssetAgentJobSpecification;
 import io.openbas.database.specification.EndpointSpecification;
 import io.openbas.rest.asset.endpoint.form.EndpointInput;
+import io.openbas.rest.asset.endpoint.form.EndpointOutput;
 import io.openbas.rest.asset.endpoint.form.EndpointRegisterInput;
 import io.openbas.utils.pagination.SearchPaginationInput;
-import jakarta.transaction.Transactional;
 import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.NotNull;
 import lombok.RequiredArgsConstructor;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.domain.Page;
-import org.springframework.data.domain.Pageable;
-import org.springframework.data.jpa.domain.Specification;
 import org.springframework.security.access.annotation.Secured;
 import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.bind.annotation.*;
 
 import java.io.IOException;
@@ -31,9 +32,9 @@
 
 import static io.openbas.database.model.User.ROLE_ADMIN;
 import static io.openbas.database.model.User.ROLE_USER;
+import static io.openbas.database.specification.EndpointSpecification.fromIds;
 import static io.openbas.executors.openbas.OpenBASExecutor.OPENBAS_EXECUTOR_ID;
 import static io.openbas.helper.StreamHelper.iterableToSet;
-import static io.openbas.utils.pagination.PaginationUtils.buildPaginationJPA;
 
 @RequiredArgsConstructor
 @RestController
@@ -44,14 +45,15 @@ public class EndpointApi {
 
   @Value("${info.app.version:unknown}") String version;
   private final EndpointService endpointService;
+  private final EndpointCriteriaBuilderService endpointCriteriaBuilderService;
   private final EndpointRepository endpointRepository;
   private final ExecutorRepository executorRepository;
   private final TagRepository tagRepository;
   private final AssetAgentJobRepository assetAgentJobRepository;
 
   @PostMapping(ENDPOINT_URI)
   @PreAuthorize("isPlanner()")
-  @Transactional(rollbackOn = Exception.class)
+  @Transactional(rollbackFor = Exception.class)
   public Endpoint createEndpoint(@Valid @RequestBody final EndpointInput input) {
     Endpoint endpoint = new Endpoint();
     endpoint.setUpdateAttributes(input);
@@ -63,7 +65,7 @@ public Endpoint createEndpoint(@Valid @RequestBody final EndpointInput input) {
 
   @Secured(ROLE_ADMIN)
   @PostMapping(ENDPOINT_URI + "/register")
-  @Transactional(rollbackOn = Exception.class)
+  @Transactional(rollbackFor = Exception.class)
   public Endpoint upsertEndpoint(@Valid @RequestBody final EndpointRegisterInput input) throws IOException {
     Optional<Endpoint> optionalEndpoint = this.endpointService.findByExternalReference(input.getExternalReference());
     Endpoint endpoint;
@@ -99,14 +101,14 @@ public Endpoint upsertEndpoint(@Valid @RequestBody final EndpointRegisterInput i
 
   @GetMapping(ENDPOINT_URI + "/jobs/{endpointExternalReference}")
   @PreAuthorize("isPlanner()")
-  @Transactional(rollbackOn = Exception.class)
+  @Transactional(rollbackFor = Exception.class)
   public List<AssetAgentJob> getEndpointJobs(@PathVariable @NotBlank final String endpointExternalReference) {
     return this.assetAgentJobRepository.findAll(AssetAgentJobSpecification.forEndpoint(endpointExternalReference));
   }
 
   @PostMapping(ENDPOINT_URI + "/jobs/{assetAgentJobId}")
   @PreAuthorize("isPlanner()")
-  @Transactional(rollbackOn = Exception.class)
+  @Transactional(rollbackFor = Exception.class)
   public void cleanupAssetAgentJob(@PathVariable @NotBlank final String assetAgentJobId) {
     this.assetAgentJobRepository.deleteById(assetAgentJobId);
   }
@@ -123,21 +125,22 @@ public Endpoint endpoint(@PathVariable @NotBlank final String endpointId) {
     return this.endpointService.endpoint(endpointId);
   }
 
+  @LogExecutionTime
   @PostMapping(ENDPOINT_URI + "/search")
-  public Page<Endpoint> endpoints(@RequestBody @Valid SearchPaginationInput searchPaginationInput) {
-    return buildPaginationJPA(
-            (Specification<Endpoint> specification, Pageable pageable) -> this.endpointRepository.findAll(
-                    EndpointSpecification.findEndpointsForInjection().and(specification),
-                    pageable
-            ),
-            searchPaginationInput,
-            Endpoint.class
-    );
+  public Page<EndpointOutput> endpoints(@RequestBody @Valid SearchPaginationInput searchPaginationInput) {
+    return this.endpointCriteriaBuilderService.endpointPagination(searchPaginationInput);
+  }
+
+  @PostMapping(ENDPOINT_URI + "/find")
+  @PreAuthorize("isPlanner()")
+  @Transactional(readOnly = true)
+  public List<EndpointOutput> findEndpoints(@RequestBody @Valid @NotNull final List<String> endpointIds) {
+    return this.endpointCriteriaBuilderService.find(fromIds(endpointIds));
   }
 
   @PutMapping(ENDPOINT_URI + "/{endpointId}")
   @PreAuthorize("isPlanner()")
-  @Transactional(rollbackOn = Exception.class)
+  @Transactional(rollbackFor = Exception.class)
   public Endpoint updateEndpoint(
       @PathVariable @NotBlank final String endpointId,
       @Valid @RequestBody final EndpointInput input) {
@@ -151,7 +154,7 @@ public Endpoint updateEndpoint(
 
   @DeleteMapping(ENDPOINT_URI + "/{endpointId}")
   @PreAuthorize("isPlanner()")
-  @Transactional(rollbackOn = Exception.class)
+  @Transactional(rollbackFor = Exception.class)
   public void deleteEndpoint(@PathVariable @NotBlank final String endpointId) {
     this.endpointService.deleteEndpoint(endpointId);
   }

openbas-api/src/main/java/io/openbas/rest/asset/endpoint/EndpointCriteriaBuilderService.java

@@ -0,0 +1,105 @@
+package io.openbas.rest.asset.endpoint;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.openbas.database.model.Endpoint;
+import io.openbas.rest.asset.endpoint.form.EndpointOutput;
+import io.openbas.utils.pagination.SearchPaginationInput;
+import jakarta.annotation.Resource;
+import jakarta.persistence.EntityManager;
+import jakarta.persistence.PersistenceContext;
+import jakarta.persistence.Tuple;
+import jakarta.persistence.TypedQuery;
+import jakarta.persistence.criteria.*;
+import lombok.RequiredArgsConstructor;
+import org.jetbrains.annotations.NotNull;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageImpl;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.jpa.domain.Specification;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+import static io.openbas.database.criteria.GenericCriteria.countQuery;
+import static io.openbas.rest.asset.endpoint.EndpointQueryHelper.execution;
+import static io.openbas.rest.asset.endpoint.EndpointQueryHelper.select;
+import static io.openbas.utils.pagination.PaginationUtils.buildPaginationCriteriaBuilder;
+import static io.openbas.utils.pagination.SortUtilsCriteriaBuilder.toSortCriteriaBuilder;
+
+@RequiredArgsConstructor
+@Service
+public class EndpointCriteriaBuilderService {
+
+    @Resource
+    protected ObjectMapper mapper;
+
+    @PersistenceContext
+    private EntityManager entityManager;
+
+    public Page<EndpointOutput> endpointPagination(
+            @NotNull SearchPaginationInput searchPaginationInput) {
+        return buildPaginationCriteriaBuilder(
+                this::paginate,
+                searchPaginationInput,
+                Endpoint.class
+        );
+    }
+
+    public List<EndpointOutput> find(Specification<Endpoint> specification) {
+        CriteriaBuilder cb = this.entityManager.getCriteriaBuilder();
+
+        CriteriaQuery<Tuple> cq = cb.createTupleQuery();
+        Root<Endpoint> root = cq.from(Endpoint.class);
+        select(cb, cq, root);
+
+        if (specification != null) {
+            Predicate predicate = specification.toPredicate(root, cq, cb);
+            if (predicate != null) {
+                cq.where(predicate);
+            }
+        }
+
+        TypedQuery<Tuple> query = entityManager.createQuery(cq);
+        return execution(query, this.mapper);
+    }
+
+    // -- PRIVATE --
+
+    private Page<EndpointOutput> paginate(
+            Specification<Endpoint> specification,
+            Specification<Endpoint> specificationCount,
+            Pageable pageable) {
+        CriteriaBuilder cb = this.entityManager.getCriteriaBuilder();
+
+        CriteriaQuery<Tuple> cq = cb.createTupleQuery();
+        Root<Endpoint> endpointRoot = cq.from(Endpoint.class);
+        select(cb, cq, endpointRoot);
+
+        // -- Specification --
+        if (specification != null) {
+            Predicate predicate = specification.toPredicate(endpointRoot, cq, cb);
+            if (predicate != null) {
+                cq.where(predicate);
+            }
+        }
+
+        // -- Sorting --
+        List<Order> orders = toSortCriteriaBuilder(cb, endpointRoot, pageable.getSort());
+        cq.orderBy(orders);
+
+        // Type Query
+        TypedQuery<Tuple> query = entityManager.createQuery(cq);
+
+        // -- Pagination --
+        query.setFirstResult((int) pageable.getOffset());
+        query.setMaxResults(pageable.getPageSize());
+
+        // -- EXECUTION --
+        List<EndpointOutput> endpoints = execution(query, this.mapper);
+
+        // -- Count Query --
+        Long total = countQuery(cb, this.entityManager, Endpoint.class, specificationCount);
+
+        return new PageImpl<>(endpoints, pageable, total);
+    }
+}

openbas-api/src/main/java/io/openbas/rest/asset/endpoint/EndpointQueryHelper.java

@@ -0,0 +1,71 @@
+package io.openbas.rest.asset.endpoint;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.openbas.database.model.*;
+import io.openbas.rest.asset.endpoint.form.EndpointOutput;
+import jakarta.persistence.Tuple;
+import jakarta.persistence.TypedQuery;
+import jakarta.persistence.criteria.*;
+
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static io.openbas.database.model.Asset.ACTIVE_THRESHOLD;
+import static io.openbas.utils.JpaUtils.createJoinArrayAggOnId;
+import static io.openbas.utils.JpaUtils.createLeftJoin;
+import static java.time.Instant.now;
+
+public class EndpointQueryHelper {
+
+    private EndpointQueryHelper() {}
+
+    // -- SELECT --
+
+    public static void select(CriteriaBuilder cb, CriteriaQuery<Tuple> cq, Root<Endpoint> endpointRoot) {
+        // Array aggregations
+        Join<Endpoint, Executor> endpointExecutorJoin = createLeftJoin(endpointRoot, "executor");
+        Expression<String[]> tagIdsExpression = createJoinArrayAggOnId(cb, endpointRoot, "tags");
+
+        // Multiselect
+        cq.multiselect(
+            endpointRoot.get("id").alias("asset_id"),
+            endpointRoot.get("name").alias("asset_name"),
+            endpointExecutorJoin.get("id").alias("asset_executor"),
+            endpointRoot.get("lastSeen").alias("asset_last_seen"),
+            endpointRoot.get("platform").alias("endpoint_platform"),
+            endpointRoot.get("arch").alias("endpoint_arch"),
+            tagIdsExpression.alias("asset_tags")
+        ).distinct(true);
+
+        // Group by
+        cq.groupBy(Collections.singletonList(
+                endpointRoot.get("id")
+        ));
+    }
+
+    // -- EXECUTION --
+
+    public static List<EndpointOutput> execution(TypedQuery<Tuple> query, ObjectMapper mapper) {
+        return query.getResultList()
+            .stream()
+            .map(tuple -> (EndpointOutput) EndpointOutput.builder()
+                .id(tuple.get("asset_id", String.class))
+                .name(tuple.get("asset_name", String.class))
+                .executor(tuple.get("asset_executor", String.class))
+                .active(isActive(tuple.get("asset_last_seen", Instant.class)))
+                .tags(Arrays.stream(tuple.get("asset_tags", String[].class)).collect(Collectors.toSet()))
+                .platform(tuple.get("endpoint_platform", Endpoint.PLATFORM_TYPE.class))
+                .arch(tuple.get("endpoint_arch", Endpoint.PLATFORM_ARCH.class))
+                .build())
+            .toList();
+    }
+
+    private static boolean isActive(Instant lastSeen) {
+        return Optional.ofNullable(lastSeen)
+            .map(last -> (now().toEpochMilli() - last.toEpochMilli()) < ACTIVE_THRESHOLD).orElse(false);
+    }
+}

openbas-api/src/main/java/io/openbas/rest/asset/endpoint/form/EndpointOutput.java

@@ -0,0 +1,22 @@
+package io.openbas.rest.asset.endpoint.form;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.openbas.database.model.Endpoint;
+import io.openbas.rest.asset.form.AssetOutput;
+import jakarta.validation.constraints.NotNull;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.SuperBuilder;
+
+@EqualsAndHashCode(callSuper = true)
+@Data
+@SuperBuilder
+public class EndpointOutput extends AssetOutput {
+    @NotNull
+    @JsonProperty("endpoint_platform")
+    private Endpoint.PLATFORM_TYPE platform;
+
+    @NotNull
+    @JsonProperty("endpoint_arch")
+    private Endpoint.PLATFORM_ARCH arch;
+}

openbas-api/src/main/java/io/openbas/rest/asset/form/AssetOutput.java

@@ -0,0 +1,29 @@
+package io.openbas.rest.asset.form;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.constraints.NotBlank;
+import lombok.Data;
+import lombok.experimental.SuperBuilder;
+
+import java.util.Set;
+
+@SuperBuilder
+@Data
+public abstract class AssetOutput {
+    @NotBlank
+    @JsonProperty("asset_id")
+    private String id;
+
+    @NotBlank
+    @JsonProperty("asset_name")
+    private String name;
+
+    @JsonProperty("asset_executor")
+    private String executor;
+
+    @JsonProperty("asset_tags")
+    private Set<String> tags;
+
+    @JsonProperty("asset_active")
+    private boolean active;
+}

openbas-api/src/main/java/io/openbas/rest/asset_group/AssetGroupApi.java

@@ -62,8 +62,8 @@ public Page<AssetGroupOutput> assetGroups(@RequestBody @Valid SearchPaginationIn
   @PostMapping(ASSET_GROUP_URI + "/find")
   @PreAuthorize("isObserver()")
   @Transactional(readOnly = true)
-  @Tracing(name = "Find teams", layer = "api", operation = "POST")
-  public List<AssetGroupOutput> findTeams(@RequestBody @Valid @NotNull final List<String> assetGroupIds) {
+  @Tracing(name = "Find asset groups", layer = "api", operation = "POST")
+  public List<AssetGroupOutput> findAssetGroups(@RequestBody @Valid @NotNull final List<String> assetGroupIds) {
     return this.assetGroupCriteriaBuilderService.find(fromIds(assetGroupIds));
   }