From daf4572048e4d94fe56259ae2c75778fb93a75f8 Mon Sep 17 00:00:00 2001 From: Vojtech Knaisl Date: Wed, 29 Jul 2020 17:18:24 +0200 Subject: [PATCH 01/12] Update JDK in readme, fix schema.org URL in pom.xml --- README.md | 27 +-------------------------- pom.xml | 2 +- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index b28c4893f..251a92b60 100644 --- a/README.md +++ b/README.md @@ -20,35 +20,10 @@ More information about FDP and how to deploy can be found at [FDP Deployment Doc **Stack:** - - **Java** (recommended JDK 11) + - **Java** (recommended JDK 14) - **Maven** (recommended 3.2.5 or higher) - **Docker** (recommended 17.09.0-ce or higher) - *for build of production image* -**Additional libraries:** - -1. Install `fairmetadata4j` - - ```bash - $ git clone https://github.com/FAIRDataTeam/fairmetadata4j - $ cd fairmetadata4j - $ mvn install - ``` -2. Install `spring-rdf-migration` - - ``` - $ git clone https://github.com/FAIRDataTeam/spring-rdf-migration.git - $ cd spring-rdf-migration - $ mvn install - ``` - -3. Install `spring-security-acl-mongodb` - - ``` - $ git clone https://github.com/FAIRDataTeam/spring-security-acl-mongodb - $ cd spring-security-acl-mongodb - $ mvn install - ``` - ### Build & Run Run these commands from the root of the project diff --git a/pom.xml b/pom.xml index 33a35cc70..4919f22c3 100644 --- a/pom.xml +++ b/pom.xml @@ -339,7 +339,7 @@ fdp - https://raw.githubusercontent.com/schemaorg/schemaorg/master/data/releases/3.4/schema.ttl + https://raw.githubusercontent.com/schemaorg/schemaorg/main/data/releases/3.4/schema.ttl http://schema.org/ schemaOrg From 38a5fbdf3bc988447beda2c5daaa1938f15e5408 Mon Sep 17 00:00:00 2001 From: Vojtech Knaisl Date: Tue, 29 Sep 2020 09:36:04 +0200 Subject: [PATCH 02/12] Move index to FDP --- .../exception/ExceptionControllerAdvice.java | 30 +-- .../api/controller/index/AdminController.java | 70 ++++++ .../index/IndexEntryController.java | 75 ++++++ .../api/controller/index/PingController.java | 64 +++++ .../controller/search/SearchController.java | 29 +++ .../api/dto/config/BootstrapConfigDTO.java | 2 + .../api/dto/index/entry/IndexEntryDTO.java | 54 ++++ .../dto/index/entry/IndexEntryDetailDTO.java | 67 +++++ .../dto/index/entry/IndexEntryInfoDTO.java | 38 +++ .../dto/index/entry/IndexEntryStateDTO.java | 36 +++ .../api/dto/index/event/EventDTO.java | 51 ++++ .../api/dto/index/ping/PingDTO.java | 43 ++++ .../dto/index/webhook/WebhookPayloadDTO.java | 37 +++ .../api/dto/search/SearchQueryDTO.java | 19 ++ .../api/dto/search/SearchResultDTO.java | 25 ++ .../fairdatapoint/config/AsyncConfig.java | 34 +++ .../fairdatapoint/config/IndexConfig.java | 55 ++++ .../fairdatapoint/config/SecurityConfig.java | 3 + .../development/MigrationRunner.java | 10 + .../index/entry/IndexEntryMigration.java | 51 ++++ .../index/entry/data/IndexEntryFixtures.java | 108 ++++++++ .../index/event/EventMigration.java | 39 +++ .../mongo/repository/EventRepository.java | 44 ++++ .../repository/IndexEntryRepository.java | 54 ++++ .../mongo/repository/WebhookRepository.java | 33 +++ .../common/AbstractMetadataRepository.java | 26 +- .../repository/common/MetadataRepository.java | 8 +- .../entity/index/config/EventsConfig.java | 38 +++ .../entity/index/entry/IndexEntry.java | 81 ++++++ .../entity/index/entry/IndexEntryState.java | 35 +++ .../index/entry/RepositoryMetadata.java | 48 ++++ .../entity/index/event/AdminTrigger.java | 36 +++ .../entity/index/event/Event.java | 126 ++++++++++ .../entity/index/event/EventType.java | 31 +++ .../entity/index/event/IncomingPing.java | 36 +++ .../entity/index/event/MetadataRetrieval.java | 38 +++ .../entity/index/event/WebhookPing.java | 38 +++ .../entity/index/event/WebhookTrigger.java | 43 ++++ .../IncorrectPingFormatException.java | 32 +++ .../index/exception/IndexException.java | 44 ++++ .../index/exception/RateLimitException.java | 32 +++ .../entity/index/http/Exchange.java | 49 ++++ .../entity/index/http/ExchangeDirection.java | 28 +++ .../entity/index/http/ExchangeState.java | 31 +++ .../entity/index/http/Request.java | 77 ++++++ .../entity/index/http/Response.java | 50 ++++ .../entity/index/webhook/Webhook.java | 63 +++++ .../entity/index/webhook/WebhookEvent.java | 33 +++ .../entity/search/SearchResult.java | 21 ++ .../entity/search/SearchResultRelation.java | 18 ++ .../service/config/ConfigService.java | 6 +- .../service/index/entry/IndexEntryMapper.java | 85 +++++++ .../index/entry/IndexEntryService.java | 126 ++++++++++ .../service/index/event/EventMapper.java | 56 +++++ .../service/index/event/EventService.java | 235 ++++++++++++++++++ .../index/event/IncomingPingUtils.java | 77 ++++++ .../index/event/MetadataRetrievalUtils.java | 190 ++++++++++++++ .../service/index/webhook/WebhookMapper.java | 46 ++++ .../service/index/webhook/WebhookService.java | 145 +++++++++++ .../service/index/webhook/WebhookUtils.java | 101 ++++++++ .../service/search/SearchService.java | 46 ++++ ...itional-spring-configuration-metadata.json | 30 +++ src/main/resources/application.yml | 21 ++ .../common/findEntityByLiteral.sparql | 9 + .../index/admin/List_Trigger_POST.java | 186 ++++++++++++++ .../acceptance/index/entry/List_All_GET.java | 137 ++++++++++ .../acceptance/index/entry/List_GET.java | 226 +++++++++++++++++ .../acceptance/index/entry/List_Info_GET.java | 81 ++++++ .../acceptance/index/ping/List_POST.java | 176 +++++++++++++ .../acceptance/search/List_POST.java | 52 ++++ .../fairdatapoint/utils/CustomPageImpl.java | 62 +++++ .../utils/TestIndexEntryFixtures.java | 70 ++++++ src/test/resources/application-testing.yml | 11 +- 73 files changed, 4273 insertions(+), 34 deletions(-) create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/controller/index/AdminController.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/controller/index/IndexEntryController.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/controller/index/PingController.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/controller/search/SearchController.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryDTO.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryDetailDTO.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryInfoDTO.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryStateDTO.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/dto/index/event/EventDTO.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/dto/index/ping/PingDTO.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/dto/index/webhook/WebhookPayloadDTO.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchQueryDTO.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchResultDTO.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/config/AsyncConfig.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/config/IndexConfig.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/entry/IndexEntryMigration.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/entry/data/IndexEntryFixtures.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/event/EventMigration.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/database/mongo/repository/EventRepository.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/database/mongo/repository/IndexEntryRepository.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/database/mongo/repository/WebhookRepository.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/config/EventsConfig.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/entry/IndexEntry.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/entry/IndexEntryState.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/entry/RepositoryMetadata.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/event/AdminTrigger.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/event/Event.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/event/EventType.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/event/IncomingPing.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/event/MetadataRetrieval.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/event/WebhookPing.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/event/WebhookTrigger.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/exception/IncorrectPingFormatException.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/exception/IndexException.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/exception/RateLimitException.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/http/Exchange.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/http/ExchangeDirection.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/http/ExchangeState.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/http/Request.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/http/Response.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/webhook/Webhook.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/index/webhook/WebhookEvent.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResult.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResultRelation.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryMapper.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryService.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/index/event/EventMapper.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/index/event/EventService.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/index/event/IncomingPingUtils.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/index/event/MetadataRetrievalUtils.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookMapper.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookService.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookUtils.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java create mode 100644 src/main/resources/nl/dtls/fairdatapoint/database/rdf/repository/common/findEntityByLiteral.sparql create mode 100644 src/test/java/nl/dtls/fairdatapoint/acceptance/index/admin/List_Trigger_POST.java create mode 100644 src/test/java/nl/dtls/fairdatapoint/acceptance/index/entry/List_All_GET.java create mode 100644 src/test/java/nl/dtls/fairdatapoint/acceptance/index/entry/List_GET.java create mode 100644 src/test/java/nl/dtls/fairdatapoint/acceptance/index/entry/List_Info_GET.java create mode 100644 src/test/java/nl/dtls/fairdatapoint/acceptance/index/ping/List_POST.java create mode 100644 src/test/java/nl/dtls/fairdatapoint/acceptance/search/List_POST.java create mode 100644 src/test/java/nl/dtls/fairdatapoint/utils/CustomPageImpl.java create mode 100644 src/test/java/nl/dtls/fairdatapoint/utils/TestIndexEntryFixtures.java diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/exception/ExceptionControllerAdvice.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/exception/ExceptionControllerAdvice.java index 8997346cf..086b7d3aa 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/controller/exception/ExceptionControllerAdvice.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/exception/ExceptionControllerAdvice.java @@ -30,12 +30,14 @@ import lombok.extern.slf4j.Slf4j; import nl.dtls.fairdatapoint.api.dto.error.ErrorDTO; import nl.dtls.fairdatapoint.entity.exception.*; +import nl.dtls.fairdatapoint.entity.index.exception.IndexException; import nl.dtls.fairdatapoint.service.metadata.exception.MetadataServiceException; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Model; import org.eclipse.rdf4j.rio.RDFFormat; import org.eclipse.rdf4j.rio.Rio; import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.AccessDeniedException; import org.springframework.security.authentication.BadCredentialsException; import org.springframework.web.bind.annotation.ControllerAdvice; @@ -112,29 +114,9 @@ public ErrorDTO handleInternalServerError(Exception e) { return new ErrorDTO(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage()); } -// @ExceptionHandler(MethodArgumentNotValidException.class) -// @ResponseStatus(HttpStatus.BAD_REQUEST) -// @ResponseBody -// public ErrorDTO processValidationError(MethodArgumentNotValidException ex) { -// BindingResult result = ex.getBindingResult(); -// FieldError error = result.getFieldError(); -// return processFieldError(error); -// } -// -// private ErrorDTO processFieldError(FieldError error) { -// ErrorResponse errorResponse = new ErrorResponse(); -// if (error != null) { -// Locale currentLocale = LocaleContextHolder.getLocale(); -//// errorResponse.setMessage(error.getDefaultMessage()); -// } -// return new ErrorDTO(HttpStatus.BAD_REQUEST, error.getDefaultMessage()); -// } -// -// @ExceptionHandler(JsonMappingException.class) -// @ResponseStatus(HttpStatus.BAD_REQUEST) -// @ResponseBody -// public ErrorDTO processValidationError(JsonMappingException ex) { -// return null; -// } + @ExceptionHandler(IndexException.class) + public ResponseEntity handleIndexException(IndexException exception) { + return new ResponseEntity<>(exception.getErrorDTO(), exception.getStatus()); + } } diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/index/AdminController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/index/AdminController.java new file mode 100644 index 000000000..52ea7be71 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/index/AdminController.java @@ -0,0 +1,70 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.api.controller.index; + +import io.swagger.annotations.ApiOperation; +import nl.dtls.fairdatapoint.entity.index.event.Event; +import nl.dtls.fairdatapoint.service.index.event.EventService; +import nl.dtls.fairdatapoint.service.index.webhook.WebhookService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; +import java.util.UUID; + +@RestController +@RequestMapping("/index/admin") +public class AdminController { + private static final Logger logger = LoggerFactory.getLogger(PingController.class); + + @Autowired + private EventService eventService; + + @Autowired + private WebhookService webhookService; + + @ApiOperation(value = "trigger", hidden = true) + @PostMapping("/trigger") + @PreAuthorize("hasRole('ADMIN')") + @ResponseStatus(HttpStatus.NO_CONTENT) + public void triggerMetadataRetrieve(@RequestParam(required = false) String clientUrl, HttpServletRequest request) { + logger.info("Received ping from {}", request.getRemoteAddr()); + final Event event = eventService.acceptAdminTrigger(request, clientUrl); + webhookService.triggerWebhooks(event); + eventService.triggerMetadataRetrieval(event); + } + + @ApiOperation(value = "ping webhook", hidden = true) + @PostMapping("/ping-webhook") + @PreAuthorize("hasRole('ADMIN')") + @ResponseStatus(HttpStatus.NO_CONTENT) + public void webhookPing(@RequestParam(required = true) UUID webhook, HttpServletRequest request) { + logger.info("Received webhook {} ping trigger from {}", webhook, request.getRemoteAddr()); + final Event event = webhookService.handleWebhookPing(request, webhook); + webhookService.triggerWebhooks(event); + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/index/IndexEntryController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/index/IndexEntryController.java new file mode 100644 index 000000000..55ea3ac84 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/index/IndexEntryController.java @@ -0,0 +1,75 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.api.controller.index; + +import nl.dtls.fairdatapoint.api.dto.index.entry.IndexEntryDTO; +import nl.dtls.fairdatapoint.api.dto.index.entry.IndexEntryDetailDTO; +import nl.dtls.fairdatapoint.api.dto.index.entry.IndexEntryInfoDTO; +import nl.dtls.fairdatapoint.service.index.entry.IndexEntryMapper; +import nl.dtls.fairdatapoint.service.index.entry.IndexEntryService; +import nl.dtls.fairdatapoint.service.index.event.EventService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.web.bind.annotation.*; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +@RestController +@RequestMapping("/index/entries") +public class IndexEntryController { + + @Autowired + private IndexEntryService service; + + @Autowired + private EventService eventService; + + @Autowired + private IndexEntryMapper mapper; + + @GetMapping("") + public Page getEntriesPage(Pageable pageable, + @RequestParam(required = false, defaultValue = "") String state) { + return service.getEntriesPage(pageable, state).map(mapper::toDTO); + } + + @RequestMapping(value = "/{uuid}", method = RequestMethod.GET) + public Optional getEntry(@PathVariable final String uuid) { + return service.getEntry(uuid).map(entry -> mapper.toDetailDTO(entry, + eventService.getEvents(entry.getUuid()))); + } + + @GetMapping("/all") + public List getEntriesAll() { + return StreamSupport.stream(service.getAllEntries().spliterator(), true).map(mapper::toDTO).collect(Collectors.toList()); + } + + @GetMapping("/info") + public IndexEntryInfoDTO getEntriesInfo() { + return service.getEntriesInfo(); + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/index/PingController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/index/PingController.java new file mode 100644 index 000000000..2118c937f --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/index/PingController.java @@ -0,0 +1,64 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.api.controller.index; + +import io.swagger.annotations.ApiOperation; +import nl.dtls.fairdatapoint.api.dto.index.ping.PingDTO; +import nl.dtls.fairdatapoint.entity.index.event.Event; +import nl.dtls.fairdatapoint.service.index.event.EventService; +import nl.dtls.fairdatapoint.service.index.webhook.WebhookService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; +import javax.validation.Valid; + +@RestController +@RequestMapping("/index") +public class PingController { + private static final Logger logger = LoggerFactory.getLogger(PingController.class); + + @Autowired + private EventService eventService; + + @Autowired + private WebhookService webhookService; + + @ApiOperation( + value = "Ping payload with FAIR Data Point info", + notes = "Inform about running FAIR Data Point. It is expected to send pings regularly (at least weekly). " + + "There is a rate limit set both per single IP within a period of time and per URL in message." + ) + @RequestMapping(method = RequestMethod.POST) + @ResponseStatus(HttpStatus.NO_CONTENT) + public void receivePing(@RequestBody @Valid PingDTO reqDto, HttpServletRequest request) { + logger.info("Received ping from {}", request.getRemoteAddr()); + final Event event = eventService.acceptIncomingPing(reqDto, request); + logger.info("Triggering metadata retrieval for {}", event.getRelatedTo().getClientUrl()); + eventService.triggerMetadataRetrieval(event); + webhookService.triggerWebhooks(event); + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/search/SearchController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/search/SearchController.java new file mode 100644 index 000000000..59850a5be --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/search/SearchController.java @@ -0,0 +1,29 @@ +package nl.dtls.fairdatapoint.api.controller.search; + +import nl.dtls.fairdatapoint.api.dto.search.SearchQueryDTO; +import nl.dtls.fairdatapoint.api.dto.search.SearchResultDTO; +import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; +import nl.dtls.fairdatapoint.service.search.SearchService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.validation.Valid; +import java.util.List; + +@RestController +@RequestMapping("/search") +public class SearchController { + + @Autowired + private SearchService searchService; + + @PostMapping + public ResponseEntity> search(@RequestBody @Valid SearchQueryDTO reqDto) throws MetadataRepositoryException { + return ResponseEntity.ok(searchService.search(reqDto)); + } + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/config/BootstrapConfigDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/config/BootstrapConfigDTO.java index 6de50027c..0c7f21edd 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/dto/config/BootstrapConfigDTO.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/config/BootstrapConfigDTO.java @@ -40,4 +40,6 @@ public class BootstrapConfigDTO { protected List resourceDefinitions; + protected boolean index; + } diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryDTO.java new file mode 100644 index 000000000..b891d7d27 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryDTO.java @@ -0,0 +1,54 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.api.dto.index.entry; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.hibernate.validator.constraints.URL; + +import javax.validation.constraints.NotNull; + +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +public class IndexEntryDTO { + + @NotNull + private String uuid; + + @NotNull + @URL + private String clientUrl; + + @NotNull + private IndexEntryStateDTO state; + + @NotNull + private String registrationTime; + + @NotNull + private String modificationTime; +} diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryDetailDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryDetailDTO.java new file mode 100644 index 000000000..28583103e --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryDetailDTO.java @@ -0,0 +1,67 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.api.dto.index.entry; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import nl.dtls.fairdatapoint.api.dto.index.event.EventDTO; +import nl.dtls.fairdatapoint.entity.index.entry.RepositoryMetadata; +import org.hibernate.validator.constraints.URL; + +import javax.validation.constraints.NotNull; +import java.util.List; + +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +public class IndexEntryDetailDTO { + + @NotNull + private String uuid; + + @NotNull + @URL + private String clientUrl; + + @NotNull + private IndexEntryStateDTO state; + + @NotNull + private RepositoryMetadata currentMetadata; + + @NotNull + private List events; + + @NotNull + private String registrationTime; + + @NotNull + private String modificationTime; + + @NotNull + private String lastRetrievalTime; + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryInfoDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryInfoDTO.java new file mode 100644 index 000000000..0799270ef --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryInfoDTO.java @@ -0,0 +1,38 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.api.dto.index.entry; + +import lombok.*; + +import java.util.Map; + +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +@EqualsAndHashCode +public class IndexEntryInfoDTO { + + private Map entriesCount; + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryStateDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryStateDTO.java new file mode 100644 index 000000000..6890aea46 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/entry/IndexEntryStateDTO.java @@ -0,0 +1,36 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.api.dto.index.entry; + +public enum IndexEntryStateDTO { + + UNKNOWN, + + ACTIVE, + + INACTIVE, + + UNREACHABLE, + + INVALID +} diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/index/event/EventDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/event/EventDTO.java new file mode 100644 index 000000000..0bae018af --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/event/EventDTO.java @@ -0,0 +1,51 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.api.dto.index.event; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import nl.dtls.fairdatapoint.entity.index.event.EventType; + +import javax.validation.constraints.NotNull; +import java.util.UUID; + +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +public class EventDTO { + + @NotNull + private UUID uuid; + + @NotNull + private EventType type; + + @NotNull + private String created; + + private String finished; + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/index/ping/PingDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/ping/PingDTO.java new file mode 100644 index 000000000..c14d82126 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/ping/PingDTO.java @@ -0,0 +1,43 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.api.dto.index.ping; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.hibernate.validator.constraints.URL; + +import javax.validation.constraints.NotNull; + +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +public class PingDTO { + + @NotNull + @URL + private String clientUrl; + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/index/webhook/WebhookPayloadDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/webhook/WebhookPayloadDTO.java new file mode 100644 index 000000000..bd2a68200 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/index/webhook/WebhookPayloadDTO.java @@ -0,0 +1,37 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.api.dto.index.webhook; + +import lombok.Data; +import lombok.NoArgsConstructor; +import nl.dtls.fairdatapoint.entity.index.webhook.WebhookEvent; + +@Data +@NoArgsConstructor +public class WebhookPayloadDTO { + private WebhookEvent event; + private String uuid; + private String clientUrl; + private String timestamp; + private String secret; +} diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchQueryDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchQueryDTO.java new file mode 100644 index 000000000..cd5a0723b --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchQueryDTO.java @@ -0,0 +1,19 @@ +package nl.dtls.fairdatapoint.api.dto.search; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import javax.validation.constraints.NotNull; + +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +public class SearchQueryDTO { + + @NotNull + private String q; + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchResultDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchResultDTO.java new file mode 100644 index 000000000..aa43503b9 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchResultDTO.java @@ -0,0 +1,25 @@ +package nl.dtls.fairdatapoint.api.dto.search; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import nl.dtls.fairdatapoint.entity.search.SearchResultRelation; + +import java.util.List; + +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +public class SearchResultDTO { + + private String uri; + + private String title; + + private String description; + + private List relations; + +} \ No newline at end of file diff --git a/src/main/java/nl/dtls/fairdatapoint/config/AsyncConfig.java b/src/main/java/nl/dtls/fairdatapoint/config/AsyncConfig.java new file mode 100644 index 000000000..716504dcc --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/config/AsyncConfig.java @@ -0,0 +1,34 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.config; + +import nl.dtls.fairdatapoint.Profiles; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.scheduling.annotation.EnableAsync; + +@Configuration +@EnableAsync +@Profile(Profiles.NON_TESTING) +public class AsyncConfig { +} diff --git a/src/main/java/nl/dtls/fairdatapoint/config/IndexConfig.java b/src/main/java/nl/dtls/fairdatapoint/config/IndexConfig.java new file mode 100644 index 000000000..7a4e43c79 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/config/IndexConfig.java @@ -0,0 +1,55 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.config; + +import nl.dtls.fairdatapoint.entity.index.config.EventsConfig; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Scope; + +import java.time.Duration; + +@Configuration +public class IndexConfig { + + @Bean + @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON) + public EventsConfig eventsConfig( + @Value("${fdp-index.events.retrieval.rateLimitWait:PT10M}") String cfgRetrievalRateLimitWait, + @Value("${fdp-index.events.retrieval.timeout:PT1M}") String cfgRetrievalTimeout, + @Value("${fdp-index.events.ping.validDuration:P7D}") String cfgPingValidDuration, + @Value("${fdp-index.events.ping.rateLimitDuration:PT6H}") String cfgPingRateLimitDuration, + @Value("${fdp-index.events.ping.rateLimitHits:10}") int cfgPingRateLimitHits + ) { + return EventsConfig.builder() + .retrievalRateLimitWait(Duration.parse(cfgRetrievalRateLimitWait)) + .retrievalTimeout(Duration.parse(cfgRetrievalTimeout)) + .pingValidDuration(Duration.parse(cfgPingValidDuration)) + .pingRateLimitDuration(Duration.parse(cfgPingRateLimitDuration)) + .pingRateLimitHits(cfgPingRateLimitHits) + .build(); + } + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/config/SecurityConfig.java b/src/main/java/nl/dtls/fairdatapoint/config/SecurityConfig.java index d52e6a9e9..4b6ccb936 100644 --- a/src/main/java/nl/dtls/fairdatapoint/config/SecurityConfig.java +++ b/src/main/java/nl/dtls/fairdatapoint/config/SecurityConfig.java @@ -59,6 +59,9 @@ protected void configure(HttpSecurity http) throws Exception { .antMatchers("/users/**").authenticated() .antMatchers("/memberships**").authenticated() .antMatchers("/tokens").permitAll() + .antMatchers("/search**").permitAll() + .antMatchers("/index/admin**").authenticated() + .antMatchers("/index**").permitAll() .antMatchers(HttpMethod.PUT, "/**").authenticated() .antMatchers(HttpMethod.POST, "/**").authenticated() .anyRequest().permitAll() diff --git a/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/MigrationRunner.java b/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/MigrationRunner.java index 59052fbc8..e816fea38 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/MigrationRunner.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/MigrationRunner.java @@ -25,6 +25,8 @@ import nl.dtls.fairdatapoint.Profiles; import nl.dtls.fairdatapoint.database.mongo.migration.development.acl.AclMigration; import nl.dtls.fairdatapoint.database.mongo.migration.development.apikey.ApiKeyMigration; +import nl.dtls.fairdatapoint.database.mongo.migration.development.index.entry.IndexEntryMigration; +import nl.dtls.fairdatapoint.database.mongo.migration.development.index.event.EventMigration; import nl.dtls.fairdatapoint.database.mongo.migration.development.membership.MembershipMigration; import nl.dtls.fairdatapoint.database.mongo.migration.development.metadata.MetadataMigration; import nl.dtls.fairdatapoint.database.mongo.migration.development.resource.ResourceDefinitionMigration; @@ -61,6 +63,12 @@ public class MigrationRunner { @Autowired private MetadataMigration metadataMigration; + @Autowired + private IndexEntryMigration indexEntryMigration; + + @Autowired + private EventMigration eventMigration; + @PostConstruct public void run() { userMigration.runMigration(); @@ -70,6 +78,8 @@ public void run() { shapeMigration.runMigration(); apiKeyMigration.runMigration(); metadataMigration.runMigration(); + indexEntryMigration.runMigration(); + eventMigration.runMigration(); } } diff --git a/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/entry/IndexEntryMigration.java b/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/entry/IndexEntryMigration.java new file mode 100644 index 000000000..587336676 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/entry/IndexEntryMigration.java @@ -0,0 +1,51 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.database.mongo.migration.development.index.entry; + +import nl.dtls.fairdatapoint.database.common.migration.Migration; +import nl.dtls.fairdatapoint.database.mongo.migration.development.index.entry.data.IndexEntryFixtures; +import nl.dtls.fairdatapoint.database.mongo.repository.IndexEntryRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class IndexEntryMigration implements Migration { + + @Autowired + private IndexEntryFixtures indexEntryFixtures; + + @Autowired + private IndexEntryRepository indexEntryRepository; + + public void runMigration() { + indexEntryRepository.deleteAll(); + + indexEntryRepository.save(indexEntryFixtures.entryActive()); + indexEntryRepository.save(indexEntryFixtures.entryActive2()); + indexEntryRepository.save(indexEntryFixtures.entryInactive()); + indexEntryRepository.save(indexEntryFixtures.entryUnreachable()); + indexEntryRepository.save(indexEntryFixtures.entryInvalid()); + indexEntryRepository.save(indexEntryFixtures.entryUnknown()); + } + +} \ No newline at end of file diff --git a/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/entry/data/IndexEntryFixtures.java b/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/entry/data/IndexEntryFixtures.java new file mode 100644 index 000000000..b9a0284e2 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/entry/data/IndexEntryFixtures.java @@ -0,0 +1,108 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.database.mongo.migration.development.index.entry.data; + +import nl.dtls.fairdatapoint.entity.index.config.EventsConfig; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; +import nl.dtls.fairdatapoint.entity.index.entry.RepositoryMetadata; +import org.springframework.stereotype.Service; + +import java.time.Instant; +import java.util.HashMap; + +import static nl.dtls.fairdatapoint.entity.index.entry.IndexEntryState.*; + +@Service +public class IndexEntryFixtures { + + private EventsConfig eventsConfig; + + public IndexEntry entryActive() { + String clientUri = "https://example.com/my-valid-fairdatapoint"; + RepositoryMetadata repositoryData = new RepositoryMetadata( + RepositoryMetadata.CURRENT_VERSION, + clientUri, + new HashMap<>() + ); + return new IndexEntry("8987abc1-15a4-4752-903c-8f8a5882cca6", clientUri, Valid, Instant.now(), Instant.now(), + Instant.now(), repositoryData); + } + + public IndexEntry entryActive2() { + String clientUri = "https://app.fairdatapoint.org"; + RepositoryMetadata repositoryData = new RepositoryMetadata( + RepositoryMetadata.CURRENT_VERSION, + clientUri, + new HashMap<>() + ); + Instant date = Instant.parse("2020-05-30T23:38:23.085Z"); + return new IndexEntry("c912331f-4a77-4300-a469-dbaf5fc0b4e2", clientUri, Valid, date, date, date, + repositoryData); + } + + public IndexEntry entryInactive() { + String clientUri = "https://example.com/my-valid-fairdatapoint"; + RepositoryMetadata repositoryData = new RepositoryMetadata( + RepositoryMetadata.CURRENT_VERSION, + clientUri, + new HashMap<>() + ); + Instant date = Instant.parse("2020-05-30T23:38:23.085Z"); + return new IndexEntry("b5851ebe-aacf-4de9-bf0a-3686e9256e73", clientUri, Valid, date, date, date, + repositoryData); + } + + public IndexEntry entryUnreachable() { + String clientUri = "https://example.com/my-unreachable-fairdatapoint"; + RepositoryMetadata repositoryData = new RepositoryMetadata( + RepositoryMetadata.CURRENT_VERSION, + clientUri, + new HashMap<>() + ); + return new IndexEntry("dae46b47-87fb-4fdf-995c-8aa3739a27fc", clientUri, Unreachable, Instant.now(), + Instant.now(), Instant.now(), repositoryData); + } + + public IndexEntry entryInvalid() { + String clientUri = "https://example.com/my-invalid-fairdatapoint"; + RepositoryMetadata repositoryData = new RepositoryMetadata( + RepositoryMetadata.CURRENT_VERSION, + clientUri, + new HashMap<>() + ); + return new IndexEntry("b37e8c1f-ac0e-49f8-8e07-35571c4f8235", clientUri, Invalid, Instant.now(), + Instant.now(), Instant.now(), repositoryData); + } + + public IndexEntry entryUnknown() { + String clientUri = "https://example.com/my-unknown-fairdatapoint"; + RepositoryMetadata repositoryData = new RepositoryMetadata( + RepositoryMetadata.CURRENT_VERSION, + clientUri, + new HashMap<>() + ); + return new IndexEntry("4471d7c5-8c5b-4581-a9bc-d175456492c4", clientUri, Unknown, Instant.now(), + Instant.now(), Instant.now(), repositoryData); + } + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/event/EventMigration.java b/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/event/EventMigration.java new file mode 100644 index 000000000..5e7768c0a --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/event/EventMigration.java @@ -0,0 +1,39 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.database.mongo.migration.development.index.event; + +import nl.dtls.fairdatapoint.database.mongo.repository.EventRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class EventMigration { + + @Autowired + private EventRepository eventRepository; + + public void runMigration() { + eventRepository.deleteAll(); + } + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/database/mongo/repository/EventRepository.java b/src/main/java/nl/dtls/fairdatapoint/database/mongo/repository/EventRepository.java new file mode 100644 index 000000000..4d3b807a4 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/database/mongo/repository/EventRepository.java @@ -0,0 +1,44 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.database.mongo.repository; + +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; +import nl.dtls.fairdatapoint.entity.index.event.Event; +import nl.dtls.fairdatapoint.entity.index.event.EventType; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; + +import java.time.Instant; +import java.util.List; + +public interface EventRepository extends MongoRepository { + + List getAllByType(EventType type); + + List getAllByFinishedIsNull(); + + Page getAllByRelatedTo(IndexEntry indexEntry, Pageable pageable); + + List findAllByIncomingPingExchangeRemoteAddrAndCreatedAfter(String remoteAddr, Instant after); +} diff --git a/src/main/java/nl/dtls/fairdatapoint/database/mongo/repository/IndexEntryRepository.java b/src/main/java/nl/dtls/fairdatapoint/database/mongo/repository/IndexEntryRepository.java new file mode 100644 index 000000000..3866e4ab0 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/database/mongo/repository/IndexEntryRepository.java @@ -0,0 +1,54 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.database.mongo.repository; + +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntryState; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; + +import java.time.Instant; +import java.util.Optional; + +public interface IndexEntryRepository extends MongoRepository { + + Optional findByUuid(String uuid); + + Optional findByClientUrl(String clientUrl); + + Page findAllByStateEquals(Pageable pageable, IndexEntryState state); + + Page findAllByStateEqualsAndLastRetrievalTimeBefore(Pageable pageable, IndexEntryState state, + Instant when); + + Page findAllByStateEqualsAndLastRetrievalTimeAfter(Pageable pageable, IndexEntryState state, + Instant when); + + long countAllByStateEquals(IndexEntryState state); + + long countAllByStateEqualsAndLastRetrievalTimeAfter(IndexEntryState state, Instant when); + + long countAllByStateEqualsAndLastRetrievalTimeBefore(IndexEntryState state, Instant when); + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/database/mongo/repository/WebhookRepository.java b/src/main/java/nl/dtls/fairdatapoint/database/mongo/repository/WebhookRepository.java new file mode 100644 index 000000000..c8331374a --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/database/mongo/repository/WebhookRepository.java @@ -0,0 +1,33 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.database.mongo.repository; + +import nl.dtls.fairdatapoint.entity.index.webhook.Webhook; +import org.springframework.data.mongodb.repository.MongoRepository; + +import java.util.Optional; +import java.util.UUID; + +public interface WebhookRepository extends MongoRepository { + Optional findByUuid(UUID uuid); +} diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java index 8bbd9b1c3..68008f38b 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java @@ -26,11 +26,10 @@ import com.google.common.io.Resources; import lombok.extern.slf4j.Slf4j; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; +import nl.dtls.fairdatapoint.entity.search.SearchResult; +import nl.dtls.fairdatapoint.entity.search.SearchResultRelation; import org.eclipse.rdf4j.common.iteration.Iterations; -import org.eclipse.rdf4j.model.IRI; -import org.eclipse.rdf4j.model.Resource; -import org.eclipse.rdf4j.model.Statement; -import org.eclipse.rdf4j.model.Value; +import org.eclipse.rdf4j.model.*; import org.eclipse.rdf4j.query.BindingSet; import org.eclipse.rdf4j.query.QueryResults; import org.eclipse.rdf4j.query.TupleQuery; @@ -45,10 +44,13 @@ import java.util.Map; import static java.lang.String.format; +import static java.util.stream.Collectors.toList; @Slf4j public abstract class AbstractMetadataRepository { + private static final String FIND_ENTITY_BY_LITERAL = "findEntityByLiteral.sparql"; + @Autowired protected Repository repository; @@ -73,6 +75,22 @@ public List find(IRI context) throws MetadataRepositoryException { } } + public List findByLiteral(Literal query) throws MetadataRepositoryException { + return runSparqlQuery(FIND_ENTITY_BY_LITERAL, AbstractMetadataRepository.class, Map.of( + "query", query)) + .stream() + .map(s -> new SearchResult( + s.getValue("entity").stringValue(), + s.getValue("title").stringValue(), + s.getValue("description").stringValue(), + new SearchResultRelation( + s.getValue("relationPredicate").stringValue(), + s.getValue("relationObject").stringValue()) + ) + ) + .collect(toList()); + } + public boolean checkExistence(Resource subject, IRI predicate, Value object) throws MetadataRepositoryException { try (RepositoryConnection conn = repository.getConnection()) { return conn.hasStatement(subject, predicate, object, false); diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/MetadataRepository.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/MetadataRepository.java index b65b02706..e471f7be9 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/MetadataRepository.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/MetadataRepository.java @@ -28,10 +28,8 @@ package nl.dtls.fairdatapoint.database.rdf.repository.common; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; -import org.eclipse.rdf4j.model.IRI; -import org.eclipse.rdf4j.model.Resource; -import org.eclipse.rdf4j.model.Statement; -import org.eclipse.rdf4j.model.Value; +import nl.dtls.fairdatapoint.entity.search.SearchResult; +import org.eclipse.rdf4j.model.*; import org.eclipse.rdf4j.query.BindingSet; import java.util.List; @@ -43,6 +41,8 @@ public interface MetadataRepository { List find(IRI context) throws MetadataRepositoryException; + List findByLiteral(Literal query) throws MetadataRepositoryException; + boolean checkExistence(Resource subject, IRI predicate, Value object) throws MetadataRepositoryException; void save(List statements, IRI context) throws MetadataRepositoryException; diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/config/EventsConfig.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/config/EventsConfig.java new file mode 100644 index 000000000..d42ef5eca --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/config/EventsConfig.java @@ -0,0 +1,38 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.config; + +import lombok.Builder; +import lombok.Data; + +import java.time.Duration; + +@Builder +@Data +public class EventsConfig { + private final Duration retrievalRateLimitWait; + private final Duration retrievalTimeout; + private final Duration pingValidDuration; + private final Duration pingRateLimitDuration; + private final int pingRateLimitHits; +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/entry/IndexEntry.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/entry/IndexEntry.java new file mode 100644 index 000000000..88640104c --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/entry/IndexEntry.java @@ -0,0 +1,81 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.entry; + +import lombok.*; +import org.bson.types.ObjectId; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.format.annotation.DateTimeFormat; + +import java.time.Duration; +import java.time.Instant; + +@Document +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +@EqualsAndHashCode +public class IndexEntry { + + @Id + protected ObjectId id; + + @Indexed(unique = true) + private String uuid; + + private String clientUrl; + + private IndexEntryState state = IndexEntryState.Unknown; + + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Instant registrationTime; + + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Instant modificationTime; + + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Instant lastRetrievalTime; + + private RepositoryMetadata currentMetadata; + + public IndexEntry(String uuid, String clientUrl, IndexEntryState state, Instant registrationTime, + Instant modificationTime, Instant lastRetrievalTime, RepositoryMetadata currentMetadata) { + this.uuid = uuid; + this.clientUrl = clientUrl; + this.state = state; + this.registrationTime = registrationTime; + this.modificationTime = modificationTime; + this.lastRetrievalTime = lastRetrievalTime; + this.currentMetadata = currentMetadata; + } + + public Duration getLastRetrievalAgo() { + if (lastRetrievalTime == null) { + return null; + } + return Duration.between(lastRetrievalTime, Instant.now()); + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/entry/IndexEntryState.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/entry/IndexEntryState.java new file mode 100644 index 000000000..a5db38f2f --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/entry/IndexEntryState.java @@ -0,0 +1,35 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.entry; + +public enum IndexEntryState { + + Unknown, + + Valid, // Active / Inactive based on timestamps + + Unreachable, + + Invalid + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/entry/RepositoryMetadata.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/entry/RepositoryMetadata.java new file mode 100644 index 000000000..d1139ae81 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/entry/RepositoryMetadata.java @@ -0,0 +1,48 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.entry; + +import lombok.*; + +import javax.validation.constraints.NotNull; +import java.util.HashMap; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +@EqualsAndHashCode +public class RepositoryMetadata { + + public static final Integer CURRENT_VERSION = 1; + + private Integer metadataVersion = CURRENT_VERSION; + + private String repositoryUri; + + @NotNull + private HashMap metadata = new HashMap<>(); + +} + diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/event/AdminTrigger.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/AdminTrigger.java new file mode 100644 index 000000000..839d3e309 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/AdminTrigger.java @@ -0,0 +1,36 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.event; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class AdminTrigger { + private String remoteAddr; + private String tokenName; + private String clientUrl; +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/event/Event.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/Event.java new file mode 100644 index 000000000..808e12191 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/Event.java @@ -0,0 +1,126 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.event; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; +import org.bson.types.ObjectId; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.DBRef; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.format.annotation.DateTimeFormat; + +import javax.validation.constraints.NotNull; +import java.time.Instant; +import java.util.UUID; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Document(collection = "event") +public class Event { + @Id + protected ObjectId id; + @Indexed(unique = true) + @NotNull + private UUID uuid = UUID.randomUUID(); + @NotNull + private EventType type; + @NotNull + private Integer version; + + @DBRef + private Event triggeredBy; + @DBRef + private IndexEntry relatedTo; + + // Content (one of those) + private IncomingPing incomingPing; + private MetadataRetrieval metadataRetrieval; + private AdminTrigger adminTrigger; + private WebhookPing webhookPing; + private WebhookTrigger webhookTrigger; + + @NotNull + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Instant created = Instant.now(); + + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Instant executed; + + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Instant finished; + + public boolean isExecuted() { + return executed != null; + } + + public void execute() { + executed = Instant.now(); + } + + public boolean isFinished() { + return finished != null; + } + + public void finish() { + finished = Instant.now(); + } + + public Event(Integer version, IncomingPing incomingPing) { + this.type = EventType.IncomingPing; + this.version = version; + this.incomingPing = incomingPing; + } + + public Event(Integer version, Event triggerEvent, IndexEntry relatedTo, MetadataRetrieval metadataRetrieval) { + this.type = EventType.MetadataRetrieval; + this.version = version; + this.triggeredBy = triggerEvent; + this.relatedTo = relatedTo; + this.metadataRetrieval = metadataRetrieval; + } + + public Event(Integer version, AdminTrigger adminTrigger) { + this.type = EventType.AdminTrigger; + this.version = version; + this.adminTrigger = adminTrigger; + } + + public Event(Integer version, WebhookTrigger webhookTrigger, Event triggerEvent) { + this.type = EventType.WebhookTrigger; + this.version = version; + this.webhookTrigger = webhookTrigger; + this.triggeredBy = triggerEvent; + this.relatedTo = triggerEvent.getRelatedTo(); + } + + public Event(Integer version, WebhookPing webhookPing) { + this.type = EventType.WebhookPing; + this.version = version; + this.webhookPing = webhookPing; + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/event/EventType.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/EventType.java new file mode 100644 index 000000000..54406f972 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/EventType.java @@ -0,0 +1,31 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.event; + +public enum EventType { + AdminTrigger, + MetadataRetrieval, + WebhookTrigger, + IncomingPing, + WebhookPing, +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/event/IncomingPing.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/IncomingPing.java new file mode 100644 index 000000000..118c3c8e2 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/IncomingPing.java @@ -0,0 +1,36 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.event; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import nl.dtls.fairdatapoint.entity.index.http.Exchange; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class IncomingPing { + private Exchange exchange; + private Boolean newEntry; +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/event/MetadataRetrieval.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/MetadataRetrieval.java new file mode 100644 index 000000000..6c65cbac2 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/MetadataRetrieval.java @@ -0,0 +1,38 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.event; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import nl.dtls.fairdatapoint.entity.index.entry.RepositoryMetadata; +import nl.dtls.fairdatapoint.entity.index.http.Exchange; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class MetadataRetrieval { + private String error; + private Exchange exchange; + private RepositoryMetadata metadata; +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/event/WebhookPing.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/WebhookPing.java new file mode 100644 index 000000000..c0776fe73 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/WebhookPing.java @@ -0,0 +1,38 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.event; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class WebhookPing { + private String remoteAddr; + private String tokenName; + private UUID webhookUuid; +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/event/WebhookTrigger.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/WebhookTrigger.java new file mode 100644 index 000000000..4c4d51673 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/WebhookTrigger.java @@ -0,0 +1,43 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.event; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import nl.dtls.fairdatapoint.entity.index.http.Exchange; +import nl.dtls.fairdatapoint.entity.index.webhook.Webhook; +import nl.dtls.fairdatapoint.entity.index.webhook.WebhookEvent; +import org.springframework.data.mongodb.core.mapping.DBRef; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class WebhookTrigger { + @DBRef + private Webhook webhook; + + private WebhookEvent matchedEvent; + + private Exchange exchange; +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/exception/IncorrectPingFormatException.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/exception/IncorrectPingFormatException.java new file mode 100644 index 000000000..5dec39b4d --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/exception/IncorrectPingFormatException.java @@ -0,0 +1,32 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.exception; + +import org.springframework.http.HttpStatus; + +public class IncorrectPingFormatException extends IndexException { + + public IncorrectPingFormatException(String message) { + super(message, HttpStatus.BAD_REQUEST); + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/exception/IndexException.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/exception/IndexException.java new file mode 100644 index 000000000..bdfcb2932 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/exception/IndexException.java @@ -0,0 +1,44 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.exception; + +import nl.dtls.fairdatapoint.api.dto.error.ErrorDTO; +import org.springframework.http.HttpStatus; + +public abstract class IndexException extends RuntimeException { + + protected final HttpStatus status; + + public IndexException(String message, HttpStatus status) { + super(message); + this.status = status; + } + + public HttpStatus getStatus() { + return status; + } + + public ErrorDTO getErrorDTO() { + return new ErrorDTO(getStatus(), getMessage()); + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/exception/RateLimitException.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/exception/RateLimitException.java new file mode 100644 index 000000000..923ccb9f7 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/exception/RateLimitException.java @@ -0,0 +1,32 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.exception; + +import org.springframework.http.HttpStatus; + +public class RateLimitException extends IndexException { + + public RateLimitException(String message) { + super(message, HttpStatus.TOO_MANY_REQUESTS); + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/http/Exchange.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/http/Exchange.java new file mode 100644 index 000000000..a6505399a --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/http/Exchange.java @@ -0,0 +1,49 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.http; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Exchange { + private ExchangeDirection direction; + private ExchangeState state = ExchangeState.Prepared; + private String remoteAddr; + private String error; + + private Request request = new Request(); + private Response response = new Response(); + + public Exchange(ExchangeDirection direction, String remoteAddr) { + this.direction = direction; + this.remoteAddr = remoteAddr; + } + + public Exchange(ExchangeDirection direction) { + this.direction = direction; + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/http/ExchangeDirection.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/http/ExchangeDirection.java new file mode 100644 index 000000000..14bebd589 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/http/ExchangeDirection.java @@ -0,0 +1,28 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.http; + +public enum ExchangeDirection { + INCOMING, + OUTGOING +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/http/ExchangeState.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/http/ExchangeState.java new file mode 100644 index 000000000..733c676e8 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/http/ExchangeState.java @@ -0,0 +1,31 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.http; + +public enum ExchangeState { + Prepared, + Requested, + Timeout, + Failed, + Retrieved +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/http/Request.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/http/Request.java new file mode 100644 index 000000000..e662148c4 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/http/Request.java @@ -0,0 +1,77 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.http; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.http.HttpEntity; + +import javax.servlet.http.HttpServletRequest; +import java.net.http.HttpRequest; +import java.util.List; +import java.util.Map; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Request { + private String method; + private String url; + + private Map> headers; + private String body; + + public Map> getHeaders() { + return headers; + } + + public void setHeaders(Map> headers) { + this.headers = headers; + } + + public String getBody() { + return body; + } + + public void setBody(String body) { + this.body = body; + } + + public void setFromHttpEntity(HttpEntity httpEntity) { + body = httpEntity.getBody(); + headers = httpEntity.getHeaders(); + } + + public void setFromHttpServletRequest(HttpServletRequest request) { + method = request.getMethod(); + url = request.getRequestURI(); + } + + public void setFromHttpRequest(HttpRequest request) { + method = request.method(); + url = request.uri().toString(); + body = request.bodyPublisher().map(Object::toString).orElse(null); + headers = request.headers().map(); + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/http/Response.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/http/Response.java new file mode 100644 index 000000000..719597c53 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/http/Response.java @@ -0,0 +1,50 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.http; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.net.http.HttpResponse; +import java.util.List; +import java.util.Map; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Response { + private Integer code; + private String url; + private String origin; + + private Map> headers; + private String body; + + public void setFromHttpResponse(HttpResponse response) { + code = response.statusCode(); + url = response.uri().toString(); + headers = response.headers().map(); + body = response.body(); + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/webhook/Webhook.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/webhook/Webhook.java new file mode 100644 index 000000000..bd9dafedb --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/webhook/Webhook.java @@ -0,0 +1,63 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.webhook; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.bson.types.ObjectId; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; + +import javax.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Document(collection = "webhook") +public class Webhook { + @Id + protected ObjectId id; + + @Indexed(unique = true) + @NotNull + private UUID uuid = UUID.randomUUID(); + + private String payloadUrl; + + private String secret; + + private boolean allEvents; + + private List events = new ArrayList<>(); + + private boolean allEntries; + + private List entries = new ArrayList<>(); + + private boolean enabled; +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/webhook/WebhookEvent.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/webhook/WebhookEvent.java new file mode 100644 index 000000000..e4d22fca8 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/webhook/WebhookEvent.java @@ -0,0 +1,33 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.entity.index.webhook; + +public enum WebhookEvent { + NewEntry, + IncomingPing, + EntryValid, + EntryInvalid, + EntryUnreachable, + AdminTrigger, + WebhookPing +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResult.java b/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResult.java new file mode 100644 index 000000000..76da792bf --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResult.java @@ -0,0 +1,21 @@ +package nl.dtls.fairdatapoint.entity.search; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +public class SearchResult { + + private String uri; + + private String title; + + private String description; + + private SearchResultRelation relation; +} diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResultRelation.java b/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResultRelation.java new file mode 100644 index 000000000..68eee93ad --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResultRelation.java @@ -0,0 +1,18 @@ +package nl.dtls.fairdatapoint.entity.search; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +public class SearchResultRelation { + + private String predicate; + + private String object; + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/config/ConfigService.java b/src/main/java/nl/dtls/fairdatapoint/service/config/ConfigService.java index a7f84ec4d..6a36daa54 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/config/ConfigService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/config/ConfigService.java @@ -27,6 +27,7 @@ import nl.dtls.fairdatapoint.entity.resource.ResourceDefinition; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.util.List; @@ -38,12 +39,15 @@ public class ConfigService { @Qualifier("persistentUrl") private String persistentUrl; + @Value("${instance.index:false}") + private boolean index; + @Autowired private ResourceDefinitionRepository resourceDefinitionRepository; public BootstrapConfigDTO getBootstrapConfig() { List resourceDefinitions = resourceDefinitionRepository.findAll(); - return new BootstrapConfigDTO(persistentUrl, resourceDefinitions); + return new BootstrapConfigDTO(persistentUrl, resourceDefinitions, index); } } diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryMapper.java b/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryMapper.java new file mode 100644 index 000000000..bb516da87 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryMapper.java @@ -0,0 +1,85 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.service.index.entry; + +import nl.dtls.fairdatapoint.api.dto.index.entry.IndexEntryDTO; +import nl.dtls.fairdatapoint.api.dto.index.entry.IndexEntryDetailDTO; +import nl.dtls.fairdatapoint.api.dto.index.entry.IndexEntryStateDTO; +import nl.dtls.fairdatapoint.entity.index.config.EventsConfig; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntryState; +import nl.dtls.fairdatapoint.entity.index.event.Event; +import nl.dtls.fairdatapoint.service.index.event.EventMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.time.Instant; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +@Service +public class IndexEntryMapper { + + @Autowired + private EventsConfig eventsConfig; + + @Autowired + private EventMapper eventMapper; + + public IndexEntryDTO toDTO(IndexEntry indexEntry) { + return new IndexEntryDTO( + indexEntry.getUuid(), + indexEntry.getClientUrl(), + toStateDTO(indexEntry.getState(), indexEntry.getLastRetrievalTime()), + indexEntry.getRegistrationTime().toString(), + indexEntry.getModificationTime().toString() + ); + } + + public IndexEntryDetailDTO toDetailDTO(IndexEntry indexEntry, Iterable events) { + return new IndexEntryDetailDTO( + indexEntry.getUuid(), + indexEntry.getClientUrl(), + toStateDTO(indexEntry.getState(), indexEntry.getLastRetrievalTime()), + indexEntry.getCurrentMetadata(), + StreamSupport.stream(events.spliterator(), false) + .map(eventMapper::toDTO) + .collect(Collectors.toList()), + indexEntry.getRegistrationTime().toString(), + indexEntry.getModificationTime().toString(), + indexEntry.getLastRetrievalTime().toString() + ); + } + + public IndexEntryStateDTO toStateDTO(IndexEntryState state, Instant lastRetrievalTime) { + return switch (state) { + case Unknown -> IndexEntryStateDTO.UNKNOWN; + case Valid -> lastRetrievalTime.isAfter(Instant.now().minus(eventsConfig.getPingValidDuration())) + ? IndexEntryStateDTO.ACTIVE + : IndexEntryStateDTO.INACTIVE; + case Invalid -> IndexEntryStateDTO.INVALID; + case Unreachable -> IndexEntryStateDTO.UNREACHABLE; + }; + } + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryService.java b/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryService.java new file mode 100644 index 000000000..05b3b0115 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryService.java @@ -0,0 +1,126 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.service.index.entry; + +import lombok.extern.log4j.Log4j2; +import nl.dtls.fairdatapoint.api.dto.index.entry.IndexEntryInfoDTO; +import nl.dtls.fairdatapoint.api.dto.index.entry.IndexEntryStateDTO; +import nl.dtls.fairdatapoint.api.dto.index.ping.PingDTO; +import nl.dtls.fairdatapoint.database.mongo.repository.IndexEntryRepository; +import nl.dtls.fairdatapoint.entity.index.config.EventsConfig; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntryState; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; +import org.springframework.validation.annotation.Validated; + +import javax.validation.Valid; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +import static nl.dtls.fairdatapoint.api.dto.index.entry.IndexEntryStateDTO.*; + +@Service +@Validated +@Log4j2 +public class IndexEntryService { + + @Autowired + private IndexEntryRepository repository; + + @Autowired + private EventsConfig eventsConfig; + + public Iterable getAllEntries() { + return repository.findAll(); + } + + public Page getEntriesPage(Pageable pageable, String state) { + if (state.equalsIgnoreCase(ACTIVE.name())) { + return repository.findAllByStateEqualsAndLastRetrievalTimeAfter(pageable, IndexEntryState.Valid, + getValidThreshold()); + } + if (state.equalsIgnoreCase(IndexEntryStateDTO.INACTIVE.name())) { + return repository.findAllByStateEqualsAndLastRetrievalTimeBefore(pageable, IndexEntryState.Valid, + getValidThreshold()); + } + if (state.equalsIgnoreCase(IndexEntryStateDTO.UNREACHABLE.name())) { + return repository.findAllByStateEquals(pageable, IndexEntryState.Unreachable); + } + if (state.equalsIgnoreCase(IndexEntryStateDTO.INVALID.name())) { + return repository.findAllByStateEquals(pageable, IndexEntryState.Invalid); + } + if (state.equalsIgnoreCase(IndexEntryStateDTO.UNKNOWN.name())) { + return repository.findAllByStateEquals(pageable, IndexEntryState.Unknown); + } + return repository.findAll(pageable); + } + + public Optional getEntry(String uuid) { + return repository.findByUuid(uuid); + } + + public IndexEntryInfoDTO getEntriesInfo() { + Map entriesCount = new HashMap<>(); + entriesCount.put("ALL", repository.count()); + entriesCount.put(UNKNOWN.name(), repository.countAllByStateEquals(IndexEntryState.Unknown)); + entriesCount.put(ACTIVE.name(), + repository.countAllByStateEqualsAndLastRetrievalTimeAfter(IndexEntryState.Valid, getValidThreshold())); + entriesCount.put(INACTIVE.name(), + repository.countAllByStateEqualsAndLastRetrievalTimeBefore(IndexEntryState.Valid, getValidThreshold())); + entriesCount.put(UNREACHABLE.name(), repository.countAllByStateEquals(IndexEntryState.Unreachable)); + entriesCount.put(INVALID.name(), repository.countAllByStateEquals(IndexEntryState.Invalid)); + return new IndexEntryInfoDTO(entriesCount); + } + + public IndexEntry storeEntry(@Valid PingDTO pingDTO) { + var clientUrl = pingDTO.getClientUrl(); + var entity = repository.findByClientUrl(clientUrl); + var now = Instant.now(); + + final IndexEntry entry; + if (entity.isPresent()) { + log.info("Updating timestamp of existing entry {}", clientUrl); + entry = entity.orElseThrow(); + } else { + log.info("Storing new entry {}", clientUrl); + entry = new IndexEntry(); + entry.setUuid(UUID.randomUUID().toString()); + entry.setClientUrl(clientUrl); + entry.setRegistrationTime(now); + } + + entry.setModificationTime(now); + return repository.save(entry); + } + + private Instant getValidThreshold() { + return Instant.now().minus(eventsConfig.getPingValidDuration()); + } + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/event/EventMapper.java b/src/main/java/nl/dtls/fairdatapoint/service/index/event/EventMapper.java new file mode 100644 index 000000000..5866fc66d --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/event/EventMapper.java @@ -0,0 +1,56 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.service.index.event; + + +import nl.dtls.fairdatapoint.api.dto.index.event.EventDTO; +import nl.dtls.fairdatapoint.entity.index.event.AdminTrigger; +import nl.dtls.fairdatapoint.entity.index.event.Event; +import org.springframework.security.core.Authentication; +import org.springframework.stereotype.Service; + +import javax.servlet.http.HttpServletRequest; + +@Service +public class EventMapper { + + private static final Integer VERSION = 1; + + public EventDTO toDTO(Event event) { + return new EventDTO( + event.getUuid(), + event.getType(), + event.getCreated().toString(), + event.getFinished().toString() + ); + } + + public Event toAdminTriggerEvent(HttpServletRequest request, Authentication authentication, String clientUrl) { + var adminTrigger = new AdminTrigger(); + adminTrigger.setRemoteAddr(request.getRemoteAddr()); + adminTrigger.setTokenName(authentication.getName()); + adminTrigger.setClientUrl(clientUrl); + return new Event(VERSION, adminTrigger); + } + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/event/EventService.java b/src/main/java/nl/dtls/fairdatapoint/service/index/event/EventService.java new file mode 100644 index 000000000..917350ac8 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/event/EventService.java @@ -0,0 +1,235 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.service.index.event; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; +import nl.dtls.fairdatapoint.api.dto.index.ping.PingDTO; +import nl.dtls.fairdatapoint.database.mongo.repository.EventRepository; +import nl.dtls.fairdatapoint.database.mongo.repository.IndexEntryRepository; +import nl.dtls.fairdatapoint.entity.exception.ResourceNotFoundException; +import nl.dtls.fairdatapoint.entity.index.config.EventsConfig; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntryState; +import nl.dtls.fairdatapoint.entity.index.event.Event; +import nl.dtls.fairdatapoint.entity.index.event.EventType; +import nl.dtls.fairdatapoint.entity.index.exception.IncorrectPingFormatException; +import nl.dtls.fairdatapoint.entity.index.exception.RateLimitException; +import nl.dtls.fairdatapoint.entity.index.http.Exchange; +import nl.dtls.fairdatapoint.entity.index.http.ExchangeState; +import nl.dtls.fairdatapoint.service.index.entry.IndexEntryService; +import nl.dtls.fairdatapoint.service.index.webhook.WebhookService; +import org.eclipse.rdf4j.util.iterators.EmptyIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import javax.servlet.http.HttpServletRequest; +import java.time.Instant; +import java.util.Optional; + +@Service +public class EventService { + private static final Logger logger = LoggerFactory.getLogger(EventService.class); + + @Autowired + private ObjectMapper objectMapper; + + @Autowired + private ThreadPoolTaskExecutor executor; + + @Autowired + private EventRepository eventRepository; + + @Autowired + private IndexEntryRepository indexEntryRepository; + + @Autowired + private IndexEntryService indexEntryService; + + @Autowired + private WebhookService webhookService; + + @Autowired + private EventsConfig eventsConfig; + + @Autowired + private EventMapper eventMapper; + + @Autowired + private IncomingPingUtils incomingPingUtils; + + public Iterable getEvents(IndexEntry indexEntry) { + // TODO: make events pagination in the future + return eventRepository.getAllByRelatedTo(indexEntry, PageRequest.of(0, 10, Sort.by(Sort.Direction.DESC, + "created"))); + } + + public Iterable getEvents(String indexEntryUuid) { + return indexEntryService.getEntry(indexEntryUuid).map(this::getEvents).orElse(EmptyIterator::new); + } + + @SneakyThrows + public Event acceptIncomingPing(PingDTO reqDto, HttpServletRequest request) { + var remoteAddr = request.getRemoteAddr(); + var rateLimitSince = Instant.now().minus(eventsConfig.getPingRateLimitDuration()); + var previousPings = eventRepository.findAllByIncomingPingExchangeRemoteAddrAndCreatedAfter(remoteAddr, + rateLimitSince); + if (previousPings.size() > eventsConfig.getPingRateLimitHits()) { + logger.warn("Rate limit for PING reached by {}", remoteAddr); + throw new RateLimitException(String.format( + "Rate limit reached for %s (max. %d per %s) - PING ignored", + remoteAddr, eventsConfig.getPingRateLimitHits(), eventsConfig.getPingRateLimitDuration().toString()) + ); + } + + var event = incomingPingUtils.prepareEvent(reqDto, request); + eventRepository.save(event); + event.execute(); + try { + var indexEntry = indexEntryService.storeEntry(reqDto); + event.getIncomingPing().setNewEntry(indexEntry.getRegistrationTime().equals(indexEntry.getModificationTime())); + event.getIncomingPing().getExchange().getResponse().setCode(204); + event.setRelatedTo(indexEntry); + logger.info("Accepted incoming ping as a new event"); + } catch (Exception e) { + var ex = new IncorrectPingFormatException("Could not parse PING: " + e.getMessage()); + event.getIncomingPing().getExchange().getResponse().setCode(400); + event.getIncomingPing().getExchange().getResponse().setBody(objectMapper.writeValueAsString(ex.getErrorDTO())); + event.setFinished(Instant.now()); + eventRepository.save(event); + logger.info("Incoming ping has incorrect format: {}", e.getMessage()); + throw ex; + } + event.setFinished(Instant.now()); + return eventRepository.save(event); + } + + private void processMetadataRetrieval(Event event) { + String clientUrl = event.getRelatedTo().getClientUrl(); + if (MetadataRetrievalUtils.shouldRetrieve(event, eventsConfig.getRetrievalRateLimitWait())) { + indexEntryRepository.save(event.getRelatedTo()); + eventRepository.save(event); + event.execute(); + + logger.info("Retrieving metadata for {}", clientUrl); + MetadataRetrievalUtils.retrieveRepositoryMetadata(event, eventsConfig.getRetrievalTimeout()); + Exchange ex = event.getMetadataRetrieval().getExchange(); + if (ex.getState() == ExchangeState.Retrieved) { + try { + logger.info("Parsing metadata for {}", clientUrl); + var metadata = MetadataRetrievalUtils.parseRepositoryMetadata(ex.getResponse().getBody()); + if (metadata.isPresent()) { + event.getMetadataRetrieval().setMetadata(metadata.get()); + event.getRelatedTo().setCurrentMetadata(metadata.get()); + event.getRelatedTo().setState(IndexEntryState.Valid); + logger.info("Storing metadata for {}", clientUrl); + indexEntryRepository.save(event.getRelatedTo()); + } else { + logger.info("Repository not found in metadata for {}", clientUrl); + event.getRelatedTo().setState(IndexEntryState.Invalid); + event.getMetadataRetrieval().setError("Repository not found in metadata"); + } + } catch (Exception e) { + logger.info("Cannot parse metadata for {}", clientUrl); + event.getRelatedTo().setState(IndexEntryState.Invalid); + event.getMetadataRetrieval().setError("Cannot parse metadata"); + } + } else { + event.getRelatedTo().setState(IndexEntryState.Unreachable); + logger.info("Cannot retrieve metadata for {}: {}", clientUrl, ex.getError()); + } + } else { + logger.info("Rate limit reached for {} (skipping metadata retrieval)", clientUrl); + event.getMetadataRetrieval().setError("Rate limit reached (skipping)"); + } + event.getRelatedTo().setLastRetrievalTime(Instant.now()); + event.finish(); + event = eventRepository.save(event); + indexEntryRepository.save(event.getRelatedTo()); + webhookService.triggerWebhooks(event); + } + + @Async + public void triggerMetadataRetrieval(Event triggerEvent) { + logger.info("Initiating metadata retrieval triggered by {}", triggerEvent.getUuid()); + Iterable events = MetadataRetrievalUtils.prepareEvents(triggerEvent, indexEntryService); + for (Event event : events) { + logger.info("Triggering metadata retrieval for {} as {}", event.getRelatedTo().getClientUrl(), + event.getUuid()); + try { + processMetadataRetrieval(event); + } catch (Exception e) { + logger.error("Failed to retrieve metadata: {}", e.getMessage()); + } + } + logger.info("Finished metadata retrieval triggered by {}", triggerEvent.getUuid()); + } + + private void resumeUnfinishedEvents() { + logger.info("Resuming unfinished events"); + for (Event event : eventRepository.getAllByFinishedIsNull()) { + logger.info("Resuming event {}", event.getUuid()); + + try { + if (event.getType() == EventType.MetadataRetrieval) { + processMetadataRetrieval(event); + } else if (event.getType() == EventType.WebhookTrigger) { + webhookService.processWebhookTrigger(event); + } else { + logger.warn("Unknown event type {} ({})", event.getUuid(), event.getType()); + } + } catch (Exception e) { + logger.error("Failed to resume event {}: {}", event.getUuid(), e.getMessage()); + } + } + logger.info("Finished unfinished events"); + } + + @PostConstruct + public void startResumeUnfinishedEvents() { + executor.submit(this::resumeUnfinishedEvents); + } + + public Event acceptAdminTrigger(HttpServletRequest request, String indexEntryUuid) { + Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); + Event event = eventMapper.toAdminTriggerEvent(request, authentication, indexEntryUuid); + if (indexEntryUuid != null) { + Optional entry = indexEntryService.getEntry(indexEntryUuid); + if (entry.isEmpty()) { + throw new ResourceNotFoundException("There is no such entry: " + indexEntryUuid); + } + event.setRelatedTo(entry.get()); + } + event.finish(); + return eventRepository.save(event); + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/event/IncomingPingUtils.java b/src/main/java/nl/dtls/fairdatapoint/service/index/event/IncomingPingUtils.java new file mode 100644 index 000000000..4f9ae523a --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/event/IncomingPingUtils.java @@ -0,0 +1,77 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.service.index.event; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import nl.dtls.fairdatapoint.api.dto.index.ping.PingDTO; +import nl.dtls.fairdatapoint.entity.index.event.Event; +import nl.dtls.fairdatapoint.entity.index.event.IncomingPing; +import nl.dtls.fairdatapoint.entity.index.http.Exchange; +import nl.dtls.fairdatapoint.entity.index.http.ExchangeDirection; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.servlet.http.HttpServletRequest; +import java.util.*; + +@Service +public class IncomingPingUtils { + + private static final Integer VERSION = 1; + + @Autowired + private ObjectMapper objectMapper; + + public Event prepareEvent(PingDTO reqDto, HttpServletRequest request) { + var incomingPing = new IncomingPing(); + var ex = new Exchange(ExchangeDirection.INCOMING, request.getRemoteAddr()); + incomingPing.setExchange(ex); + + ex.getRequest().setHeaders(getHeaders(request)); + ex.getRequest().setFromHttpServletRequest(request); + try { + ex.getRequest().setBody(objectMapper.writeValueAsString(reqDto)); + } catch (JsonProcessingException e) { + ex.getRequest().setBody(null); + } + + return new Event(VERSION, incomingPing); + } + + private Map> getHeaders(HttpServletRequest request) { + Map> map = new HashMap<>(); + Iterator requestI = request.getHeaderNames().asIterator(); + while (requestI.hasNext()) { + String headerName = requestI.next(); + List headerValues = new ArrayList<>(); + Iterator headerI = request.getHeaders(headerName).asIterator(); + while (headerI.hasNext()) { + headerValues.add(headerI.next()); + } + map.put(headerName, headerValues); + } + return map; + } + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/event/MetadataRetrievalUtils.java b/src/main/java/nl/dtls/fairdatapoint/service/index/event/MetadataRetrievalUtils.java new file mode 100644 index 000000000..b4ee686a8 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/event/MetadataRetrievalUtils.java @@ -0,0 +1,190 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.service.index.event; + +import nl.dtls.fairdatapoint.entity.index.entry.RepositoryMetadata; +import nl.dtls.fairdatapoint.entity.index.event.Event; +import nl.dtls.fairdatapoint.entity.index.event.EventType; +import nl.dtls.fairdatapoint.entity.index.event.MetadataRetrieval; +import nl.dtls.fairdatapoint.entity.index.http.Exchange; +import nl.dtls.fairdatapoint.entity.index.http.ExchangeDirection; +import nl.dtls.fairdatapoint.entity.index.http.ExchangeState; +import nl.dtls.fairdatapoint.service.index.entry.IndexEntryService; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.Statement; +import org.eclipse.rdf4j.model.Value; +import org.eclipse.rdf4j.model.impl.SimpleValueFactory; +import org.eclipse.rdf4j.model.vocabulary.DCTERMS; +import org.eclipse.rdf4j.model.vocabulary.FOAF; +import org.eclipse.rdf4j.model.vocabulary.RDF; +import org.eclipse.rdf4j.rio.RDFFormat; +import org.eclipse.rdf4j.rio.RDFParser; +import org.eclipse.rdf4j.rio.Rio; +import org.eclipse.rdf4j.rio.helpers.StatementCollector; +import org.springframework.http.HttpHeaders; + +import java.io.IOException; +import java.io.StringReader; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Map; +import java.util.Optional; + +public class MetadataRetrievalUtils { + + private static final EventType EVENT_TYPE = EventType.MetadataRetrieval; + + private static final Integer VERSION = 1; + + private static final IRI REPOSITORY = SimpleValueFactory.getInstance().createIRI("http://www.re3data" + + ".org/schema/3-0#Repository"); + + private static final IRI COUNTRY = SimpleValueFactory.getInstance().createIRI("http://www.re3data" + + ".org/schema/3-0#institutionCountry"); + + private static final Map MAPPING = Map.of( + DCTERMS.TITLE, "title", + DCTERMS.DESCRIPTION, "description", + DCTERMS.HAS_VERSION, "version", + DCTERMS.PUBLISHER, "publisher", + COUNTRY, "country" + ); + + private static final HttpClient client = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_2) + .followRedirects(HttpClient.Redirect.ALWAYS) + .build(); + + public static boolean shouldRetrieve(Event triggerEvent, Duration rateLimitWait) { + if (triggerEvent.getRelatedTo() == null) { + return false; + } + Instant lastRetrieval = triggerEvent.getRelatedTo().getLastRetrievalTime(); + if (lastRetrieval == null) { + return true; + } + return Duration.between(lastRetrieval, Instant.now()).compareTo(rateLimitWait) > 0; + } + + public static Iterable prepareEvents(Event triggerEvent, IndexEntryService indexEntryService) { + ArrayList events = new ArrayList<>(); + if (triggerEvent.getType() == EventType.IncomingPing) { + events.add(new Event(VERSION, triggerEvent, triggerEvent.getRelatedTo(), new MetadataRetrieval())); + } else if (triggerEvent.getType() == EventType.AdminTrigger) { + if (triggerEvent.getAdminTrigger().getClientUrl() == null) { + indexEntryService.getAllEntries().forEach( + entry -> events.add(new Event(VERSION, triggerEvent, entry, new MetadataRetrieval())) + ); + } else { + events.add(new Event(VERSION, triggerEvent, triggerEvent.getRelatedTo(), new MetadataRetrieval())); + } + } + return events; + } + + public static void retrieveRepositoryMetadata(Event event, Duration timeout) { + if (event.getType() != EVENT_TYPE) { + throw new IllegalArgumentException("Invalid event type"); + } + var ex = new Exchange(ExchangeDirection.OUTGOING); + event.getMetadataRetrieval().setExchange(ex); + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(event.getRelatedTo().getClientUrl())) + .timeout(timeout) + .header(HttpHeaders.ACCEPT, RDFFormat.TURTLE.getDefaultMIMEType()) + .GET().build(); + ex.getRequest().setFromHttpRequest(request); + ex.setState(ExchangeState.Requested); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + ex.getResponse().setFromHttpResponse(response); + ex.setState(ExchangeState.Retrieved); + } catch (InterruptedException e) { + ex.setState(ExchangeState.Timeout); + ex.setError("Timeout"); + } catch (IllegalArgumentException e) { + ex.setState(ExchangeState.Failed); + ex.setError("Invalid URI: " + e.getMessage()); + } catch (IOException e) { + ex.setState(ExchangeState.Failed); + ex.setError("IO error: " + e.getMessage()); + } + } + + public static Optional parseRepositoryMetadata(String metadata) throws IOException { + RDFParser parser = Rio.createParser(RDFFormat.TURTLE); + StatementCollector collector = new StatementCollector(); + parser.setRDFHandler(collector); + + parser.parse(new StringReader(metadata), String.valueOf(StandardCharsets.UTF_8)); + ArrayList statements = new ArrayList<>(collector.getStatements()); + + return findRepository(statements).map(repository -> extractRepositoryMetadata(statements, repository)); + } + + private static RepositoryMetadata extractRepositoryMetadata(ArrayList statements, Resource repository) { + var repositoryMetadata = new RepositoryMetadata(); + repositoryMetadata.setMetadataVersion(VERSION); + repositoryMetadata.setRepositoryUri(repository.toString()); + + Value publisher = null; + for (Statement st : statements) { + if (st.getSubject().equals(repository)) { + if (MAPPING.containsKey(st.getPredicate())) { + repositoryMetadata.getMetadata().put(MAPPING.get(st.getPredicate()), st.getObject().stringValue()); + } + if (st.getPredicate().equals(DCTERMS.PUBLISHER)) { + publisher = st.getObject(); + } + } + } + + if (publisher != null) { + for (Statement st : statements) { + if (st.getSubject().equals(publisher)) { + if (st.getPredicate().equals(FOAF.NAME)) { + repositoryMetadata.getMetadata().put("publisherName", st.getObject().stringValue()); + } + } + } + } + + return repositoryMetadata; + } + + private static Optional findRepository(ArrayList statements) { + for (Statement st : statements) { + if (st.getPredicate().equals(RDF.TYPE) && st.getObject().equals(REPOSITORY)) { + return Optional.of(st.getSubject()); + } + } + return Optional.empty(); + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookMapper.java b/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookMapper.java new file mode 100644 index 000000000..0edbf2120 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookMapper.java @@ -0,0 +1,46 @@ +package nl.dtls.fairdatapoint.service.index.webhook; + +import nl.dtls.fairdatapoint.api.dto.index.webhook.WebhookPayloadDTO; +import nl.dtls.fairdatapoint.entity.index.event.Event; +import nl.dtls.fairdatapoint.entity.index.event.WebhookPing; +import nl.dtls.fairdatapoint.entity.index.event.WebhookTrigger; +import nl.dtls.fairdatapoint.entity.index.webhook.Webhook; +import nl.dtls.fairdatapoint.entity.index.webhook.WebhookEvent; +import org.springframework.security.core.Authentication; +import org.springframework.stereotype.Service; + +import javax.servlet.http.HttpServletRequest; +import java.time.Instant; +import java.util.UUID; + +@Service +public class WebhookMapper { + + private static final Integer VERSION = 1; + + public Event toTriggerEvent(Webhook webhook, WebhookEvent webhookEvent, Event triggerEvent) { + var webhookTrigger = new WebhookTrigger(); + webhookTrigger.setWebhook(webhook); + webhookTrigger.setMatchedEvent(webhookEvent); + return new Event(VERSION, webhookTrigger, triggerEvent); + } + + public Event toPingEvent(HttpServletRequest request, Authentication authentication, UUID webhookUuid) { + var webhookPing = new WebhookPing(); + webhookPing.setWebhookUuid(webhookUuid); + webhookPing.setRemoteAddr(request.getRemoteAddr()); + webhookPing.setTokenName(authentication.getName()); + return new Event(VERSION, webhookPing); + } + + public WebhookPayloadDTO toWebhookPayloadDTO(Event event) { + WebhookPayloadDTO webhookPayload = new WebhookPayloadDTO(); + webhookPayload.setEvent(event.getWebhookTrigger().getMatchedEvent()); + webhookPayload.setClientUrl(event.getRelatedTo().getClientUrl()); + webhookPayload.setSecret(event.getWebhookTrigger().getWebhook().getSecret()); + webhookPayload.setUuid(event.getUuid().toString()); + webhookPayload.setTimestamp(Instant.now().toString()); + return webhookPayload; + } + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookService.java b/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookService.java new file mode 100644 index 000000000..dc706f3ed --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookService.java @@ -0,0 +1,145 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.service.index.webhook; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import nl.dtls.fairdatapoint.api.dto.index.webhook.WebhookPayloadDTO; +import nl.dtls.fairdatapoint.database.mongo.repository.EventRepository; +import nl.dtls.fairdatapoint.database.mongo.repository.WebhookRepository; +import nl.dtls.fairdatapoint.entity.exception.ResourceNotFoundException; +import nl.dtls.fairdatapoint.entity.index.config.EventsConfig; +import nl.dtls.fairdatapoint.entity.index.event.Event; +import nl.dtls.fairdatapoint.entity.index.webhook.Webhook; +import nl.dtls.fairdatapoint.entity.index.webhook.WebhookEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Service; + +import javax.servlet.http.HttpServletRequest; +import java.security.NoSuchAlgorithmException; +import java.util.Optional; +import java.util.UUID; + + +@Service +public class WebhookService { + private static final Logger logger = LoggerFactory.getLogger(WebhookService.class); + + @Autowired + private WebhookMapper webhookMapper; + + @Autowired + private ObjectMapper objectMapper; + + @Autowired + WebhookRepository webhookRepository; + + @Autowired + EventRepository eventRepository; + + @Autowired + private EventsConfig eventsConfig; + + private static final String SECRET_PLACEHOLDER = "*** HIDDEN ***"; + + public void processWebhookTrigger(Event event) { + event.execute(); + eventRepository.save(event); + WebhookPayloadDTO webhookPayload = webhookMapper.toWebhookPayloadDTO(event); + try { + String payloadWithSecret = objectMapper.writeValueAsString(webhookPayload); + String signature = WebhookUtils.computeHashSignature(payloadWithSecret); + webhookPayload.setSecret(SECRET_PLACEHOLDER); + String payloadWithoutSecret = objectMapper.writeValueAsString(webhookPayload); + WebhookUtils.postWebhook(event, eventsConfig.getRetrievalTimeout(), payloadWithoutSecret, signature); + } catch (JsonProcessingException e) { + logger.error("Failed to convert webhook payload to string"); + } catch (NoSuchAlgorithmException e) { + logger.error("Could not compute SHA-1 signature of payload"); + } + event.finish(); + eventRepository.save(event); + } + + @Async + public void triggerWebhook(Webhook webhook, WebhookEvent webhookEvent, Event triggerEvent) { + Event event = webhookMapper.toTriggerEvent(webhook, webhookEvent, triggerEvent); + processWebhookTrigger(event); + } + + @Async + public void triggerWebhooks(WebhookEvent webhookEvent, Event triggerEvent) { + logger.info("Triggered webhook event {} by event {}", webhookEvent, triggerEvent.getUuid()); + WebhookUtils.filterMatching(webhookRepository.findAll(), webhookEvent, triggerEvent).forEach(webhook -> triggerWebhook(webhook, webhookEvent, triggerEvent)); + } + + public Event handleWebhookPing(HttpServletRequest request, UUID webhookUuid) { + Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); + Optional webhook = webhookRepository.findByUuid(webhookUuid); + Event event = eventRepository.save(webhookMapper.toPingEvent(request, authentication, webhookUuid)); + if (webhook.isEmpty()) { + throw new ResourceNotFoundException("There is no such webhook: " + webhookUuid); + } + return event; + } + + @Async + public void triggerWebhooks(Event triggerEvent) { + switch (triggerEvent.getType()) { + case AdminTrigger: + triggerWebhooks(WebhookEvent.AdminTrigger, triggerEvent); + break; + case IncomingPing: + triggerWebhooks(WebhookEvent.IncomingPing, triggerEvent); + if (triggerEvent.getIncomingPing().getNewEntry()) { + triggerWebhooks(WebhookEvent.NewEntry, triggerEvent); + } + break; + case MetadataRetrieval: + switch (triggerEvent.getRelatedTo().getState()) { + case Valid: + triggerWebhooks(WebhookEvent.EntryValid, triggerEvent); + break; + case Invalid: + triggerWebhooks(WebhookEvent.EntryInvalid, triggerEvent); + break; + case Unreachable: + triggerWebhooks(WebhookEvent.EntryUnreachable, triggerEvent); + break; + default: + logger.warn("Invalid state of MetadataRetrieval: {}", triggerEvent.getRelatedTo().getState()); + } + break; + case WebhookPing: + triggerWebhooks(WebhookEvent.WebhookPing, triggerEvent); + break; + default: + logger.warn("Invalid event type for webhook trigger: {}", triggerEvent.getType()); + } + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookUtils.java b/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookUtils.java new file mode 100644 index 000000000..ae263aa51 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookUtils.java @@ -0,0 +1,101 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.service.index.webhook; + +import nl.dtls.fairdatapoint.entity.index.event.Event; +import nl.dtls.fairdatapoint.entity.index.http.Exchange; +import nl.dtls.fairdatapoint.entity.index.http.ExchangeDirection; +import nl.dtls.fairdatapoint.entity.index.http.ExchangeState; +import nl.dtls.fairdatapoint.entity.index.webhook.Webhook; +import nl.dtls.fairdatapoint.entity.index.webhook.WebhookEvent; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; + +import java.io.IOException; +import java.math.BigInteger; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.List; +import java.util.stream.Stream; + +public class WebhookUtils { + + private static final HttpClient client = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_2) + .followRedirects(HttpClient.Redirect.ALWAYS) + .build(); + + private static boolean webhookMatches(Webhook webhook, WebhookEvent webhookEvent, Event triggerEvent) { + boolean matchEvent = webhook.isAllEvents() || webhook.getEvents().contains(webhookEvent); + boolean matchEntry = + webhook.isAllEntries() || triggerEvent.getRelatedTo() == null || webhook.getEntries().contains(triggerEvent.getRelatedTo().getClientUrl()); + return matchEvent && matchEntry && webhook.isEnabled(); + } + + public static Stream filterMatching(List webhooks, WebhookEvent webhookEvent, + Event triggerEvent) { + return webhooks.parallelStream().filter(webhook -> WebhookUtils.webhookMatches(webhook, webhookEvent, + triggerEvent)); + } + + public static String computeHashSignature(String value) throws NoSuchAlgorithmException { + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + digest.reset(); + digest.update(value.getBytes(StandardCharsets.UTF_8)); + return String.format("sha1=%040x", new BigInteger(1, digest.digest())); + } + + public static void postWebhook(Event event, Duration timeout, String payload, String signature) { + var ex = new Exchange(ExchangeDirection.OUTGOING); + event.getWebhookTrigger().setExchange(ex); + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(event.getWebhookTrigger().getWebhook().getPayloadUrl())) + .timeout(timeout) + .header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON.toString()) + .header("X-Signature", signature) + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + ex.getRequest().setFromHttpRequest(request); + ex.setState(ExchangeState.Requested); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + ex.getResponse().setFromHttpResponse(response); + ex.setState(ExchangeState.Retrieved); + } catch (InterruptedException e) { + ex.setState(ExchangeState.Timeout); + ex.setError("Timeout"); + } catch (IllegalArgumentException e) { + ex.setState(ExchangeState.Failed); + ex.setError("Invalid URI: " + e.getMessage()); + } catch (IOException e) { + ex.setState(ExchangeState.Failed); + ex.setError("IO error: " + e.getMessage()); + } + } +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java b/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java new file mode 100644 index 000000000..1c72eced3 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java @@ -0,0 +1,46 @@ +package nl.dtls.fairdatapoint.service.search; + +import nl.dtls.fairdatapoint.api.dto.search.SearchQueryDTO; +import nl.dtls.fairdatapoint.api.dto.search.SearchResultDTO; +import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; +import nl.dtls.fairdatapoint.database.rdf.repository.generic.GenericMetadataRepository; +import nl.dtls.fairdatapoint.entity.metadata.MetadataState; +import nl.dtls.fairdatapoint.entity.search.SearchResult; +import nl.dtls.fairdatapoint.service.metadata.state.MetadataStateService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.toList; +import static nl.dtls.fairdatapoint.util.ValueFactoryHelper.i; +import static nl.dtls.fairdatapoint.util.ValueFactoryHelper.l; + +@Service +public class SearchService { + + @Autowired + private GenericMetadataRepository metadataRepository; + + @Autowired + private MetadataStateService metadataStateService; + + public List search(SearchQueryDTO reqDto) throws MetadataRepositoryException { + return metadataRepository.findByLiteral(l(reqDto.getQ())) + .stream() + .collect(Collectors.groupingBy(SearchResult::getUri, Collectors.mapping(i -> i, toList()))) + .entrySet() + .stream() + .filter(entry -> !metadataStateService.get(i(entry.getKey())).getState().equals(MetadataState.DRAFT)) + .map(entry -> new SearchResultDTO( + entry.getKey(), + entry.getValue().get(0).getTitle(), + entry.getValue().get(0).getDescription(), + entry.getValue().stream() + .map(SearchResult::getRelation).collect(Collectors.toList()) + )) + .collect(toList()); + } + +} diff --git a/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/src/main/resources/META-INF/additional-spring-configuration-metadata.json index 765671c56..ad1500c57 100644 --- a/src/main/resources/META-INF/additional-spring-configuration-metadata.json +++ b/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -10,6 +10,11 @@ "type": "java.lang.String", "description": "Persistent URL." }, + { + "name": "instance.index", + "type": "java.lang.Boolean", + "description": "Turn on if you want to run FAIR Data Point as FAIR Data Point Index" + }, { "name": "security.jwt.token.secret-key", "type": "java.lang.String", @@ -24,6 +29,31 @@ "name": "ping.enabled", "type": "java.lang.String", "description": "States if 'Call home' feature should be enabled" + }, + { + "name": "fdp-index.api.url", + "type": "java.lang.String", + "description": "Server URL where API is running" + }, + { + "name": "fdp-index.api.title", + "type": "java.lang.String", + "description": "Title for OpenAPI docs" + }, + { + "name": "fdp-index.api.description", + "type": "java.lang.String", + "description": "Description for OpenAPI docs" + }, + { + "name": "fdp-index.api.contactUrl", + "type": "java.lang.String", + "description": "Contact URL for OpenAPI docs" + }, + { + "name": "fdp-index.api.contactName", + "type": "java.lang.String", + "description": "Contact name/label for OpenAPI docs" } ] } \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index f73dd02a3..653e4c8af 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -5,6 +5,17 @@ spring: data: mongodb: uri: mongodb://mongo:27017/fdp + web: + pageable: + default-page-size: 50 + qualifier-delimiter: _ + task: + execution: + pool: + core-size: 2 + max-size: 5 + queue-capacity: 5000 + thread-name-prefix: fdpindex-task- management: health: @@ -45,3 +56,13 @@ threadPoolSize: 4 metadataMetrics: https://purl.org/fair-metrics/FM_F1A: https://www.ietf.org/rfc/rfc3986.txt https://purl.org/fair-metrics/FM_A1.1: https://www.wikidata.org/wiki/Q8777 + +fdp-index: + events: + retrieval: + rateLimitWait: PT10M # 10 minutes (ISO 8601) + timeout: PT1M # 1 minute (ISO 8601) + ping: + validDuration: P7D # 7 days (ISO 8601) + rateLimitDuration: PT6H + rateLimitHits: 10 diff --git a/src/main/resources/nl/dtls/fairdatapoint/database/rdf/repository/common/findEntityByLiteral.sparql b/src/main/resources/nl/dtls/fairdatapoint/database/rdf/repository/common/findEntityByLiteral.sparql new file mode 100644 index 000000000..53987e28a --- /dev/null +++ b/src/main/resources/nl/dtls/fairdatapoint/database/rdf/repository/common/findEntityByLiteral.sparql @@ -0,0 +1,9 @@ +prefix dct: + +SELECT ?entity ?title ?description ?relationPredicate ?relationObject WHERE { + ?entity ?relationPredicate ?relationObject . + ?entity dct:title ?title . + ?entity dct:description ?description . + filter isLiteral(?relationObject) + filter CONTAINS(LCASE(str(?relationObject)), LCASE(str(?query))) +} diff --git a/src/test/java/nl/dtls/fairdatapoint/acceptance/index/admin/List_Trigger_POST.java b/src/test/java/nl/dtls/fairdatapoint/acceptance/index/admin/List_Trigger_POST.java new file mode 100644 index 000000000..11207073a --- /dev/null +++ b/src/test/java/nl/dtls/fairdatapoint/acceptance/index/admin/List_Trigger_POST.java @@ -0,0 +1,186 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.acceptance.index.admin; + +import nl.dtls.fairdatapoint.WebIntegrationTest; +import nl.dtls.fairdatapoint.database.mongo.repository.EventRepository; +import nl.dtls.fairdatapoint.database.mongo.repository.IndexEntryRepository; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; +import nl.dtls.fairdatapoint.entity.index.event.Event; +import nl.dtls.fairdatapoint.entity.index.event.EventType; +import nl.dtls.fairdatapoint.utils.TestIndexEntryFixtures; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; + +import java.net.URI; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; + +@DisplayName("POST /index/admin/trigger") +public class List_Trigger_POST extends WebIntegrationTest { + + @Autowired + private EventRepository eventRepository; + + @Autowired + private IndexEntryRepository indexEntryRepository; + + private final ParameterizedTypeReference responseType = new ParameterizedTypeReference<>() { + }; + + private URI url() { + return URI.create("/index/admin/trigger"); + } + + private URI url(String clientUrl) { + return URI.create("/index/admin/trigger?clientUrl=" + clientUrl); + } + + @Test + @DisplayName("HTTP 403: no token") + public void res403_noToken() { + // GIVEN: prepare data + String clientUrl = "http://example.com"; + + // AND: prepare request + RequestEntity request = RequestEntity + .post(url(clientUrl)) + .build(); + + // WHEN + ResponseEntity result = client.exchange(request, responseType); + + // THEN: + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.FORBIDDEN))); + } + + @Test + @DisplayName("HTTP 403: incorrect token") + public void res403_incorrectToken() { + // GIVEN: prepare data + String clientUrl = "http://example.com"; + + // AND: prepare request + RequestEntity request = RequestEntity + .post(url(clientUrl)) + .header(HttpHeaders.AUTHORIZATION, ALBERT_TOKEN) + .build(); + + // WHEN + ResponseEntity result = client.exchange(request, responseType); + + // THEN: + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.FORBIDDEN))); + } + + @Test + @DisplayName("HTTP 403: non-admin token") + public void res403_nonAdminToken() { + // GIVEN: prepare data + String clientUrl = "http://example.com"; + + // AND: prepare request + RequestEntity request = RequestEntity + .post(url(clientUrl)) + .header(HttpHeaders.AUTHORIZATION, ALBERT_TOKEN) + .build(); + + // WHEN + ResponseEntity result = client.exchange(request, responseType); + + // THEN: + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.FORBIDDEN))); + } + + @Test + @DisplayName("HTTP 204: trigger one") + public void res204_triggerOne() { + // GIVEN: prepare data + IndexEntry entry = TestIndexEntryFixtures.entryExample(); + indexEntryRepository.save(entry); + + // AND: prepare request + RequestEntity request = RequestEntity + .post(url(entry.getUuid())) + .header(HttpHeaders.AUTHORIZATION, ADMIN_TOKEN) + .build(); + + // WHEN + ResponseEntity result = client.exchange(request, responseType); + List events = eventRepository.getAllByType(EventType.AdminTrigger); + + // THEN: + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.NO_CONTENT))); + assertThat("One AdminTrigger event is created", events.size(), is(equalTo(1))); + assertThat("Records correct client URL", events.get(0).getAdminTrigger().getClientUrl(), + is(equalTo(entry.getUuid()))); + } + + @Test + @DisplayName("HTTP 204: trigger all") + public void res204_triggerAll() { + // GIVEN: prepare request + RequestEntity request = RequestEntity + .post(url()) + .header(HttpHeaders.AUTHORIZATION, ADMIN_TOKEN) + .build(); + + // WHEN + ResponseEntity result = client.exchange(request, responseType); + List events = eventRepository.getAllByType(EventType.AdminTrigger); + + // THEN: + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.NO_CONTENT))); + assertThat("One AdminTrigger event is created", events.size(), is(equalTo(1))); + assertThat("Records correct client URL as null", events.get(0).getAdminTrigger().getClientUrl(), + is(equalTo(null))); + } + + @Test + @DisplayName("HTTP 404: trigger non-existing") + public void res404_triggerOne() { + // GIVEN: prepare data + IndexEntry entry = TestIndexEntryFixtures.entryExample(); + + // AND: prepare request + RequestEntity request = RequestEntity + .post(url(entry.getUuid())) + .header(HttpHeaders.AUTHORIZATION, ADMIN_TOKEN) + .build(); + + // WHEN + ResponseEntity result = client.exchange(request, responseType); + + // THEN: + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.NOT_FOUND))); + } +} diff --git a/src/test/java/nl/dtls/fairdatapoint/acceptance/index/entry/List_All_GET.java b/src/test/java/nl/dtls/fairdatapoint/acceptance/index/entry/List_All_GET.java new file mode 100644 index 000000000..06821ff6c --- /dev/null +++ b/src/test/java/nl/dtls/fairdatapoint/acceptance/index/entry/List_All_GET.java @@ -0,0 +1,137 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.acceptance.index.entry; + +import nl.dtls.fairdatapoint.WebIntegrationTest; +import nl.dtls.fairdatapoint.api.dto.index.entry.IndexEntryDTO; +import nl.dtls.fairdatapoint.database.mongo.repository.IndexEntryRepository; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; +import nl.dtls.fairdatapoint.utils.TestIndexEntryFixtures; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; + +import java.net.URI; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.hamcrest.core.IsNull.notNullValue; + +@DisplayName("GET /index/entries/all") +public class List_All_GET extends WebIntegrationTest { + + @Autowired + private IndexEntryRepository indexEntryRepository; + + private final ParameterizedTypeReference> responseType = new ParameterizedTypeReference<>() { + }; + + private URI url() { + return URI.create("/index/entries/all"); + } + + @Test + @DisplayName("HTTP 200: list empty") + public void res200_listEmpty() { + // GIVEN: prepare data + indexEntryRepository.deleteAll(); + + // AND: prepare request + RequestEntity request = RequestEntity + .get(url()) + .accept(MediaType.APPLICATION_JSON) + .build(); + + // WHEN: + ResponseEntity> result = client.exchange(request, responseType); + + // THEN: + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.OK))); + assertThat("Response body is not null", result.getBody(), is(notNullValue())); + assertThat("There are no entries in the response", result.getBody().size(), is(equalTo(0))); + } + + @Test + @DisplayName("HTTP 200: list few") + public void res200_listFew() { + // GIVEN: prepare data + indexEntryRepository.deleteAll(); + List entries = TestIndexEntryFixtures.entriesFew(); + indexEntryRepository.saveAll(entries); + int size = 9; + + // AND: prepare request + RequestEntity request = RequestEntity + .get(url()) + .accept(MediaType.APPLICATION_JSON) + .build(); + + // WHEN + ResponseEntity> result = client.exchange(request, responseType); + + // THEN + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.OK))); + assertThat("Response body is not null", result.getBody(), is(notNullValue())); + assertThat("Correct number of entries is in the response", result.getBody().size(), + is(equalTo(entries.size()))); + for (int i = 0; i < entries.size(); i++) { + assertThat("Entry matches: " + entries.get(i).getClientUrl(), result.getBody().get(i).getClientUrl(), + is(equalTo(entries.get(i).getClientUrl()))); + } + } + + @Test + @DisplayName("HTTP 200: list many") + public void res200_listMany() { + // GIVEN: prepare data + indexEntryRepository.deleteAll(); + List entries = TestIndexEntryFixtures.entriesN(300); + indexEntryRepository.saveAll(entries); + + // AND: prepare request + RequestEntity request = RequestEntity + .get(url()) + .accept(MediaType.APPLICATION_JSON) + .build(); + + // WHEN + ResponseEntity> result = client.exchange(request, responseType); + + // THEN + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.OK))); + assertThat("Response body is not null", result.getBody(), is(notNullValue())); + assertThat("Correct number of entries is in the response", result.getBody().size(), + is(equalTo(entries.size()))); + for (int i = 0; i < entries.size(); i++) { + assertThat("Entry matches: " + entries.get(i).getClientUrl(), result.getBody().get(i).getClientUrl(), + is(equalTo(entries.get(i).getClientUrl()))); + } + } +} diff --git a/src/test/java/nl/dtls/fairdatapoint/acceptance/index/entry/List_GET.java b/src/test/java/nl/dtls/fairdatapoint/acceptance/index/entry/List_GET.java new file mode 100644 index 000000000..40e834e7d --- /dev/null +++ b/src/test/java/nl/dtls/fairdatapoint/acceptance/index/entry/List_GET.java @@ -0,0 +1,226 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.acceptance.index.entry; + +import nl.dtls.fairdatapoint.WebIntegrationTest; +import nl.dtls.fairdatapoint.api.dto.index.entry.IndexEntryDTO; +import nl.dtls.fairdatapoint.database.mongo.repository.IndexEntryRepository; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; +import nl.dtls.fairdatapoint.utils.CustomPageImpl; +import nl.dtls.fairdatapoint.utils.TestIndexEntryFixtures; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.web.util.UriComponentsBuilder; + +import java.net.URI; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.hamcrest.core.IsNull.notNullValue; + +@DisplayName("GET /index/entries") +public class List_GET extends WebIntegrationTest { + + @Autowired + private IndexEntryRepository indexEntryRepository; + + private final ParameterizedTypeReference> responseType = + new ParameterizedTypeReference<>() { + }; + + private URI url() { + return URI.create("/index/entries"); + } + + private URI urlWithPage(int page) { + return UriComponentsBuilder.fromUri(url()) + .queryParam("page", page) + .build().toUri(); + } + + private URI urlWithPageSize(int page, int size) { + return UriComponentsBuilder.fromUri(url()) + .queryParam("page", page) + .queryParam("size", size) + .build().toUri(); + } + + @Test + @DisplayName("HTTP 200: page empty") + public void res200_pageEmpty() { + // GIVEN: prepare data + indexEntryRepository.deleteAll(); + + // AND: prepare request + RequestEntity request = RequestEntity + .get(url()) + .accept(MediaType.APPLICATION_JSON) + .build(); + + // WHEN + ResponseEntity> result = client.exchange(request, responseType); + + // THEN + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.OK))); + assertThat("Response body is not null", result.getBody(), is(notNullValue())); + assertThat("Current page is the first page", result.getBody().isFirst(), is(Boolean.TRUE)); + assertThat("Current page is the last page", result.getBody().isLast(), is(Boolean.TRUE)); + assertThat("Current page is empty", result.getBody().isEmpty(), is(Boolean.TRUE)); + assertThat("Number of pages is 0", result.getBody().getTotalPages(), is(equalTo(0))); + assertThat("Number of elements is 0", result.getBody().getTotalElements(), is(equalTo(0L))); + assertThat("There are no entries in the response", result.getBody().getContent().size(), is(equalTo(0))); + } + + @Test + @DisplayName("HTTP 200: out-of-bounds page") + public void res200_outOfBoundsPage() { + // GIVEN: prepare data + indexEntryRepository.deleteAll(); + + // AND: prepare request + RequestEntity request = RequestEntity + .get(urlWithPage(7)) + .accept(MediaType.APPLICATION_JSON) + .build(); + + // WHEN: + ResponseEntity> result = client.exchange(request, responseType); + + // THEN: + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.OK))); + assertThat("Response body is not null", result.getBody(), is(notNullValue())); + assertThat("Current page is not the first page", result.getBody().isFirst(), is(Boolean.FALSE)); + assertThat("Current page is the last page", result.getBody().isLast(), is(Boolean.TRUE)); + assertThat("Current page is empty", result.getBody().isEmpty(), is(Boolean.TRUE)); + assertThat("Number of pages is 0", result.getBody().getTotalPages(), is(equalTo(0))); + assertThat("Number of elements is 0", result.getBody().getTotalElements(), is(equalTo(0L))); + assertThat("There are no entries in the response", result.getBody().getContent().size(), is(equalTo(0))); + } + + @Test + @DisplayName("HTTP 200: page few") + public void res200_pageFew() { + // GIVEN: prepare data + indexEntryRepository.deleteAll(); + List entries = TestIndexEntryFixtures.entriesFew(); + indexEntryRepository.saveAll(entries); + + // AND: prepare request + RequestEntity request = RequestEntity + .get(url()) + .accept(MediaType.APPLICATION_JSON) + .build(); + + // WHEN + ResponseEntity> result = client.exchange(request, responseType); + + // THEN + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.OK))); + assertThat("Response body is not null", result.getBody(), is(notNullValue())); + assertThat("Current page is the first page", result.getBody().isFirst(), is(Boolean.TRUE)); + assertThat("Current page is the last page", result.getBody().isLast(), is(Boolean.TRUE)); + assertThat("Current page is not empty", result.getBody().isEmpty(), is(Boolean.FALSE)); + assertThat("Number of pages is 1", result.getBody().getTotalPages(), is(equalTo(1))); + assertThat("Number of elements is correct", result.getBody().getTotalElements(), + is(equalTo(Integer.toUnsignedLong(entries.size())))); + assertThat("There is correct number of entries in the response", result.getBody().getContent().size(), + is(equalTo(entries.size()))); + for (int i = 0; i < entries.size(); i++) { + assertThat("Entry matches: " + entries.get(i).getClientUrl(), + result.getBody().getContent().get(i).getClientUrl(), is(equalTo(entries.get(i).getClientUrl()))); + } + } + + @Test + @DisplayName("HTTP 200: page many (middle)") + public void res200_pageManyMiddle() { + // GIVEN: prepare data + long items = 300L; + int size = 30; + int page = 3; + indexEntryRepository.deleteAll(); + List entries = TestIndexEntryFixtures.entriesN(items); + indexEntryRepository.saveAll(entries); + + // AND (prepare request) + RequestEntity request = RequestEntity + .get(urlWithPageSize(page, size)) + .accept(MediaType.APPLICATION_JSON) + .build(); + + // WHEN + ResponseEntity> result = client.exchange(request, responseType); + + // THEN + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.OK))); + assertThat("Response body is not null", result.getBody(), is(notNullValue())); + assertThat("Current page is the first page", result.getBody().isFirst(), is(Boolean.FALSE)); + assertThat("Current page is the last page", result.getBody().isLast(), is(Boolean.FALSE)); + assertThat("Current page is not empty", result.getBody().isEmpty(), is(Boolean.FALSE)); + assertThat("Number of pages is correct", result.getBody().getTotalPages(), is(equalTo(10))); + assertThat("Number of elements is correct", result.getBody().getTotalElements(), is(equalTo(items))); + assertThat("There is correct number of entries in the response", result.getBody().getContent().size(), + is(equalTo(size))); + } + + @Test + @DisplayName("HTTP 200: page many (last)") + public void res200_pageManyLast() { + // GIVEN (prepare data) + long items = 666; + int size = 300; + int lastSize = 66; + int page = 2; + indexEntryRepository.deleteAll(); + List entries = TestIndexEntryFixtures.entriesN(items); + indexEntryRepository.saveAll(entries); + + // AND (prepare request) + RequestEntity request = RequestEntity + .get(urlWithPageSize(page, size)) + .accept(MediaType.APPLICATION_JSON) + .build(); + + // WHEN + ResponseEntity> result = client.exchange(request, responseType); + + // THEN + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.OK))); + assertThat("Response body is not null", result.getBody(), is(notNullValue())); + assertThat("Current page is the first page", result.getBody().isFirst(), is(Boolean.FALSE)); + assertThat("Current page is the last page", result.getBody().isLast(), is(Boolean.TRUE)); + assertThat("Current page is not empty", result.getBody().isEmpty(), is(Boolean.FALSE)); + assertThat("Number of pages is correct", result.getBody().getTotalPages(), is(equalTo(3))); + assertThat("Number of elements is correct", result.getBody().getTotalElements(), is(equalTo(items))); + assertThat("There is correct number of entries in the response", result.getBody().getContent().size(), + is(equalTo(lastSize))); + } +} diff --git a/src/test/java/nl/dtls/fairdatapoint/acceptance/index/entry/List_Info_GET.java b/src/test/java/nl/dtls/fairdatapoint/acceptance/index/entry/List_Info_GET.java new file mode 100644 index 000000000..29c2ce0c7 --- /dev/null +++ b/src/test/java/nl/dtls/fairdatapoint/acceptance/index/entry/List_Info_GET.java @@ -0,0 +1,81 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.acceptance.index.entry; + +import nl.dtls.fairdatapoint.WebIntegrationTest; +import nl.dtls.fairdatapoint.api.dto.index.entry.IndexEntryInfoDTO; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; + +import java.net.URI; +import java.util.HashMap; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.hamcrest.core.IsNull.notNullValue; + +@DisplayName("GET /index/entries/info") +public class List_Info_GET extends WebIntegrationTest { + + private final ParameterizedTypeReference responseType = new ParameterizedTypeReference<>() { + }; + + private URI url() { + return URI.create("/index/entries/info"); + } + + @Test + @DisplayName("HTTP 200") + public void res200_listMany() { + // GIVEN: Prepare request + RequestEntity request = RequestEntity + .get(url()) + .accept(MediaType.APPLICATION_JSON) + .build(); + + // AND: Prepare expectation + IndexEntryInfoDTO expDto = new IndexEntryInfoDTO(new HashMap<>() {{ + put("ALL", 6L); + put("ACTIVE", 1L); + put("INACTIVE", 2L); + put("UNKNOWN", 1L); + put("INVALID", 1L); + put("UNREACHABLE", 1L); + }}); + + // WHEN: + ResponseEntity result = client.exchange(request, responseType); + + // THEN: + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.OK))); + assertThat("Response body is not null", result.getBody(), is(notNullValue())); + assertThat("Correct number of entries is in the response", result.getBody(), + is(equalTo(expDto))); + } +} diff --git a/src/test/java/nl/dtls/fairdatapoint/acceptance/index/ping/List_POST.java b/src/test/java/nl/dtls/fairdatapoint/acceptance/index/ping/List_POST.java new file mode 100644 index 000000000..a3d0a29f8 --- /dev/null +++ b/src/test/java/nl/dtls/fairdatapoint/acceptance/index/ping/List_POST.java @@ -0,0 +1,176 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.acceptance.index.ping; + +import nl.dtls.fairdatapoint.WebIntegrationTest; +import nl.dtls.fairdatapoint.api.dto.index.ping.PingDTO; +import nl.dtls.fairdatapoint.database.mongo.repository.IndexEntryRepository; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; +import nl.dtls.fairdatapoint.utils.TestIndexEntryFixtures; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; + +import java.net.URI; +import java.util.HashMap; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; + +@DisplayName("POST /index") +public class List_POST extends WebIntegrationTest { + + @Autowired + private IndexEntryRepository indexEntryRepository; + + @Autowired + private MongoTemplate mongoTemplate; + + private final ParameterizedTypeReference responseType = new ParameterizedTypeReference<>() { + }; + + private URI url() { + return URI.create("/index"); + } + + private PingDTO reqDTO(String clientUrl) { + PingDTO dto = new PingDTO(); + dto.setClientUrl(clientUrl); + return dto; + } + + @Test + @DisplayName("HTTP 204: new entry") + public void res204_newEntry() { + // GIVEN: prepare data + String clientUrl = "http://example.com"; + PingDTO reqDto = reqDTO(clientUrl); + + // AND: prepare request + RequestEntity request = RequestEntity + .post(url()) + .accept(MediaType.APPLICATION_JSON) + .body(reqDto); + + // WHEN: + assertThat("Entry does not exist before the ping", + indexEntryRepository.findByClientUrl(clientUrl).isPresent(), is(Boolean.FALSE)); + ResponseEntity result = client.exchange(request, responseType); + + // THEN: + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.NO_CONTENT))); + assertThat("Entry exists after the ping", indexEntryRepository.findByClientUrl(clientUrl).isPresent(), + is(Boolean.TRUE)); + } + + @Test + @DisplayName("HTTP 204: existing entry") + public void res204_existingEnty() { + // GIVEN: prepare data + IndexEntry indexEntry = TestIndexEntryFixtures.entryExample(); + String clientUrl = indexEntry.getClientUrl(); + PingDTO reqDto = reqDTO(clientUrl); + + // AND: prepare request + RequestEntity request = RequestEntity + .post(url()) + .accept(MediaType.APPLICATION_JSON) + .body(reqDto); + + // WHEN: + indexEntryRepository.save(indexEntry); + assertThat("Entry exists before the ping", indexEntryRepository.findByClientUrl(clientUrl).isPresent(), + is(Boolean.TRUE)); + ResponseEntity result = client.exchange(request, responseType); + + // THEN: + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.NO_CONTENT))); + assertThat("Entry exists after the ping", indexEntryRepository.findByClientUrl(clientUrl).isPresent(), + is(Boolean.TRUE)); + } + + @Test + @DisplayName("HTTP 400: null client url") + public void res400_nullClientUrl() { + // GIVEN (prepare data) + PingDTO reqDto = reqDTO(null); + + // AND (prepare request) + RequestEntity request = RequestEntity + .post(url()) + .accept(MediaType.APPLICATION_JSON) + .body(reqDto); + + // WHEN + ResponseEntity result = client.exchange(request, responseType); + + // THEN + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.BAD_REQUEST))); + } + + @Test + @DisplayName("HTTP 400: non-URL client url") + public void res400_nonUrlClientUrl() { + // GIVEN (prepare data) + PingDTO reqDto = reqDTO("testing"); + + // AND (prepare request) + RequestEntity request = RequestEntity + .post(url()) + .accept(MediaType.APPLICATION_JSON) + .body(reqDto); + + // WHEN + ResponseEntity result = client.exchange(request, responseType); + + // THEN + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.BAD_REQUEST))); + } + + @Test + @DisplayName("HTTP 400: different body") + public void res400_differentBody() { + // GIVEN (prepare data) + HashMap dummyData = new HashMap<>(); + dummyData.put("content", "http://test"); + + // AND (prepare request) + RequestEntity> request = RequestEntity + .post(url()) + .accept(MediaType.APPLICATION_JSON) + .body(dummyData); + + // WHEN + ResponseEntity result = client.exchange(request, responseType); + + // THEN + assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.BAD_REQUEST))); + } +} diff --git a/src/test/java/nl/dtls/fairdatapoint/acceptance/search/List_POST.java b/src/test/java/nl/dtls/fairdatapoint/acceptance/search/List_POST.java new file mode 100644 index 000000000..ea8d74287 --- /dev/null +++ b/src/test/java/nl/dtls/fairdatapoint/acceptance/search/List_POST.java @@ -0,0 +1,52 @@ +package nl.dtls.fairdatapoint.acceptance.search; + +import nl.dtls.fairdatapoint.WebIntegrationTest; +import nl.dtls.fairdatapoint.api.dto.search.SearchQueryDTO; +import nl.dtls.fairdatapoint.api.dto.search.SearchResultDTO; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.*; + +import java.net.URI; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; + +@DisplayName("POST /search") +public class List_POST extends WebIntegrationTest { + + private URI url() { + return URI.create("/search"); + } + + private SearchQueryDTO reqDto(String query) { + return new SearchQueryDTO(query); + } + + @Test + @DisplayName("HTTP 200") + public void res200() { + // GIVEN: Prepare data + SearchQueryDTO reqDto = reqDto("catalog"); + + // AND: Prepare request + RequestEntity request = RequestEntity + .post(url()) + .header(HttpHeaders.AUTHORIZATION, ADMIN_TOKEN) + .accept(MediaType.APPLICATION_JSON) + .body(reqDto); + ParameterizedTypeReference> responseType = new ParameterizedTypeReference<>() { + }; + + // WHEN: + ResponseEntity> result = client.exchange(request, responseType); + + // THEN: + assertThat(result.getStatusCode(), is(equalTo(HttpStatus.OK))); + assertThat(result.getBody().size(), is(equalTo(1))); + } + +} \ No newline at end of file diff --git a/src/test/java/nl/dtls/fairdatapoint/utils/CustomPageImpl.java b/src/test/java/nl/dtls/fairdatapoint/utils/CustomPageImpl.java new file mode 100644 index 000000000..9f525e943 --- /dev/null +++ b/src/test/java/nl/dtls/fairdatapoint/utils/CustomPageImpl.java @@ -0,0 +1,62 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.utils; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +import java.util.List; + +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class CustomPageImpl { + + private int totalPages; + private long totalElements; + private boolean first; + private CustomSort sort; + private CustomPageable pageable; + private int number; + private int numberOfElements; + private boolean last; + private int size; + private List content; + private boolean empty; + + @Data + @JsonIgnoreProperties(ignoreUnknown = true) + public static class CustomPageable { + private int page; + private int size; + private CustomSort sort; + } + + @Data + @JsonIgnoreProperties(ignoreUnknown = true) + public static class CustomSort { + private boolean sorted; + private boolean unsorted; + private boolean empty; + } + +} diff --git a/src/test/java/nl/dtls/fairdatapoint/utils/TestIndexEntryFixtures.java b/src/test/java/nl/dtls/fairdatapoint/utils/TestIndexEntryFixtures.java new file mode 100644 index 000000000..3216d75a0 --- /dev/null +++ b/src/test/java/nl/dtls/fairdatapoint/utils/TestIndexEntryFixtures.java @@ -0,0 +1,70 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.utils; + +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; +import nl.dtls.fairdatapoint.entity.index.entry.IndexEntryState; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +public class TestIndexEntryFixtures { + + private static IndexEntry newIndexEntry(String uuid, String clientUrl, Instant timestamp) { + IndexEntry indexEntry = new IndexEntry(); + indexEntry.setUuid(uuid); + indexEntry.setClientUrl(clientUrl); + indexEntry.setModificationTime(timestamp); + indexEntry.setRegistrationTime(timestamp); + indexEntry.setState(IndexEntryState.Invalid); + return indexEntry; + } + + public static IndexEntry entryExample() { + return newIndexEntry("7663c0c2-2b9d-4787-968d-d284ff3fc5bd", "http://example.com", Instant.now()); + } + + public static List entriesFew() { + Instant ref = Instant.now(); + return Arrays.asList( + newIndexEntry("09200532-18b4-4721-86dd-fbfa13ec78c3", "http://example.com", ref), + newIndexEntry("b6cfa934-dc67-4b88-b8f9-c63448c8272c", "http://test.com", ref.minusSeconds(1)), + newIndexEntry("da9ddfb8-6fdb-41b1-889e-387c8cbafc39", "http://localhost", ref.minusSeconds(2)) + ); + } + + public static List entriesN(long n) { + ArrayList entries = new ArrayList<>(); + Instant ref = Instant.now(); + for (int i = 0; i < n; i++) { + Instant entryTime = ref.minusSeconds(i); + entries.add(newIndexEntry(UUID.randomUUID().toString(), "http://example" + i + ".com", + entryTime)); + } + return entries; + } + +} diff --git a/src/test/resources/application-testing.yml b/src/test/resources/application-testing.yml index cd2f909ac..9a9beeabf 100644 --- a/src/test/resources/application-testing.yml +++ b/src/test/resources/application-testing.yml @@ -12,4 +12,13 @@ spring: security: jwt: token: - expiration: 9999 \ No newline at end of file + expiration: 9999 + + +fdp-index: + events: + retrieval: + rateLimitWait: PT10M # 10 minutes (ISO 8601) + timeout: PT1M # 1 minute (ISO 8601) + ping: + validDuration: P7D # 7 days (ISO 8601) \ No newline at end of file From 3c55eaf1cd94909bb2be48cee25a21fb99e80f2a Mon Sep 17 00:00:00 2001 From: Vojtech Knaisl Date: Tue, 29 Sep 2020 09:36:19 +0200 Subject: [PATCH 03/12] Add possibility to change profile and password for current user --- .../api/controller/user/UserController.java | 32 ++++-- .../api/dto/user/UserProfileChangeDTO.java | 26 +++++ .../service/user/UserMapper.java | 10 ++ .../service/user/UserService.java | 43 ++++--- .../service/user/UserValidator.java | 26 +++++ .../fairdatapoint/acceptance/user/Common.java | 7 ++ .../acceptance/user/Detail_Current_PUT.java | 108 ++++++++++++++++++ .../user/Detail_Current_Password_PUT.java | 86 ++++++++++++++ 8 files changed, 316 insertions(+), 22 deletions(-) create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/dto/user/UserProfileChangeDTO.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/user/UserValidator.java create mode 100644 src/test/java/nl/dtls/fairdatapoint/acceptance/user/Detail_Current_PUT.java create mode 100644 src/test/java/nl/dtls/fairdatapoint/acceptance/user/Detail_Current_Password_PUT.java diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/user/UserController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/user/UserController.java index 56e061598..b223fe046 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/controller/user/UserController.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/user/UserController.java @@ -22,10 +22,7 @@ */ package nl.dtls.fairdatapoint.api.controller.user; -import nl.dtls.fairdatapoint.api.dto.user.UserChangeDTO; -import nl.dtls.fairdatapoint.api.dto.user.UserCreateDTO; -import nl.dtls.fairdatapoint.api.dto.user.UserDTO; -import nl.dtls.fairdatapoint.api.dto.user.UserPasswordDTO; +import nl.dtls.fairdatapoint.api.dto.user.*; import nl.dtls.fairdatapoint.entity.exception.ForbiddenException; import nl.dtls.fairdatapoint.entity.exception.ResourceNotFoundException; import nl.dtls.fairdatapoint.service.user.UserService; @@ -61,8 +58,7 @@ public ResponseEntity createUser(@RequestBody @Valid UserCreateDTO reqD } @RequestMapping(value = "/current", method = RequestMethod.GET) - public ResponseEntity getUserCurrent() - throws ResourceNotFoundException { + public ResponseEntity getUserCurrent() throws ResourceNotFoundException { Optional oDto = userService.getCurrentUser(); if (oDto.isPresent()) { return new ResponseEntity<>(oDto.get(), HttpStatus.OK); @@ -72,8 +68,7 @@ public ResponseEntity getUserCurrent() } @RequestMapping(value = "/{uuid}", method = RequestMethod.GET) - public ResponseEntity getUser(@PathVariable final String uuid) - throws ResourceNotFoundException { + public ResponseEntity getUser(@PathVariable final String uuid) throws ResourceNotFoundException { Optional oDto = userService.getUserByUuid(uuid); if (oDto.isPresent()) { return new ResponseEntity<>(oDto.get(), HttpStatus.OK); @@ -82,6 +77,16 @@ public ResponseEntity getUser(@PathVariable final String uuid) } } + @RequestMapping(value = "/current", method = RequestMethod.PUT) + public ResponseEntity putUserCurrent(@RequestBody @Valid UserProfileChangeDTO reqDto) throws ResourceNotFoundException { + Optional oDto = userService.updateCurrentUser(reqDto); + if (oDto.isPresent()) { + return new ResponseEntity<>(oDto.get(), HttpStatus.OK); + } else { + throw new ForbiddenException("You have to be login at first"); + } + } + @RequestMapping(value = "/{uuid}", method = RequestMethod.PUT) public ResponseEntity putUser(@PathVariable final String uuid, @RequestBody @Valid UserChangeDTO reqDto) throws ResourceNotFoundException { @@ -93,6 +98,17 @@ public ResponseEntity putUser(@PathVariable final String uuid, } } + @RequestMapping(value = "/current/password", method = RequestMethod.PUT) + public ResponseEntity putUserCurrentPassword(@RequestBody @Valid UserPasswordDTO reqDto) + throws ResourceNotFoundException { + Optional oDto = userService.updatePasswordForCurrentUser(reqDto); + if (oDto.isPresent()) { + return new ResponseEntity<>(oDto.get(), HttpStatus.OK); + } else { + throw new ForbiddenException("You have to be login at first"); + } + } + @RequestMapping(value = "/{uuid}/password", method = RequestMethod.PUT) public ResponseEntity putUserPassword(@PathVariable final String uuid, @RequestBody @Valid UserPasswordDTO reqDto) throws ResourceNotFoundException { diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/user/UserProfileChangeDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/user/UserProfileChangeDTO.java new file mode 100644 index 000000000..76c439c53 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/user/UserProfileChangeDTO.java @@ -0,0 +1,26 @@ +package nl.dtls.fairdatapoint.api.dto.user; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import javax.validation.constraints.NotBlank; + +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +public class UserProfileChangeDTO { + + @NotBlank + protected String firstName; + + @NotBlank + protected String lastName; + + @NotBlank + protected String email; + + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/user/UserMapper.java b/src/main/java/nl/dtls/fairdatapoint/service/user/UserMapper.java index 829f131bf..94107ce2f 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/user/UserMapper.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/user/UserMapper.java @@ -75,6 +75,16 @@ public User fromChangeDTO(UserChangeDTO dto, User user) { .build(); } + public User fromProfileChangeDTO(UserProfileChangeDTO dto, User user) { + return + user + .toBuilder() + .firstName(dto.getFirstName()) + .lastName(dto.getLastName()) + .email(dto.getEmail()) + .build(); + } + public User fromPasswordDTO(UserPasswordDTO reqDto, User user) { return user diff --git a/src/main/java/nl/dtls/fairdatapoint/service/user/UserService.java b/src/main/java/nl/dtls/fairdatapoint/service/user/UserService.java index 322a1f355..97e09d9b0 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/user/UserService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/user/UserService.java @@ -22,12 +22,8 @@ */ package nl.dtls.fairdatapoint.service.user; -import nl.dtls.fairdatapoint.api.dto.user.UserChangeDTO; -import nl.dtls.fairdatapoint.api.dto.user.UserCreateDTO; -import nl.dtls.fairdatapoint.api.dto.user.UserDTO; -import nl.dtls.fairdatapoint.api.dto.user.UserPasswordDTO; +import nl.dtls.fairdatapoint.api.dto.user.*; import nl.dtls.fairdatapoint.database.mongo.repository.UserRepository; -import nl.dtls.fairdatapoint.entity.exception.ValidationException; import nl.dtls.fairdatapoint.entity.user.User; import nl.dtls.fairdatapoint.service.member.MemberService; import org.springframework.beans.factory.annotation.Autowired; @@ -39,7 +35,6 @@ import java.util.Optional; import java.util.UUID; -import static java.lang.String.format; import static java.util.Optional.empty; import static java.util.Optional.of; import static java.util.stream.Collectors.toList; @@ -53,6 +48,9 @@ public class UserService { @Autowired private UserMapper userMapper; + @Autowired + private UserValidator userValidator; + @Autowired private MemberService memberService; @@ -87,10 +85,7 @@ public Optional getCurrentUser() { @PreAuthorize("hasRole('ADMIN')") public UserDTO createUser(UserCreateDTO reqDto) { - Optional oUser = userRepository.findByEmail(reqDto.getEmail()); - if (oUser.isPresent()) { - throw new ValidationException(format("Email '%s' is already taken", reqDto.getEmail())); - } + userValidator.validateEmail(null, reqDto.getEmail()); String uuid = UUID.randomUUID().toString(); User user = userMapper.fromCreateDTO(reqDto, uuid); userRepository.save(user); @@ -99,20 +94,29 @@ public UserDTO createUser(UserCreateDTO reqDto) { @PreAuthorize("hasRole('ADMIN')") public Optional updateUser(String uuid, UserChangeDTO reqDto) { - Optional oUserEmail = userRepository.findByEmail(reqDto.getEmail()); - if (oUserEmail.isPresent() && !uuid.equals(oUserEmail.get().getUuid())) { - throw new ValidationException(format("Email '%s' is already taken", reqDto.getEmail())); - } Optional oUser = userRepository.findByUuid(uuid); if (oUser.isEmpty()) { return empty(); } User user = oUser.get(); + userValidator.validateEmail(uuid, reqDto.getEmail()); User updatedUser = userMapper.fromChangeDTO(reqDto, user); userRepository.save(updatedUser); return of(userMapper.toDTO(updatedUser)); } + public Optional updateCurrentUser(UserProfileChangeDTO reqDto) { + Optional oUser = getCurrentUserUuid().flatMap(uuid -> userRepository.findByUuid(uuid)); + if (oUser.isEmpty()) { + return empty(); + } + User user = oUser.get(); + userValidator.validateEmail(user.getUuid(), reqDto.getEmail()); + User updatedUser = userMapper.fromProfileChangeDTO(reqDto, user); + userRepository.save(updatedUser); + return of(userMapper.toDTO(updatedUser)); + } + @PreAuthorize("hasRole('ADMIN')") public Optional updatePassword(String uuid, UserPasswordDTO reqDto) { Optional oUser = userRepository.findByUuid(uuid); @@ -125,6 +129,17 @@ public Optional updatePassword(String uuid, UserPasswordDTO reqDto) { return of(userMapper.toDTO(updatedUser)); } + public Optional updatePasswordForCurrentUser(UserPasswordDTO reqDto) { + Optional oUser = getCurrentUserUuid().flatMap(uuid -> userRepository.findByUuid(uuid)); + if (oUser.isEmpty()) { + return empty(); + } + User user = oUser.get(); + User updatedUser = userMapper.fromPasswordDTO(reqDto, user); + userRepository.save(updatedUser); + return of(userMapper.toDTO(updatedUser)); + } + @PreAuthorize("hasRole('ADMIN')") public boolean deleteUser(String uuid) { Optional oUser = userRepository.findByUuid(uuid); diff --git a/src/main/java/nl/dtls/fairdatapoint/service/user/UserValidator.java b/src/main/java/nl/dtls/fairdatapoint/service/user/UserValidator.java new file mode 100644 index 000000000..50bd12528 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/user/UserValidator.java @@ -0,0 +1,26 @@ +package nl.dtls.fairdatapoint.service.user; + +import nl.dtls.fairdatapoint.database.mongo.repository.UserRepository; +import nl.dtls.fairdatapoint.entity.exception.ValidationException; +import nl.dtls.fairdatapoint.entity.user.User; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Optional; + +import static java.lang.String.format; + +@Service +public class UserValidator { + + @Autowired + private UserRepository userRepository; + + public void validateEmail(String uuid, String email) { + Optional oUserEmail = userRepository.findByEmail(email); + if (oUserEmail.isPresent() && !oUserEmail.get().getUuid().equals(uuid)) { + throw new ValidationException(format("Email '%s' is already taken", email)); + } + } + +} diff --git a/src/test/java/nl/dtls/fairdatapoint/acceptance/user/Common.java b/src/test/java/nl/dtls/fairdatapoint/acceptance/user/Common.java index df2fabfd7..0218fc27f 100644 --- a/src/test/java/nl/dtls/fairdatapoint/acceptance/user/Common.java +++ b/src/test/java/nl/dtls/fairdatapoint/acceptance/user/Common.java @@ -25,6 +25,7 @@ import nl.dtls.fairdatapoint.api.dto.user.UserChangeDTO; import nl.dtls.fairdatapoint.api.dto.user.UserCreateDTO; import nl.dtls.fairdatapoint.api.dto.user.UserDTO; +import nl.dtls.fairdatapoint.api.dto.user.UserProfileChangeDTO; import nl.dtls.fairdatapoint.entity.user.User; import static org.hamcrest.MatcherAssert.assertThat; @@ -45,6 +46,12 @@ public static void compare(UserChangeDTO entity, UserDTO dto) { assertThat(dto.getEmail(), is(equalTo(entity.getEmail()))); } + public static void compare(UserProfileChangeDTO entity, UserDTO dto) { + assertThat(dto.getFirstName(), is(equalTo(entity.getFirstName()))); + assertThat(dto.getLastName(), is(equalTo(entity.getLastName()))); + assertThat(dto.getEmail(), is(equalTo(entity.getEmail()))); + } + public static void compare(User entity, UserDTO dto) { assertThat(dto.getUuid(), is(equalTo(entity.getUuid()))); assertThat(dto.getFirstName(), is(equalTo(entity.getFirstName()))); diff --git a/src/test/java/nl/dtls/fairdatapoint/acceptance/user/Detail_Current_PUT.java b/src/test/java/nl/dtls/fairdatapoint/acceptance/user/Detail_Current_PUT.java new file mode 100644 index 000000000..2dbba395a --- /dev/null +++ b/src/test/java/nl/dtls/fairdatapoint/acceptance/user/Detail_Current_PUT.java @@ -0,0 +1,108 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.acceptance.user; + +import nl.dtls.fairdatapoint.WebIntegrationTest; +import nl.dtls.fairdatapoint.api.dto.error.ErrorDTO; +import nl.dtls.fairdatapoint.api.dto.user.UserDTO; +import nl.dtls.fairdatapoint.api.dto.user.UserProfileChangeDTO; +import nl.dtls.fairdatapoint.database.mongo.migration.development.user.data.UserFixtures; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.*; + +import java.net.URI; + +import static java.lang.String.format; +import static nl.dtls.fairdatapoint.acceptance.common.ForbiddenTest.createNoUserForbiddenTestPut; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; + +@DisplayName("PUT /users/:userUuid") +public class Detail_Current_PUT extends WebIntegrationTest { + + private URI url() { + return URI.create("/users/current"); + } + + private UserProfileChangeDTO reqDto() { + return new UserProfileChangeDTO("EDITED: Albert", "EDITED: Einstein", "albert.einstein.edited@example.com"); + } + + @Autowired + private UserFixtures userFixtures; + + @Test + @DisplayName("HTTP 200") + public void res200() { + // GIVEN: + RequestEntity request = RequestEntity + .put(url()) + .header(HttpHeaders.AUTHORIZATION, ALBERT_TOKEN) + .accept(MediaType.APPLICATION_JSON) + .body(reqDto()); + ParameterizedTypeReference responseType = new ParameterizedTypeReference<>() { + }; + + // WHEN: + ResponseEntity result = client.exchange(request, responseType); + + // THEN: + assertThat(result.getStatusCode(), is(equalTo(HttpStatus.OK))); + Common.compare(reqDto(), result.getBody()); + } + + @Test + @DisplayName("HTTP 400: Email Already Exists") + public void res400_emailAlreadyExists() { + // GIVEN: + UserProfileChangeDTO reqDto = new UserProfileChangeDTO( + "EDITED: Albert", + "EDITED: Einstein", + "nikola.tesla@example.com"); + RequestEntity request = RequestEntity + .put(url()) + .header(HttpHeaders.AUTHORIZATION, ALBERT_TOKEN) + .accept(MediaType.APPLICATION_JSON) + .body(reqDto); + ParameterizedTypeReference responseType = new ParameterizedTypeReference<>() { + }; + + // WHEN: + ResponseEntity result = client.exchange(request, responseType); + + // THEN: + assertThat(result.getStatusCode(), is(equalTo(HttpStatus.BAD_REQUEST))); + assertThat(result.getBody().getMessage(), is(format("Email '%s' is already taken", reqDto.getEmail()))); + } + + @Test + @DisplayName("HTTP 403: User is not authenticated") + public void res403_notAuthenticated() { + createNoUserForbiddenTestPut(client, url(), reqDto()); + } + +} diff --git a/src/test/java/nl/dtls/fairdatapoint/acceptance/user/Detail_Current_Password_PUT.java b/src/test/java/nl/dtls/fairdatapoint/acceptance/user/Detail_Current_Password_PUT.java new file mode 100644 index 000000000..1061af9bf --- /dev/null +++ b/src/test/java/nl/dtls/fairdatapoint/acceptance/user/Detail_Current_Password_PUT.java @@ -0,0 +1,86 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.acceptance.user; + +import nl.dtls.fairdatapoint.WebIntegrationTest; +import nl.dtls.fairdatapoint.api.dto.user.UserDTO; +import nl.dtls.fairdatapoint.api.dto.user.UserPasswordDTO; +import nl.dtls.fairdatapoint.database.mongo.migration.development.user.data.UserFixtures; +import nl.dtls.fairdatapoint.entity.user.User; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; + +import java.net.URI; + +import static nl.dtls.fairdatapoint.acceptance.common.ForbiddenTest.createNoUserForbiddenTestPut; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; + +@DisplayName("PUT /users/current/password") +public class Detail_Current_Password_PUT extends WebIntegrationTest { + + private URI url() { + return URI.create("/users/current/password"); + } + + private UserPasswordDTO reqDto() { + return new UserPasswordDTO("newPassword"); + } + + @Autowired + private UserFixtures userFixtures; + + @Test + @DisplayName("HTTP 200") + public void res200() { + // GIVEN: + User user = userFixtures.albert(); + RequestEntity request = RequestEntity + .put(url()) + .header(HttpHeaders.AUTHORIZATION, ALBERT_TOKEN) + .body(reqDto()); + ParameterizedTypeReference responseType = new ParameterizedTypeReference<>() { + }; + + // WHEN: + ResponseEntity result = client.exchange(request, responseType); + + // THEN: + assertThat(result.getStatusCode(), is(equalTo(HttpStatus.OK))); + Common.compare(user, result.getBody()); + } + + @Test + @DisplayName("HTTP 403: User is not authenticated") + public void res403_notAuthenticated() { + createNoUserForbiddenTestPut(client, url(), reqDto()); + } + +} From f3313b28087259e105c913516508aa2d7f32a83a Mon Sep 17 00:00:00 2001 From: Vojtech Knaisl Date: Tue, 29 Sep 2020 13:55:35 +0200 Subject: [PATCH 04/12] Allow to turn on the index functionality --- pom.xml | 4 +++ .../api/controller/index/PingController.java | 4 +-- .../metadata/GenericController.java | 17 ++++++++---- .../fairdatapoint/config/SecurityConfig.java | 1 - .../exception/FeatureDisabledException.java | 13 ++++++++++ .../index/common/IndexFeatureAspect.java | 26 +++++++++++++++++++ .../common/RequiredEnabledIndexFeature.java | 12 +++++++++ .../index/entry/IndexEntryService.java | 5 ++++ .../service/index/event/EventService.java | 5 ++++ .../service/index/webhook/WebhookService.java | 6 +++++ ...itional-spring-configuration-metadata.json | 5 ++++ src/main/resources/application.yml | 3 ++- .../acceptance/general/SecurityTest.java | 4 +-- .../acceptance/index/ping/List_POST.java | 7 ++--- src/test/resources/application-testing.yml | 1 + 15 files changed, 97 insertions(+), 16 deletions(-) create mode 100644 src/main/java/nl/dtls/fairdatapoint/entity/exception/FeatureDisabledException.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/index/common/IndexFeatureAspect.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/index/common/RequiredEnabledIndexFeature.java diff --git a/pom.xml b/pom.xml index 4919f22c3..d6f84623b 100644 --- a/pom.xml +++ b/pom.xml @@ -130,6 +130,10 @@ org.springframework.boot spring-boot-starter-actuator + + org.springframework.boot + spring-boot-starter-aop + org.springframework.security spring-security-acl-mongodb diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/index/PingController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/index/PingController.java index 2118c937f..04df278ce 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/controller/index/PingController.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/index/PingController.java @@ -37,7 +37,7 @@ import javax.validation.Valid; @RestController -@RequestMapping("/index") +@RequestMapping("/") public class PingController { private static final Logger logger = LoggerFactory.getLogger(PingController.class); @@ -52,7 +52,7 @@ public class PingController { notes = "Inform about running FAIR Data Point. It is expected to send pings regularly (at least weekly). " + "There is a rate limit set both per single IP within a period of time and per URL in message." ) - @RequestMapping(method = RequestMethod.POST) + @RequestMapping(method = RequestMethod.POST, consumes = "application/json", produces = "application/json") @ResponseStatus(HttpStatus.NO_CONTENT) public void receivePing(@RequestBody @Valid PingDTO reqDto, HttpServletRequest request) { logger.info("Received ping from {}", request.getRemoteAddr()); diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericController.java index 31c8695d5..1a8d046ac 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericController.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericController.java @@ -192,15 +192,22 @@ public ResponseEntity storeMetaData(HttpServletRequest request, @RequestBody String reqBody, @RequestHeader(value = "Content-Type", required = false) String contentType) throws MetadataServiceException { - // 1. Init + // 1. Check if user is authenticated + // - it can't be in SecurityConfig because the authentication is done based on content-type + Optional oUser = currentUserService.getCurrentUser(); + if (oUser.isEmpty()) { + throw new ForbiddenException("You have to be login at first"); + } + + // 2. Init String urlPrefix = getResourceNameForList(getRequestURL(request, persistentUrl)); MetadataService metadataService = metadataServiceFactory.getMetadataServiceByUrlPrefix(urlPrefix); ResourceDefinition rd = resourceDefinitionService.getByUrlPrefix(urlPrefix); - // 2. Generate URI + // 3. Generate URI IRI uri = generateNewIRI(request, persistentUrl); - // 3. Parse reqDto + // 4. Parse reqDto RDFFormat rdfContentType = getRdfContentType(contentType); Model oldDto = read(reqBody, uri.stringValue(), rdfContentType); Model reqDto = changeBaseUri(oldDto, uri.stringValue(), rd.getTargetClassUris()); @@ -208,10 +215,10 @@ public ResponseEntity storeMetaData(HttpServletRequest request, reqDto.remove(null, i(rdChild.getRelationUri()), null); } - // 4. Store metadata + // 5. Store metadata Model metadata = metadataService.store(reqDto, uri, rd); - // 5. Create response + // 6. Create response return ResponseEntity .created(URI.create(uri.stringValue())) .body(metadata); diff --git a/src/main/java/nl/dtls/fairdatapoint/config/SecurityConfig.java b/src/main/java/nl/dtls/fairdatapoint/config/SecurityConfig.java index 4b6ccb936..c78f034c8 100644 --- a/src/main/java/nl/dtls/fairdatapoint/config/SecurityConfig.java +++ b/src/main/java/nl/dtls/fairdatapoint/config/SecurityConfig.java @@ -63,7 +63,6 @@ protected void configure(HttpSecurity http) throws Exception { .antMatchers("/index/admin**").authenticated() .antMatchers("/index**").permitAll() .antMatchers(HttpMethod.PUT, "/**").authenticated() - .antMatchers(HttpMethod.POST, "/**").authenticated() .anyRequest().permitAll() .and() .apply(filterConfigurer); diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/exception/FeatureDisabledException.java b/src/main/java/nl/dtls/fairdatapoint/entity/exception/FeatureDisabledException.java new file mode 100644 index 000000000..39235dde2 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/entity/exception/FeatureDisabledException.java @@ -0,0 +1,13 @@ +package nl.dtls.fairdatapoint.entity.exception; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseStatus(value = HttpStatus.BAD_REQUEST) +public class FeatureDisabledException extends RuntimeException { + + public FeatureDisabledException(String message) { + super(message); + } + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/common/IndexFeatureAspect.java b/src/main/java/nl/dtls/fairdatapoint/service/index/common/IndexFeatureAspect.java new file mode 100644 index 000000000..7bab849bc --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/common/IndexFeatureAspect.java @@ -0,0 +1,26 @@ +package nl.dtls.fairdatapoint.service.index.common; + +import nl.dtls.fairdatapoint.entity.exception.FeatureDisabledException; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Aspect +@Component +public class IndexFeatureAspect { + + @Value("${fdp-index.enabled:false}") + private boolean fdpIndexEnabled; + + @Around("@annotation(nl.dtls.fairdatapoint.service.index.common.RequiredEnabledIndexFeature)") + public Object logExecutionTime(ProceedingJoinPoint joinPoint) throws Throwable { + if (!fdpIndexEnabled) { + throw new FeatureDisabledException("Index functionality is turn off"); + } + return joinPoint.proceed(); + } + + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/common/RequiredEnabledIndexFeature.java b/src/main/java/nl/dtls/fairdatapoint/service/index/common/RequiredEnabledIndexFeature.java new file mode 100644 index 000000000..c51b9087b --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/common/RequiredEnabledIndexFeature.java @@ -0,0 +1,12 @@ +package nl.dtls.fairdatapoint.service.index.common; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +public @interface RequiredEnabledIndexFeature { + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryService.java b/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryService.java index 05b3b0115..99dcd5bae 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/entry/IndexEntryService.java @@ -30,6 +30,7 @@ import nl.dtls.fairdatapoint.entity.index.config.EventsConfig; import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; import nl.dtls.fairdatapoint.entity.index.entry.IndexEntryState; +import nl.dtls.fairdatapoint.service.index.common.RequiredEnabledIndexFeature; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -60,6 +61,7 @@ public Iterable getAllEntries() { return repository.findAll(); } + @RequiredEnabledIndexFeature public Page getEntriesPage(Pageable pageable, String state) { if (state.equalsIgnoreCase(ACTIVE.name())) { return repository.findAllByStateEqualsAndLastRetrievalTimeAfter(pageable, IndexEntryState.Valid, @@ -81,10 +83,12 @@ public Page getEntriesPage(Pageable pageable, String state) { return repository.findAll(pageable); } + @RequiredEnabledIndexFeature public Optional getEntry(String uuid) { return repository.findByUuid(uuid); } + @RequiredEnabledIndexFeature public IndexEntryInfoDTO getEntriesInfo() { Map entriesCount = new HashMap<>(); entriesCount.put("ALL", repository.count()); @@ -98,6 +102,7 @@ public IndexEntryInfoDTO getEntriesInfo() { return new IndexEntryInfoDTO(entriesCount); } + @RequiredEnabledIndexFeature public IndexEntry storeEntry(@Valid PingDTO pingDTO) { var clientUrl = pingDTO.getClientUrl(); var entity = repository.findByClientUrl(clientUrl); diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/event/EventService.java b/src/main/java/nl/dtls/fairdatapoint/service/index/event/EventService.java index 917350ac8..ebd78038e 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/index/event/EventService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/event/EventService.java @@ -37,6 +37,7 @@ import nl.dtls.fairdatapoint.entity.index.exception.RateLimitException; import nl.dtls.fairdatapoint.entity.index.http.Exchange; import nl.dtls.fairdatapoint.entity.index.http.ExchangeState; +import nl.dtls.fairdatapoint.service.index.common.RequiredEnabledIndexFeature; import nl.dtls.fairdatapoint.service.index.entry.IndexEntryService; import nl.dtls.fairdatapoint.service.index.webhook.WebhookService; import org.eclipse.rdf4j.util.iterators.EmptyIterator; @@ -93,10 +94,12 @@ public Iterable getEvents(IndexEntry indexEntry) { "created"))); } + @RequiredEnabledIndexFeature public Iterable getEvents(String indexEntryUuid) { return indexEntryService.getEntry(indexEntryUuid).map(this::getEvents).orElse(EmptyIterator::new); } + @RequiredEnabledIndexFeature @SneakyThrows public Event acceptIncomingPing(PingDTO reqDto, HttpServletRequest request) { var remoteAddr = request.getRemoteAddr(); @@ -179,6 +182,7 @@ private void processMetadataRetrieval(Event event) { } @Async + @RequiredEnabledIndexFeature public void triggerMetadataRetrieval(Event triggerEvent) { logger.info("Initiating metadata retrieval triggered by {}", triggerEvent.getUuid()); Iterable events = MetadataRetrievalUtils.prepareEvents(triggerEvent, indexEntryService); @@ -219,6 +223,7 @@ public void startResumeUnfinishedEvents() { executor.submit(this::resumeUnfinishedEvents); } + @RequiredEnabledIndexFeature public Event acceptAdminTrigger(HttpServletRequest request, String indexEntryUuid) { Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); Event event = eventMapper.toAdminTriggerEvent(request, authentication, indexEntryUuid); diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookService.java b/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookService.java index dc706f3ed..73ed5a8fb 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookService.java @@ -32,6 +32,7 @@ import nl.dtls.fairdatapoint.entity.index.event.Event; import nl.dtls.fairdatapoint.entity.index.webhook.Webhook; import nl.dtls.fairdatapoint.entity.index.webhook.WebhookEvent; +import nl.dtls.fairdatapoint.service.index.common.RequiredEnabledIndexFeature; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -67,6 +68,7 @@ public class WebhookService { private static final String SECRET_PLACEHOLDER = "*** HIDDEN ***"; + @RequiredEnabledIndexFeature public void processWebhookTrigger(Event event) { event.execute(); eventRepository.save(event); @@ -87,17 +89,20 @@ public void processWebhookTrigger(Event event) { } @Async + @RequiredEnabledIndexFeature public void triggerWebhook(Webhook webhook, WebhookEvent webhookEvent, Event triggerEvent) { Event event = webhookMapper.toTriggerEvent(webhook, webhookEvent, triggerEvent); processWebhookTrigger(event); } @Async + @RequiredEnabledIndexFeature public void triggerWebhooks(WebhookEvent webhookEvent, Event triggerEvent) { logger.info("Triggered webhook event {} by event {}", webhookEvent, triggerEvent.getUuid()); WebhookUtils.filterMatching(webhookRepository.findAll(), webhookEvent, triggerEvent).forEach(webhook -> triggerWebhook(webhook, webhookEvent, triggerEvent)); } + @RequiredEnabledIndexFeature public Event handleWebhookPing(HttpServletRequest request, UUID webhookUuid) { Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); Optional webhook = webhookRepository.findByUuid(webhookUuid); @@ -109,6 +114,7 @@ public Event handleWebhookPing(HttpServletRequest request, UUID webhookUuid) { } @Async + @RequiredEnabledIndexFeature public void triggerWebhooks(Event triggerEvent) { switch (triggerEvent.getType()) { case AdminTrigger: diff --git a/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/src/main/resources/META-INF/additional-spring-configuration-metadata.json index ad1500c57..bcdcf50fd 100644 --- a/src/main/resources/META-INF/additional-spring-configuration-metadata.json +++ b/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -30,6 +30,11 @@ "type": "java.lang.String", "description": "States if 'Call home' feature should be enabled" }, + { + "name": "fdp-index.enabled", + "type": "java.lang.Boolean", + "description": "States if 'index' functionality should be enabled" + }, { "name": "fdp-index.api.url", "type": "java.lang.String", diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 653e4c8af..1c5cdd3be 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -15,7 +15,7 @@ spring: core-size: 2 max-size: 5 queue-capacity: 5000 - thread-name-prefix: fdpindex-task- + thread-name-prefix: fdp-task- management: health: @@ -58,6 +58,7 @@ metadataMetrics: https://purl.org/fair-metrics/FM_A1.1: https://www.wikidata.org/wiki/Q8777 fdp-index: + enabled: false events: retrieval: rateLimitWait: PT10M # 10 minutes (ISO 8601) diff --git a/src/test/java/nl/dtls/fairdatapoint/acceptance/general/SecurityTest.java b/src/test/java/nl/dtls/fairdatapoint/acceptance/general/SecurityTest.java index f3f4e6b21..52b9566ec 100644 --- a/src/test/java/nl/dtls/fairdatapoint/acceptance/general/SecurityTest.java +++ b/src/test/java/nl/dtls/fairdatapoint/acceptance/general/SecurityTest.java @@ -59,11 +59,11 @@ public void postRequestsAreSecured() { .post(URI.create("/distribution")) .header(HttpHeaders.CONTENT_TYPE, "text/turtle") .body(reqDto); - ParameterizedTypeReference responseType = new ParameterizedTypeReference<>() { + ParameterizedTypeReference responseType = new ParameterizedTypeReference<>() { }; // WHEN: - ResponseEntity result = client.exchange(request, responseType); + ResponseEntity result = client.exchange(request, responseType); // THEN: assertThat(result.getStatusCode(), is(equalTo(HttpStatus.FORBIDDEN))); diff --git a/src/test/java/nl/dtls/fairdatapoint/acceptance/index/ping/List_POST.java b/src/test/java/nl/dtls/fairdatapoint/acceptance/index/ping/List_POST.java index a3d0a29f8..e17336677 100644 --- a/src/test/java/nl/dtls/fairdatapoint/acceptance/index/ping/List_POST.java +++ b/src/test/java/nl/dtls/fairdatapoint/acceptance/index/ping/List_POST.java @@ -31,7 +31,6 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.ParameterizedTypeReference; -import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.RequestEntity; @@ -50,14 +49,11 @@ public class List_POST extends WebIntegrationTest { @Autowired private IndexEntryRepository indexEntryRepository; - @Autowired - private MongoTemplate mongoTemplate; - private final ParameterizedTypeReference responseType = new ParameterizedTypeReference<>() { }; private URI url() { - return URI.create("/index"); + return URI.create("/"); } private PingDTO reqDTO(String clientUrl) { @@ -76,6 +72,7 @@ public void res204_newEntry() { // AND: prepare request RequestEntity request = RequestEntity .post(url()) + .contentType(MediaType.APPLICATION_JSON) .accept(MediaType.APPLICATION_JSON) .body(reqDto); diff --git a/src/test/resources/application-testing.yml b/src/test/resources/application-testing.yml index 9a9beeabf..023d98d40 100644 --- a/src/test/resources/application-testing.yml +++ b/src/test/resources/application-testing.yml @@ -16,6 +16,7 @@ security: fdp-index: + enabled: true events: retrieval: rateLimitWait: PT10M # 10 minutes (ISO 8601) From fc422a1b5898406ab62d71f275d3236badc68337 Mon Sep 17 00:00:00 2001 From: Vojtech Knaisl Date: Tue, 29 Sep 2020 17:44:36 +0200 Subject: [PATCH 05/12] Show results even if the description is absent --- .../rdf/repository/common/AbstractMetadataRepository.java | 3 ++- .../database/rdf/repository/common/findEntityByLiteral.sparql | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java index 68008f38b..938b429f7 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java @@ -44,6 +44,7 @@ import java.util.Map; import static java.lang.String.format; +import static java.util.Optional.ofNullable; import static java.util.stream.Collectors.toList; @Slf4j @@ -82,7 +83,7 @@ public List findByLiteral(Literal query) throws MetadataRepository .map(s -> new SearchResult( s.getValue("entity").stringValue(), s.getValue("title").stringValue(), - s.getValue("description").stringValue(), + ofNullable(s.getValue("description")).map(Value::stringValue).orElse(""), new SearchResultRelation( s.getValue("relationPredicate").stringValue(), s.getValue("relationObject").stringValue()) diff --git a/src/main/resources/nl/dtls/fairdatapoint/database/rdf/repository/common/findEntityByLiteral.sparql b/src/main/resources/nl/dtls/fairdatapoint/database/rdf/repository/common/findEntityByLiteral.sparql index 53987e28a..c32ba05a2 100644 --- a/src/main/resources/nl/dtls/fairdatapoint/database/rdf/repository/common/findEntityByLiteral.sparql +++ b/src/main/resources/nl/dtls/fairdatapoint/database/rdf/repository/common/findEntityByLiteral.sparql @@ -3,7 +3,7 @@ prefix dct: SELECT ?entity ?title ?description ?relationPredicate ?relationObject WHERE { ?entity ?relationPredicate ?relationObject . ?entity dct:title ?title . - ?entity dct:description ?description . + Optional { ?entity dct:description ?description } filter isLiteral(?relationObject) filter CONTAINS(LCASE(str(?relationObject)), LCASE(str(?query))) } From c1624639488ac5a95ed86e783114d7754094b307 Mon Sep 17 00:00:00 2001 From: Vojtech Knaisl Date: Mon, 26 Oct 2020 10:00:31 +0100 Subject: [PATCH 06/12] Add harvestor --- .../api/controller/index/AdminController.java | 9 +- .../api/controller/index/PingController.java | 8 +- .../metadata/GenericController.java | 9 +- .../controller/search/SearchController.java | 22 +++ .../api/dto/search/SearchQueryDTO.java | 22 +++ .../api/dto/search/SearchResultDTO.java | 22 +++ .../api/dto/user/UserProfileChangeDTO.java | 22 +++ .../config/HttpClientConfig.java | 13 +- .../index/entry/data/IndexEntryFixtures.java | 3 - .../metadata/data/RdfMetadataFixtures.java | 6 +- .../exception/FeatureDisabledException.java | 22 +++ .../entity/index/entry/IndexEntry.java | 2 - .../entity/index/event/Event.java | 2 - .../entity/index/webhook/Webhook.java | 2 - .../entity/search/SearchResult.java | 22 +++ .../entity/search/SearchResultRelation.java | 22 +++ .../index/common/IndexFeatureAspect.java | 22 +++ .../common/RequiredEnabledIndexFeature.java | 22 +++ .../index/harvester/HarvesterService.java | 157 ++++++++++++++++++ .../service/index/webhook/WebhookMapper.java | 22 +++ .../metadata/enhance/MetadataEnhancer.java | 25 +++ .../service/search/SearchService.java | 31 +++- .../service/user/UserValidator.java | 22 +++ src/main/resources/defaultNavigationShacl.ttl | 34 ++++ .../acceptance/search/List_POST.java | 22 +++ .../index/harvester/HarvesterServiceTest.java | 153 +++++++++++++++++ .../utils/TestRdfMetadataFixtures.java | 22 ++- 27 files changed, 713 insertions(+), 27 deletions(-) create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/index/harvester/HarvesterService.java create mode 100644 src/main/resources/defaultNavigationShacl.ttl create mode 100644 src/test/java/nl/dtls/fairdatapoint/service/index/harvester/HarvesterServiceTest.java diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/index/AdminController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/index/AdminController.java index 52ea7be71..ebbe66b6c 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/controller/index/AdminController.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/index/AdminController.java @@ -23,11 +23,10 @@ package nl.dtls.fairdatapoint.api.controller.index; import io.swagger.annotations.ApiOperation; +import lombok.extern.log4j.Log4j2; import nl.dtls.fairdatapoint.entity.index.event.Event; import nl.dtls.fairdatapoint.service.index.event.EventService; import nl.dtls.fairdatapoint.service.index.webhook.WebhookService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; @@ -36,10 +35,10 @@ import javax.servlet.http.HttpServletRequest; import java.util.UUID; +@Log4j2 @RestController @RequestMapping("/index/admin") public class AdminController { - private static final Logger logger = LoggerFactory.getLogger(PingController.class); @Autowired private EventService eventService; @@ -52,7 +51,7 @@ public class AdminController { @PreAuthorize("hasRole('ADMIN')") @ResponseStatus(HttpStatus.NO_CONTENT) public void triggerMetadataRetrieve(@RequestParam(required = false) String clientUrl, HttpServletRequest request) { - logger.info("Received ping from {}", request.getRemoteAddr()); + log.info("Received ping from {}", request.getRemoteAddr()); final Event event = eventService.acceptAdminTrigger(request, clientUrl); webhookService.triggerWebhooks(event); eventService.triggerMetadataRetrieval(event); @@ -63,7 +62,7 @@ public void triggerMetadataRetrieve(@RequestParam(required = false) String clien @PreAuthorize("hasRole('ADMIN')") @ResponseStatus(HttpStatus.NO_CONTENT) public void webhookPing(@RequestParam(required = true) UUID webhook, HttpServletRequest request) { - logger.info("Received webhook {} ping trigger from {}", webhook, request.getRemoteAddr()); + log.info("Received webhook {} ping trigger from {}", webhook, request.getRemoteAddr()); final Event event = webhookService.handleWebhookPing(request, webhook); webhookService.triggerWebhooks(event); } diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/index/PingController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/index/PingController.java index 04df278ce..9337d12de 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/controller/index/PingController.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/index/PingController.java @@ -24,8 +24,10 @@ import io.swagger.annotations.ApiOperation; import nl.dtls.fairdatapoint.api.dto.index.ping.PingDTO; +import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import nl.dtls.fairdatapoint.entity.index.event.Event; import nl.dtls.fairdatapoint.service.index.event.EventService; +import nl.dtls.fairdatapoint.service.index.harvester.HarvesterService; import nl.dtls.fairdatapoint.service.index.webhook.WebhookService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +49,9 @@ public class PingController { @Autowired private WebhookService webhookService; + @Autowired + private HarvesterService harvesterService; + @ApiOperation( value = "Ping payload with FAIR Data Point info", notes = "Inform about running FAIR Data Point. It is expected to send pings regularly (at least weekly). " + @@ -54,11 +59,12 @@ public class PingController { ) @RequestMapping(method = RequestMethod.POST, consumes = "application/json", produces = "application/json") @ResponseStatus(HttpStatus.NO_CONTENT) - public void receivePing(@RequestBody @Valid PingDTO reqDto, HttpServletRequest request) { + public void receivePing(@RequestBody @Valid PingDTO reqDto, HttpServletRequest request) throws MetadataRepositoryException { logger.info("Received ping from {}", request.getRemoteAddr()); final Event event = eventService.acceptIncomingPing(reqDto, request); logger.info("Triggering metadata retrieval for {}", event.getRelatedTo().getClientUrl()); eventService.triggerMetadataRetrieval(event); + harvesterService.harvest(reqDto.getClientUrl()); webhookService.triggerWebhooks(event); } } diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericController.java index 1a8d046ac..a53155b0d 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericController.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/metadata/GenericController.java @@ -30,6 +30,7 @@ import nl.dtls.fairdatapoint.entity.resource.ResourceDefinitionChild; import nl.dtls.fairdatapoint.entity.user.User; import nl.dtls.fairdatapoint.service.metadata.common.MetadataService; +import nl.dtls.fairdatapoint.service.metadata.enhance.MetadataEnhancer; import nl.dtls.fairdatapoint.service.metadata.exception.MetadataServiceException; import nl.dtls.fairdatapoint.service.metadata.factory.MetadataServiceFactory; import nl.dtls.fairdatapoint.service.metadata.state.MetadataStateService; @@ -76,6 +77,9 @@ public class GenericController { @Autowired private MetadataStateService metadataStateService; + @Autowired + private MetadataEnhancer metadataEnhancer; + @Autowired private CurrentUserService currentUserService; @@ -180,7 +184,10 @@ public Model getMetaData(HttpServletRequest request) throws MetadataServiceExcep } } - // 6. Create response + // 6. Add links + metadataEnhancer.enhanceWithLinks(entityUri, entity, rd, persistentUrl, resultRdf); + + // 7. Create response return resultRdf; } diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/search/SearchController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/search/SearchController.java index 59850a5be..2767062c2 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/controller/search/SearchController.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/search/SearchController.java @@ -1,3 +1,25 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package nl.dtls.fairdatapoint.api.controller.search; import nl.dtls.fairdatapoint.api.dto.search.SearchQueryDTO; diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchQueryDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchQueryDTO.java index cd5a0723b..9916c1f13 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchQueryDTO.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchQueryDTO.java @@ -1,3 +1,25 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package nl.dtls.fairdatapoint.api.dto.search; import lombok.AllArgsConstructor; diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchResultDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchResultDTO.java index aa43503b9..5f26e6992 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchResultDTO.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchResultDTO.java @@ -1,3 +1,25 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package nl.dtls.fairdatapoint.api.dto.search; import lombok.AllArgsConstructor; diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/user/UserProfileChangeDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/user/UserProfileChangeDTO.java index 76c439c53..307da1ff3 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/dto/user/UserProfileChangeDTO.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/user/UserProfileChangeDTO.java @@ -1,3 +1,25 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package nl.dtls.fairdatapoint.api.dto.user; import lombok.AllArgsConstructor; diff --git a/src/main/java/nl/dtls/fairdatapoint/config/HttpClientConfig.java b/src/main/java/nl/dtls/fairdatapoint/config/HttpClientConfig.java index 0eee403db..cfc186cf6 100644 --- a/src/main/java/nl/dtls/fairdatapoint/config/HttpClientConfig.java +++ b/src/main/java/nl/dtls/fairdatapoint/config/HttpClientConfig.java @@ -27,6 +27,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.web.client.RestTemplate; +import java.net.http.HttpClient; import java.time.Duration; @Configuration @@ -35,8 +36,16 @@ public class HttpClientConfig { @Bean public RestTemplate restTemplate(RestTemplateBuilder restTemplateBuilder) { return restTemplateBuilder - .setConnectTimeout(Duration.ofSeconds(2)) - .setReadTimeout(Duration.ofSeconds(2)) + .setConnectTimeout(Duration.ofSeconds(5)) + .setReadTimeout(Duration.ofSeconds(5)) + .build(); + } + + @Bean + public HttpClient httpClient() { + return HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_2) + .followRedirects(HttpClient.Redirect.ALWAYS) .build(); } diff --git a/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/entry/data/IndexEntryFixtures.java b/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/entry/data/IndexEntryFixtures.java index b9a0284e2..a7e91361c 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/entry/data/IndexEntryFixtures.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/mongo/migration/development/index/entry/data/IndexEntryFixtures.java @@ -22,7 +22,6 @@ */ package nl.dtls.fairdatapoint.database.mongo.migration.development.index.entry.data; -import nl.dtls.fairdatapoint.entity.index.config.EventsConfig; import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; import nl.dtls.fairdatapoint.entity.index.entry.RepositoryMetadata; import org.springframework.stereotype.Service; @@ -35,8 +34,6 @@ @Service public class IndexEntryFixtures { - private EventsConfig eventsConfig; - public IndexEntry entryActive() { String clientUri = "https://example.com/my-valid-fairdatapoint"; RepositoryMetadata repositoryData = new RepositoryMetadata( diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/development/metadata/data/RdfMetadataFixtures.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/development/metadata/data/RdfMetadataFixtures.java index e19812861..d842acaec 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/development/metadata/data/RdfMetadataFixtures.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/migration/development/metadata/data/RdfMetadataFixtures.java @@ -33,9 +33,13 @@ @Service public class RdfMetadataFixtures { - @Autowired protected MetadataFactory metadataFactory; + @Autowired + public RdfMetadataFixtures(MetadataFactory metadataFactory) { + this.metadataFactory = metadataFactory; + } + public Model repositoryMetadata(String repositoryUrl) { return metadataFactory.createFDPMetadata( "My FAIR Data Point", diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/exception/FeatureDisabledException.java b/src/main/java/nl/dtls/fairdatapoint/entity/exception/FeatureDisabledException.java index 39235dde2..e8e678965 100644 --- a/src/main/java/nl/dtls/fairdatapoint/entity/exception/FeatureDisabledException.java +++ b/src/main/java/nl/dtls/fairdatapoint/entity/exception/FeatureDisabledException.java @@ -1,3 +1,25 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package nl.dtls.fairdatapoint.entity.exception; import org.springframework.http.HttpStatus; diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/entry/IndexEntry.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/entry/IndexEntry.java index 88640104c..ca5cead25 100644 --- a/src/main/java/nl/dtls/fairdatapoint/entity/index/entry/IndexEntry.java +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/entry/IndexEntry.java @@ -25,7 +25,6 @@ import lombok.*; import org.bson.types.ObjectId; import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; import org.springframework.format.annotation.DateTimeFormat; @@ -43,7 +42,6 @@ public class IndexEntry { @Id protected ObjectId id; - @Indexed(unique = true) private String uuid; private String clientUrl; diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/event/Event.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/Event.java index 808e12191..3443a8ab2 100644 --- a/src/main/java/nl/dtls/fairdatapoint/entity/index/event/Event.java +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/event/Event.java @@ -28,7 +28,6 @@ import nl.dtls.fairdatapoint.entity.index.entry.IndexEntry; import org.bson.types.ObjectId; import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.DBRef; import org.springframework.data.mongodb.core.mapping.Document; import org.springframework.format.annotation.DateTimeFormat; @@ -44,7 +43,6 @@ public class Event { @Id protected ObjectId id; - @Indexed(unique = true) @NotNull private UUID uuid = UUID.randomUUID(); @NotNull diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/index/webhook/Webhook.java b/src/main/java/nl/dtls/fairdatapoint/entity/index/webhook/Webhook.java index bd9dafedb..8721c74bc 100644 --- a/src/main/java/nl/dtls/fairdatapoint/entity/index/webhook/Webhook.java +++ b/src/main/java/nl/dtls/fairdatapoint/entity/index/webhook/Webhook.java @@ -27,7 +27,6 @@ import lombok.NoArgsConstructor; import org.bson.types.ObjectId; import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; import javax.validation.constraints.NotNull; @@ -43,7 +42,6 @@ public class Webhook { @Id protected ObjectId id; - @Indexed(unique = true) @NotNull private UUID uuid = UUID.randomUUID(); diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResult.java b/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResult.java index 76da792bf..9be7f8cb1 100644 --- a/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResult.java +++ b/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResult.java @@ -1,3 +1,25 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package nl.dtls.fairdatapoint.entity.search; import lombok.AllArgsConstructor; diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResultRelation.java b/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResultRelation.java index 68eee93ad..f779a15ce 100644 --- a/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResultRelation.java +++ b/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResultRelation.java @@ -1,3 +1,25 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package nl.dtls.fairdatapoint.entity.search; import lombok.AllArgsConstructor; diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/common/IndexFeatureAspect.java b/src/main/java/nl/dtls/fairdatapoint/service/index/common/IndexFeatureAspect.java index 7bab849bc..bd56ba0e8 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/index/common/IndexFeatureAspect.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/common/IndexFeatureAspect.java @@ -1,3 +1,25 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package nl.dtls.fairdatapoint.service.index.common; import nl.dtls.fairdatapoint.entity.exception.FeatureDisabledException; diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/common/RequiredEnabledIndexFeature.java b/src/main/java/nl/dtls/fairdatapoint/service/index/common/RequiredEnabledIndexFeature.java index c51b9087b..705a810e2 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/index/common/RequiredEnabledIndexFeature.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/common/RequiredEnabledIndexFeature.java @@ -1,3 +1,25 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package nl.dtls.fairdatapoint.service.index.common; import java.lang.annotation.ElementType; diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/harvester/HarvesterService.java b/src/main/java/nl/dtls/fairdatapoint/service/index/harvester/HarvesterService.java new file mode 100644 index 000000000..19e08e8a4 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/harvester/HarvesterService.java @@ -0,0 +1,157 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.service.index.harvester; + +import lombok.extern.log4j.Log4j2; +import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; +import nl.dtls.fairdatapoint.database.rdf.repository.generic.GenericMetadataRepository; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Model; +import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.Value; +import org.eclipse.rdf4j.model.vocabulary.LDP; +import org.eclipse.rdf4j.model.vocabulary.RDF; +import org.eclipse.rdf4j.rio.RDFFormat; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.*; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static java.util.Optional.ofNullable; +import static nl.dtls.fairdatapoint.entity.metadata.MetadataGetter.getChildren; +import static nl.dtls.fairdatapoint.util.HttpUtil.getRdfContentType; +import static nl.dtls.fairdatapoint.util.RdfIOUtil.read; +import static nl.dtls.fairdatapoint.util.RdfIOUtil.readFile; +import static nl.dtls.fairdatapoint.util.RdfUtil.getObjectsBy; +import static nl.dtls.fairdatapoint.util.RdfUtil.getSubjectsBy; +import static nl.dtls.fairdatapoint.util.ValueFactoryHelper.i; + +@Service +@Log4j2 +public class HarvesterService { + + private static final String DEFAULT_NAVIGATION_SHACL = "defaultNavigationShacl.ttl"; + + @Autowired + private RestTemplate restTemplate; + + @Autowired + private GenericMetadataRepository genericMetadataRepository; + + @Async + public void harvest(String clientUrl) throws MetadataRepositoryException { + log.info(format("Start harvesting '%s'", clientUrl)); + + // 1. Get navigation relationships + List navigationRelationships = getNavigationRelationships(clientUrl); + + // 2. Harvest data + Map result = visitNode(clientUrl, navigationRelationships, new HashMap<>()); + + // 3. Store data + for (Map.Entry item : result.entrySet()) { + genericMetadataRepository.save(new ArrayList<>(item.getValue()), i(item.getKey())); + } + + log.info(format("Harvesting for '%s' completed", clientUrl)); + } + + private List getNavigationRelationships(String uri) { + Model model = readFile(DEFAULT_NAVIGATION_SHACL, "http://fairdatapoint.org"); + return getObjectsBy(model, null, "http://www.w3.org/ns/shacl#path") + .stream() + .map(i -> i(i.stringValue())) + .distinct() + .collect(Collectors.toList()); + } + + private Map visitNode(String uri, List relationships, Map nodes) { + try { + Model model = makeRequest(uri); + nodes.put(uri, model); + + List containers = getSubjectsBy(model, RDF.TYPE, LDP.DIRECT_CONTAINER); + if (containers.size() > 0) { + // Get children through LDP links + for (Value container : containers) { + for (Value child : getObjectsBy(model, i(container.stringValue()), LDP.CONTAINS)) { + if (!nodes.containsKey(child.stringValue())) { + nodes = visitNode(child.stringValue(), relationships, nodes); + } + } + } + } else { + // Get children through default navigation SHACL + for (IRI relationship : relationships) { + List children = getChildren(model, relationship); + for (IRI child : children) { + if (!nodes.containsKey(child.stringValue())) { + nodes = visitNode(child.stringValue(), relationships, nodes); + } + } + } + } + + return nodes; + } catch (HttpClientErrorException ex) { + return nodes; + } + } + + private Model makeRequest(String uri) { + log.info(format("Making request to '%s'", uri)); + HttpHeaders headers = new HttpHeaders(); + headers.setAccept(List.of(MediaType.parseMediaType(RDFFormat.TURTLE.getDefaultMIMEType()))); + HttpEntity entity = new HttpEntity<>(null, headers); + try { + ResponseEntity response = restTemplate.exchange(uri, HttpMethod.GET, entity, String.class); + if (!response.getStatusCode().is2xxSuccessful()) { + log.info(format("Request to '%s' failed", uri)); + throw new HttpClientErrorException(response.getStatusCode()); + } + RDFFormat rdfContentType = getRdfContentType(response.getHeaders().getContentType().getType()); + log.info(format("Request to '%s' successfully received", uri)); + Model result = read(response.getBody(), uri, rdfContentType); + log.info(format("Request to '%s' successfully parsed", uri)); + return result; + } catch (RestClientException e) { + log.info(format("Request to '%s' failed", uri)); + throw new HttpClientErrorException( + HttpStatus.BAD_GATEWAY, + ofNullable(e.getMessage()).orElse("HTTP request failed to proceed") + ); + } + } + + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookMapper.java b/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookMapper.java index 0edbf2120..b0e26546c 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookMapper.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/index/webhook/WebhookMapper.java @@ -1,3 +1,25 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package nl.dtls.fairdatapoint.service.index.webhook; import nl.dtls.fairdatapoint.api.dto.index.webhook.WebhookPayloadDTO; diff --git a/src/main/java/nl/dtls/fairdatapoint/service/metadata/enhance/MetadataEnhancer.java b/src/main/java/nl/dtls/fairdatapoint/service/metadata/enhance/MetadataEnhancer.java index 13de6fe3c..31a0c8f02 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/metadata/enhance/MetadataEnhancer.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/metadata/enhance/MetadataEnhancer.java @@ -24,12 +24,16 @@ import nl.dtls.fairdatapoint.entity.metadata.Identifier; import nl.dtls.fairdatapoint.entity.resource.ResourceDefinition; +import nl.dtls.fairdatapoint.entity.resource.ResourceDefinitionChild; import nl.dtls.fairdatapoint.service.metadata.metric.MetricsMetadataService; +import nl.dtls.fairdatapoint.service.resource.ResourceDefinitionCache; import nl.dtls.fairdatapoint.util.ValueFactoryHelper; import nl.dtls.fairdatapoint.vocabulary.DATACITE; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Model; import org.eclipse.rdf4j.model.vocabulary.DCTERMS; +import org.eclipse.rdf4j.model.vocabulary.LDP; +import org.eclipse.rdf4j.model.vocabulary.RDF; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -39,9 +43,11 @@ import java.util.List; import java.util.stream.Collectors; +import static java.lang.String.format; import static nl.dtls.fairdatapoint.entity.metadata.MetadataGetter.*; import static nl.dtls.fairdatapoint.entity.metadata.MetadataSetter.*; import static nl.dtls.fairdatapoint.util.RdfUtil.containsObject; +import static nl.dtls.fairdatapoint.util.RdfUtil.getObjectsBy; import static nl.dtls.fairdatapoint.util.ValueFactoryHelper.i; import static nl.dtls.fairdatapoint.util.ValueFactoryHelper.l; @@ -62,6 +68,9 @@ public class MetadataEnhancer { @Autowired private MetricsMetadataService metricsMetadataService; + @Autowired + private ResourceDefinitionCache resourceDefinitionCache; + public void enhance(Model metadata, IRI uri, ResourceDefinition rd, Model oldMetadata) { enhance(metadata, uri, rd); @@ -121,6 +130,22 @@ public void enhance(Model metadata, IRI uri, ResourceDefinition rd) { } } + public void enhanceWithLinks(IRI entityUri, Model entity, ResourceDefinition rd, String persistentUrl, + Model resultRdf) { + for (ResourceDefinitionChild child : rd.getChildren()) { + ResourceDefinition rdChild = resourceDefinitionCache.getByUuid(child.getResourceDefinitionUuid()); + IRI container = i(format("%s/%s/", persistentUrl, rdChild.getUrlPrefix())); + + resultRdf.add(container, RDF.TYPE, LDP.DIRECT_CONTAINER); + resultRdf.add(container, DCTERMS.TITLE, l(child.getListView().getTitle())); + resultRdf.add(container, LDP.MEMBERSHIP_RESOURCE, entityUri); + resultRdf.add(container, LDP.HAS_MEMBER_RELATION, i(child.getRelationUri())); + for (org.eclipse.rdf4j.model.Value childUri : getObjectsBy(entity, entityUri, i(child.getRelationUri()))) { + resultRdf.add(container, LDP.CONTAINS, i(childUri.stringValue())); + } + } + } + private Identifier createMetadataIdentifier(IRI uri) { IRI identifierUri = i(uri.stringValue() + "#identifier"); return new Identifier(identifierUri, DATACITE.IDENTIFIER, l(uri)); diff --git a/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java b/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java index 1c72eced3..cab8990ba 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java @@ -1,9 +1,32 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package nl.dtls.fairdatapoint.service.search; import nl.dtls.fairdatapoint.api.dto.search.SearchQueryDTO; import nl.dtls.fairdatapoint.api.dto.search.SearchResultDTO; import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; import nl.dtls.fairdatapoint.database.rdf.repository.generic.GenericMetadataRepository; +import nl.dtls.fairdatapoint.entity.exception.ResourceNotFoundException; import nl.dtls.fairdatapoint.entity.metadata.MetadataState; import nl.dtls.fairdatapoint.entity.search.SearchResult; import nl.dtls.fairdatapoint.service.metadata.state.MetadataStateService; @@ -32,7 +55,13 @@ public List search(SearchQueryDTO reqDto) throws MetadataReposi .collect(Collectors.groupingBy(SearchResult::getUri, Collectors.mapping(i -> i, toList()))) .entrySet() .stream() - .filter(entry -> !metadataStateService.get(i(entry.getKey())).getState().equals(MetadataState.DRAFT)) + .filter(entry -> { + try { + return !metadataStateService.get(i(entry.getKey())).getState().equals(MetadataState.DRAFT); + } catch (ResourceNotFoundException e) { + return true; + } + }) .map(entry -> new SearchResultDTO( entry.getKey(), entry.getValue().get(0).getTitle(), diff --git a/src/main/java/nl/dtls/fairdatapoint/service/user/UserValidator.java b/src/main/java/nl/dtls/fairdatapoint/service/user/UserValidator.java index 50bd12528..f904d38bf 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/user/UserValidator.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/user/UserValidator.java @@ -1,3 +1,25 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package nl.dtls.fairdatapoint.service.user; import nl.dtls.fairdatapoint.database.mongo.repository.UserRepository; diff --git a/src/main/resources/defaultNavigationShacl.ttl b/src/main/resources/defaultNavigationShacl.ttl new file mode 100644 index 000000000..ec3601ea6 --- /dev/null +++ b/src/main/resources/defaultNavigationShacl.ttl @@ -0,0 +1,34 @@ +@prefix : . +@prefix sh: . +@prefix r3d: . +@prefix dcat: . +@prefix dct: . + +:repoNavShape a sh:NodeShape ; + sh:targetClass r3d:Repository ; + sh:property [ + sh:path r3d:dataCatalog ; + sh:node :catNavShape ; + ] . + +:catNavShape a sh:NodeShape ; + sh:targetClass dcat:Catalog ; + sh:property [ + sh:path dcat:dataset ; + sh:node :datasetNavShape ; + ] ; + sh:property [ + sh:path dct:isPartOf ; + sh:node :repoNavShape ; + ] . + +:datasetNavShape a sh:NodeShape ; + sh:targetClass dcat:Dataset ; + sh:property [ + sh:path dcat:distribution ; + sh:node :distNavShape ; + ] ; + sh:property [ + sh:path dct:isPartOf ; + sh:node :catNavShape ; + ] . diff --git a/src/test/java/nl/dtls/fairdatapoint/acceptance/search/List_POST.java b/src/test/java/nl/dtls/fairdatapoint/acceptance/search/List_POST.java index ea8d74287..f6616f3f0 100644 --- a/src/test/java/nl/dtls/fairdatapoint/acceptance/search/List_POST.java +++ b/src/test/java/nl/dtls/fairdatapoint/acceptance/search/List_POST.java @@ -1,3 +1,25 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package nl.dtls.fairdatapoint.acceptance.search; import nl.dtls.fairdatapoint.WebIntegrationTest; diff --git a/src/test/java/nl/dtls/fairdatapoint/service/index/harvester/HarvesterServiceTest.java b/src/test/java/nl/dtls/fairdatapoint/service/index/harvester/HarvesterServiceTest.java new file mode 100644 index 000000000..b65f33ddc --- /dev/null +++ b/src/test/java/nl/dtls/fairdatapoint/service/index/harvester/HarvesterServiceTest.java @@ -0,0 +1,153 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.service.index.harvester; + +import nl.dtls.fairdatapoint.database.mongo.migration.development.resource.data.ResourceDefinitionFixtures; +import nl.dtls.fairdatapoint.database.rdf.migration.development.metadata.data.RdfMetadataFixtures; +import nl.dtls.fairdatapoint.database.rdf.migration.development.metadata.factory.MetadataFactoryImpl; +import nl.dtls.fairdatapoint.database.rdf.repository.exception.MetadataRepositoryException; +import nl.dtls.fairdatapoint.database.rdf.repository.generic.GenericMetadataRepository; +import nl.dtls.fairdatapoint.entity.resource.ResourceDefinition; +import nl.dtls.fairdatapoint.service.metadata.enhance.MetadataEnhancer; +import nl.dtls.fairdatapoint.service.resource.ResourceDefinitionCache; +import nl.dtls.fairdatapoint.vocabulary.R3D; +import org.eclipse.rdf4j.model.Model; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.RestTemplate; + +import static nl.dtls.fairdatapoint.entity.metadata.MetadataGetter.getUri; +import static nl.dtls.fairdatapoint.util.RdfIOUtil.write; +import static nl.dtls.fairdatapoint.util.ValueFactoryHelper.i; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +public class HarvesterServiceTest { + + @Mock + private RestTemplate restTemplate; + + @Mock + private ResourceDefinitionCache resourceDefinitionCache; + + @Spy + private GenericMetadataRepository genericMetadataRepository; + + @InjectMocks + private static MetadataEnhancer metadataEnhancer; + + @InjectMocks + private HarvesterService harvesterService; + + private final String repositoryUrl = "http://fairdatapoint.example"; + + private Model repository; + + private final String catalogUrl = "http://fairdatapoint.example/catalog/catalog-1"; + + private Model catalog; + + @BeforeEach + private void setup() { + // Setup resource definition; + ResourceDefinitionFixtures resourceDefinitionFixtures = new ResourceDefinitionFixtures(); + when(resourceDefinitionCache.getByUuid(resourceDefinitionFixtures.catalogDefinition().getUuid())) + .thenReturn(resourceDefinitionFixtures.catalogDefinition()); + + // Setup RDF fixtures + RdfMetadataFixtures fixtures = new RdfMetadataFixtures(new MetadataFactoryImpl()); + + // Create repository + repository = fixtures.repositoryMetadata(repositoryUrl); + ResourceDefinition rdRepository = resourceDefinitionFixtures.repositoryDefinition(); + + // Create catalog + catalog = fixtures.catalog1(repositoryUrl, getUri(repository)); + repository.add(i(repositoryUrl), R3D.DATACATALOG, i(catalogUrl)); + + // Enhance repository with links + metadataEnhancer.enhanceWithLinks(i(repositoryUrl), repository, rdRepository, repositoryUrl, repository); + } + + @Test + public void harvestSucceed() throws MetadataRepositoryException { + // GIVEN: Mock webserver + mockEndpoint(repositoryUrl, repository); + mockEndpoint(catalogUrl, catalog); + + // WHEN: + harvesterService.harvest(repositoryUrl); + + // THEN: + verify(genericMetadataRepository, times(1)).save(anyList(), eq(i(repositoryUrl))); + verify(genericMetadataRepository, times(1)).save(anyList(), eq(i(catalogUrl))); + } + + @Test + public void harvestFailedForLinkedChildren() throws MetadataRepositoryException { + // GIVEN: Mock webserver + mockEndpoint(repositoryUrl, repository); + mockEndpoint404(catalogUrl); + + // WHEN: + harvesterService.harvest(repositoryUrl); + + // THEN: + verify(genericMetadataRepository, times(1)).save(anyList(), eq(i(repositoryUrl))); + verify(genericMetadataRepository, times(0)).save(anyList(), eq(i(catalogUrl))); + } + + private void mockEndpoint(String url, Model body) { + // Create response + MultiValueMap headers = new LinkedMultiValueMap<>(); + headers.set("Content-Type", "text/turtle"); + ResponseEntity responseBody = new ResponseEntity<>(write(body), headers, HttpStatus.OK); + + // Mock + when(restTemplate.exchange(eq(url), eq(HttpMethod.GET), any(HttpEntity.class), eq(String.class))) + .thenReturn(responseBody); + } + + private void mockEndpoint404(String url) { + // Create response + ResponseEntity responseBody = new ResponseEntity<>("", HttpStatus.NOT_FOUND); + + // Mock + when(restTemplate.exchange(eq(url), eq(HttpMethod.GET), any(HttpEntity.class), eq(String.class))) + .thenReturn(responseBody); + } + +} diff --git a/src/test/java/nl/dtls/fairdatapoint/utils/TestRdfMetadataFixtures.java b/src/test/java/nl/dtls/fairdatapoint/utils/TestRdfMetadataFixtures.java index d130c8f78..424a63265 100644 --- a/src/test/java/nl/dtls/fairdatapoint/utils/TestRdfMetadataFixtures.java +++ b/src/test/java/nl/dtls/fairdatapoint/utils/TestRdfMetadataFixtures.java @@ -24,6 +24,7 @@ import nl.dtls.fairdatapoint.database.mongo.migration.development.resource.data.ResourceDefinitionFixtures; import nl.dtls.fairdatapoint.database.rdf.migration.development.metadata.data.RdfMetadataFixtures; +import nl.dtls.fairdatapoint.database.rdf.migration.development.metadata.factory.MetadataFactory; import nl.dtls.fairdatapoint.entity.resource.ResourceDefinition; import nl.dtls.fairdatapoint.service.metadata.enhance.MetadataEnhancer; import org.eclipse.rdf4j.model.Model; @@ -36,17 +37,24 @@ @Service public class TestRdfMetadataFixtures extends RdfMetadataFixtures { - @Autowired - @Qualifier("persistentUrl") - private String persistentUrl; - public String alternativePersistentUrl = "https://lorentz.fair-dtls.surf-hosted.nl/fdp"; - @Autowired - private MetadataEnhancer metadataEnhancer; + private final String persistentUrl; + + private final MetadataEnhancer metadataEnhancer; + + private final ResourceDefinitionFixtures resourceDefinitionFixtures; @Autowired - private ResourceDefinitionFixtures resourceDefinitionFixtures; + public TestRdfMetadataFixtures(MetadataFactory metadataFactory, + @Qualifier("persistentUrl") String persistentUrl, + MetadataEnhancer metadataEnhancer, + ResourceDefinitionFixtures resourceDefinitionFixtures) { + super(metadataFactory); + this.persistentUrl = persistentUrl; + this.metadataEnhancer = metadataEnhancer; + this.resourceDefinitionFixtures = resourceDefinitionFixtures; + } public Model repositoryMetadata() { Model metadata = super.repositoryMetadata(persistentUrl); From e1f1cad541502dab0df01fb831670f6472b1787c Mon Sep 17 00:00:00 2001 From: Vojtech Knaisl Date: Tue, 1 Dec 2020 18:38:49 +0100 Subject: [PATCH 07/12] Add profile --- .../controller/profile/ProfileController.java | 70 +++++++++++++++++++ .../api/controller/shape/ShapeController.java | 14 +++- .../service/profile/ProfileService.java | 60 ++++++++++++++++ .../service/shape/ShapeService.java | 7 ++ .../util/ValueFactoryHelper.java | 4 ++ 5 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 src/main/java/nl/dtls/fairdatapoint/api/controller/profile/ProfileController.java create mode 100644 src/main/java/nl/dtls/fairdatapoint/service/profile/ProfileService.java diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/profile/ProfileController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/profile/ProfileController.java new file mode 100644 index 000000000..a1e7c6b46 --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/profile/ProfileController.java @@ -0,0 +1,70 @@ +/** + * The MIT License + * Copyright © 2017 DTL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package nl.dtls.fairdatapoint.api.controller.profile; + +import nl.dtls.fairdatapoint.api.dto.shape.ShapeChangeDTO; +import nl.dtls.fairdatapoint.api.dto.shape.ShapeDTO; +import nl.dtls.fairdatapoint.entity.exception.ResourceNotFoundException; +import nl.dtls.fairdatapoint.service.profile.ProfileService; +import nl.dtls.fairdatapoint.service.shape.ShapeService; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Model; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; +import javax.validation.Valid; +import java.util.List; +import java.util.Optional; + +import static java.lang.String.format; +import static nl.dtls.fairdatapoint.util.HttpUtil.getRequestURL; +import static nl.dtls.fairdatapoint.util.ValueFactoryHelper.i; + +@RestController +@RequestMapping("/profile") +public class ProfileController { + + @Autowired + @Qualifier("persistentUrl") + private String persistentUrl; + + @Autowired + private ProfileService profileService; + + @RequestMapping(value = "/{uuid}", method = RequestMethod.GET, produces = {"!application/json"}) + public ResponseEntity getShapeContent(HttpServletRequest request, @PathVariable final String uuid) + throws ResourceNotFoundException { + IRI uri = i(getRequestURL(request, persistentUrl)); + Optional oDto = profileService.getProfileByUuid(uuid, uri); + if (oDto.isPresent()) { + return new ResponseEntity<>(oDto.get(), HttpStatus.OK); + } else { + throw new ResourceNotFoundException(format("Profile '%s' doesn't exist", uuid)); + } + } + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/api/controller/shape/ShapeController.java b/src/main/java/nl/dtls/fairdatapoint/api/controller/shape/ShapeController.java index 078fbcca6..6ef141ef5 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/controller/shape/ShapeController.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/controller/shape/ShapeController.java @@ -26,6 +26,7 @@ import nl.dtls.fairdatapoint.api.dto.shape.ShapeDTO; import nl.dtls.fairdatapoint.entity.exception.ResourceNotFoundException; import nl.dtls.fairdatapoint.service.shape.ShapeService; +import org.eclipse.rdf4j.model.Model; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -56,7 +57,7 @@ public ResponseEntity createShape(@RequestBody @Valid ShapeChangeDTO r return new ResponseEntity<>(dto, HttpStatus.OK); } - @RequestMapping(value = "/{uuid}", method = RequestMethod.GET) + @RequestMapping(value = "/{uuid}", method = RequestMethod.GET, produces = {"application/json"}) public ResponseEntity getShape(@PathVariable final String uuid) throws ResourceNotFoundException { Optional oDto = shapeService.getShapeByUuid(uuid); @@ -67,6 +68,17 @@ public ResponseEntity getShape(@PathVariable final String uuid) } } + @RequestMapping(value = "/{uuid}", method = RequestMethod.GET, produces = {"!application/json"}) + public ResponseEntity getShapeContent(@PathVariable final String uuid) + throws ResourceNotFoundException { + Optional oDto = shapeService.getShapeContentByUuid(uuid); + if (oDto.isPresent()) { + return new ResponseEntity<>(oDto.get(), HttpStatus.OK); + } else { + throw new ResourceNotFoundException(format("Shape '%s' doesn't exist", uuid)); + } + } + @RequestMapping(value = "/{uuid}", method = RequestMethod.PUT) public ResponseEntity putShape(@PathVariable final String uuid, @RequestBody @Valid ShapeChangeDTO reqDto) throws ResourceNotFoundException { diff --git a/src/main/java/nl/dtls/fairdatapoint/service/profile/ProfileService.java b/src/main/java/nl/dtls/fairdatapoint/service/profile/ProfileService.java new file mode 100644 index 000000000..f6b18349b --- /dev/null +++ b/src/main/java/nl/dtls/fairdatapoint/service/profile/ProfileService.java @@ -0,0 +1,60 @@ +package nl.dtls.fairdatapoint.service.profile; + +import nl.dtls.fairdatapoint.service.shape.ShapeService; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Model; +import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.impl.LinkedHashModel; +import org.eclipse.rdf4j.model.util.ModelBuilder; +import org.eclipse.rdf4j.model.vocabulary.DCTERMS; +import org.eclipse.rdf4j.model.vocabulary.RDF; +import org.eclipse.rdf4j.model.vocabulary.RDFS; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Optional; + +import static java.lang.String.format; +import static nl.dtls.fairdatapoint.util.ValueFactoryHelper.*; + +@Service +public class ProfileService { + + private static final String PROFILE_PREFIX = "http://www.w3.org/ns/dx/prof"; + + @Autowired + @Qualifier("persistentUrl") + private String persistentUrl; + + @Autowired + private ShapeService shapeService; + + public Optional getProfileByUuid(String uuid, IRI uri) { + return shapeService.getShapeByUuid(uuid) + .map(shape -> { + ModelBuilder modelBuilder = new ModelBuilder(); + Resource resource = bn(); + modelBuilder.subject(resource); + modelBuilder.add(RDF.TYPE, i(format("%s#ResourceDescriptor", PROFILE_PREFIX))); + modelBuilder.add(DCTERMS.FORMAT, i("https://w3id.org/mediatype/text/turtle")); + modelBuilder.add(DCTERMS.CONFORMS_TO, i("https://www.w3.org/TR/shacl/")); + modelBuilder.add(i(format("%s#hasRole", PROFILE_PREFIX)), i(format("%s/role#Validation", + PROFILE_PREFIX))); + modelBuilder.add(i(format("%s#hasArtifact", PROFILE_PREFIX)), i(format("%s/shapes/%s", + persistentUrl, uuid))); + + Model profile = new LinkedHashModel(); + profile.add(uri, RDF.TYPE, i(format("%s#Profile", PROFILE_PREFIX))); + profile.add(uri, RDFS.LABEL, l(format("Profile for %s", shape.getName()))); + profile.add(uri, i(format("%s#isProfileOf", PROFILE_PREFIX)), i(format("%s/profile/core", + persistentUrl))); + profile.add(uri, i(format("%s#hasResource", PROFILE_PREFIX)), resource); + profile.addAll(new ArrayList<>(modelBuilder.build())); + + return profile; + }); + } + +} diff --git a/src/main/java/nl/dtls/fairdatapoint/service/shape/ShapeService.java b/src/main/java/nl/dtls/fairdatapoint/service/shape/ShapeService.java index 3c500a291..2429ece3f 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/shape/ShapeService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/shape/ShapeService.java @@ -72,6 +72,13 @@ public Optional getShapeByUuid(String uuid) { .map(shapeMapper::toDTO); } + public Optional getShapeContentByUuid(String uuid) { + return + shapeRepository + .findByUuid(uuid) + .map(shape -> RdfIOUtil.read(shape.getDefinition(), "")); + } + @PreAuthorize("hasRole('ADMIN')") public ShapeDTO createShape(ShapeChangeDTO reqDto) { shapeValidator.validate(reqDto); diff --git a/src/main/java/nl/dtls/fairdatapoint/util/ValueFactoryHelper.java b/src/main/java/nl/dtls/fairdatapoint/util/ValueFactoryHelper.java index ea060a198..eae3d1b9f 100644 --- a/src/main/java/nl/dtls/fairdatapoint/util/ValueFactoryHelper.java +++ b/src/main/java/nl/dtls/fairdatapoint/util/ValueFactoryHelper.java @@ -120,4 +120,8 @@ public static Statement s(Resource subject, IRI predicate, Value object) { public static Statement s(Resource subject, IRI predicate, Value object, Resource context) { return VF.createStatement(subject, predicate, object, context); } + + public static Resource bn() { + return VF.createBNode(); + } } From 6a7dec8abc3302accd673bc0dae9b5d3f98dd633 Mon Sep 17 00:00:00 2001 From: Vojtech Knaisl Date: Wed, 16 Dec 2020 20:59:06 +0100 Subject: [PATCH 08/12] Add RDF type to search --- .../api/dto/search/SearchResultDTO.java | 2 ++ .../common/AbstractMetadataRepository.java | 1 + .../fairdatapoint/entity/search/SearchResult.java | 2 ++ .../entity/search/SearchResultRelation.java | 6 ++---- .../fairdatapoint/service/search/SearchService.java | 13 +++++++++++-- .../repository/common/findEntityByLiteral.sparql | 4 +++- 6 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchResultDTO.java b/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchResultDTO.java index 5f26e6992..22c4c8ea1 100644 --- a/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchResultDTO.java +++ b/src/main/java/nl/dtls/fairdatapoint/api/dto/search/SearchResultDTO.java @@ -38,6 +38,8 @@ public class SearchResultDTO { private String uri; + private List types; + private String title; private String description; diff --git a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java index 938b429f7..3b8c49e04 100644 --- a/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java +++ b/src/main/java/nl/dtls/fairdatapoint/database/rdf/repository/common/AbstractMetadataRepository.java @@ -82,6 +82,7 @@ public List findByLiteral(Literal query) throws MetadataRepository .stream() .map(s -> new SearchResult( s.getValue("entity").stringValue(), + s.getValue("rdfType").stringValue(), s.getValue("title").stringValue(), ofNullable(s.getValue("description")).map(Value::stringValue).orElse(""), new SearchResultRelation( diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResult.java b/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResult.java index 9be7f8cb1..54e51088d 100644 --- a/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResult.java +++ b/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResult.java @@ -35,6 +35,8 @@ public class SearchResult { private String uri; + private String type; + private String title; private String description; diff --git a/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResultRelation.java b/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResultRelation.java index f779a15ce..365909c8f 100644 --- a/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResultRelation.java +++ b/src/main/java/nl/dtls/fairdatapoint/entity/search/SearchResultRelation.java @@ -22,13 +22,11 @@ */ package nl.dtls.fairdatapoint.entity.search; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; +import lombok.*; @NoArgsConstructor @AllArgsConstructor +@EqualsAndHashCode @Getter @Setter public class SearchResultRelation { diff --git a/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java b/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java index cab8990ba..1960a0028 100644 --- a/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java +++ b/src/main/java/nl/dtls/fairdatapoint/service/search/SearchService.java @@ -64,10 +64,19 @@ public List search(SearchQueryDTO reqDto) throws MetadataReposi }) .map(entry -> new SearchResultDTO( entry.getKey(), + entry.getValue() + .stream() + .map(SearchResult::getType) + .distinct() + .filter(t -> !t.equals("http://www.w3.org/ns/dcat#Resource")) + .collect(Collectors.toList()), entry.getValue().get(0).getTitle(), entry.getValue().get(0).getDescription(), - entry.getValue().stream() - .map(SearchResult::getRelation).collect(Collectors.toList()) + entry.getValue() + .stream() + .map(SearchResult::getRelation) + .distinct() + .collect(Collectors.toList()) )) .collect(toList()); } diff --git a/src/main/resources/nl/dtls/fairdatapoint/database/rdf/repository/common/findEntityByLiteral.sparql b/src/main/resources/nl/dtls/fairdatapoint/database/rdf/repository/common/findEntityByLiteral.sparql index c32ba05a2..77e9907a7 100644 --- a/src/main/resources/nl/dtls/fairdatapoint/database/rdf/repository/common/findEntityByLiteral.sparql +++ b/src/main/resources/nl/dtls/fairdatapoint/database/rdf/repository/common/findEntityByLiteral.sparql @@ -1,7 +1,9 @@ +prefix rdf: prefix dct: -SELECT ?entity ?title ?description ?relationPredicate ?relationObject WHERE { +SELECT ?entity ?rdfType ?title ?description ?relationPredicate ?relationObject WHERE { ?entity ?relationPredicate ?relationObject . + ?entity rdf:type ?rdfType . ?entity dct:title ?title . Optional { ?entity dct:description ?description } filter isLiteral(?relationObject) From 41c1361bbe1857da2309904c20f6606474d0e194 Mon Sep 17 00:00:00 2001 From: kburger Date: Mon, 16 Nov 2020 09:55:05 +0100 Subject: [PATCH 09/12] Added option for graphdb user credentials. --- .../fairdatapoint/config/RepositoryConfig.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/nl/dtls/fairdatapoint/config/RepositoryConfig.java b/src/main/java/nl/dtls/fairdatapoint/config/RepositoryConfig.java index 16c28dc3b..4ddf318be 100644 --- a/src/main/java/nl/dtls/fairdatapoint/config/RepositoryConfig.java +++ b/src/main/java/nl/dtls/fairdatapoint/config/RepositoryConfig.java @@ -61,6 +61,12 @@ public class RepositoryConfig { @Value("${repository.graphDb.repository:}") private String graphDbRepository; + + @Value("${repository.graphDb.username:}") + private String graphDbUsername; + + @Value("${repository.graphDb.password:}") + private String graphDbPassword; @Value("${repository.blazegraph.url:}") private String blazegraphUrl; @@ -149,8 +155,13 @@ private Repository getGraphDBRepository() { log.info("Setting up GraphDB Store"); try { if (!graphDbUrl.isEmpty() && !graphDbRepository.isEmpty()) { - RepositoryManager repositoryManager = new RemoteRepositoryManager(graphDbUrl); - repositoryManager.initialize(); + final RepositoryManager repositoryManager; + if (!graphDbUsername.isEmpty() && !graphDbPassword.isEmpty()) { + repositoryManager = RemoteRepositoryManager.getInstance(graphDbUrl, graphDbUsername, graphDbPassword); + } else { + repositoryManager = RemoteRepositoryManager.getInstance(graphDbUrl); + } + return repositoryManager.getRepository(graphDbRepository); } } catch (RepositoryConfigException | RepositoryException e) { From 5343ea4a05863e9aa4998caa42f07cb2750337df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Wed, 20 Jan 2021 18:08:31 +0100 Subject: [PATCH 10/12] Add CHANGELOG.md (#100) --- CHANGELOG.md | 155 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..d3bc8f386 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,155 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- Possibility to change profile and password for current user +- FDP Index functionality (moved from [FAIRDataPoint-index](https://github.com/FAIRDataTeam/FAIRDataPoint-index)) +- Harvestor for collecting metadata +- Support for PROF profiles +- Metadata search including RDF type + +### Fixed + +- Fix schema.org URL in pom.xml + +## [1.6.0] + +### Added + +- API keys +- Draft state for stored metadata + +## [1.5.0] + +### Added + +- Support custom resource definitions allowing more children + +### Changed + +- Upgrade Java JDK from 11 to 14 +- Switch to `OffsetDateTime` + +## [1.4.0] + +### Added + +- Ping service (for *call home* functionality) + +### Fixed + +- Fix saving of nested entities in metadata +- Fix Git app info in actuator endpoint + +### Removed + +- `themeTaxonomies` on incoming catalog + +## [1.3.0] + +### Added + +- Shape definitions with DASH support +- Endpoint for bootstrapping [Client] +- Validation for SHACL definitions in shapes +- Production migration for shape definitions + +### Changed + +- Embed fairmetadata4j into the project +- Split instanceUrl to clientUrl and persistentUrl + +### Removed + +- Internal PID system +- Dashboard identifier + +## [1.2.1] + +### Fixed + +- HTTP XFF (`X-Forwarded-For`) headers and `PUBLIC_PATH` envvar replaced using `instanceUrl` + +## [1.2.0] + +### Added + +- References to related repositories +- Option to customize metamodel (metadata layers) + +### Changed + +- Switch to GitHub Actions (from Travis CI) +- Swagger config reflects `instanceUrl` +- Reformatted and updated SHACL definitions to use `sh:and` according to the FDP specification + +## [1.1.0] + +### Added + +- Endpoint for [Client] configuration +- Spring Boot Actuator for monitoring and service info + +### Changed + +- Unified package names (`dtl` to `dtls`) +- Loading of production configuration + +### Fixed + +- Fix crashing on mapping null descriptions, licenses, etc. + +### Removed + +- Fallback to `InMemory` when a connection to the configured repository fails + +## [1.0.0] + +Refactored and cleaned-up version of reference FAIR Data Point implementation supporting a new [FAIRDataPoint-client](https://github.com/FAIRDataTeam/FAIRDataPoint-client). + +### Added + +- ACLs and use Spring Security with Mongo for authentication +- User Management and Metadata for [FAIRDataPoint-client](https://github.com/FAIRDataTeam/FAIRDataPoint-client) +- Themes caching for catalog and several other optimizations + +### Changed + +- Complete refactoring, cleaning accumulated changed and deprecations, reformatting code +- Upgrade Java JDK from 8 to 11 +- Enhanced Swagger UI API documentation +- Improve CI (on Travis) to automatically build and publish Docker image + +### Fixed + +- Several fixes of metadata, configuration, tests, and convertors + +## [0.1-beta] + +The first release of reference FAIR Data Point implementation. + +### Added + +- REST API according to the FDP specification supporting `GET`, `POST`, and `PATCH` for **repository**, **catalog**, **dataset**, and **distribution** layers +- API documentation using Swagger UI + + +[Client]: https://github.com/FAIRDataTeam/FAIRDataPoint-client + +[Unreleased]: /../../compare/master...develop +[0.1-beta]: /../../tree/0.1-beta +[1.0.0]: /../../tree/v1.0.0 +[1.1.0]: /../../tree/v1.1.0 +[1.2.0]: /../../tree/v1.2.0 +[1.2.1]: /../../tree/v1.2.1 +[1.3.0]: /../../tree/v1.3.0 +[1.4.0]: /../../tree/v1.4.0 +[1.5.0]: /../../tree/v1.5.0 +[1.6.0]: /../../tree/v1.6.0 From 8e2ac5ec9c8315e71d0235d9e38244e00fd520be Mon Sep 17 00:00:00 2001 From: kburger <6997485+kburger@users.noreply.github.com> Date: Wed, 20 Jan 2021 18:09:26 +0100 Subject: [PATCH 11/12] Readme improvements (#99) * Updated readme to reflect need for jdk14+, and add draft section on mongo setup. * Added default mongo host for localhost on development profile. * Fixed protocol for mongo uri. --- README.md | 12 +++++++++--- src/main/resources/application-development.yml | 7 ++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 251a92b60..d8349e5a3 100644 --- a/README.md +++ b/README.md @@ -20,16 +20,22 @@ More information about FDP and how to deploy can be found at [FDP Deployment Doc **Stack:** - - **Java** (recommended JDK 14) + - **Java** (minimally JDK 14, or higher) - **Maven** (recommended 3.2.5 or higher) - **Docker** (recommended 17.09.0-ce or higher) - *for build of production image* ### Build & Run -Run these commands from the root of the project +To run the application, a mongodb instance is required to be running. To configure the mongodb address, instruct spring-boot to use the `development` profile. Run these commands from the root of the project. + +```bash +$ mvn spring-boot:run -Dspring-boot.run.profiles=development +``` + +Alternatively, create an `application.yml` file in the project root and [configure the mongodb address](https://fairdatapoint.readthedocs.io/en/latest/deployment/advanced-configuration.html#mongo-db), and then run these commands from the root of the project. ```bash -$ mvn spring-boot:start +$ mvn spring-boot:run ``` ### Run tests diff --git a/src/main/resources/application-development.yml b/src/main/resources/application-development.yml index 3f80c8345..a9e2f1908 100644 --- a/src/main/resources/application-development.yml +++ b/src/main/resources/application-development.yml @@ -4,4 +4,9 @@ ping: security: jwt: token: - expiration: 999 \ No newline at end of file + expiration: 999 + +spring: + data: + mongodb: + uri: mongodb://localhost:27017/fdp From fc86c4903b2a149593b17bd430f06107bc3f7468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Wed, 20 Jan 2021 18:11:37 +0100 Subject: [PATCH 12/12] Release 1.7.0 --- CHANGELOG.md | 3 +++ pom.xml | 2 +- src/main/java/nl/dtls/fairdatapoint/config/SwaggerConfig.java | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3bc8f386..2ea850b0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.7.0] + ### Added - Possibility to change profile and password for current user @@ -153,3 +155,4 @@ The first release of reference FAIR Data Point implementation. [1.4.0]: /../../tree/v1.4.0 [1.5.0]: /../../tree/v1.5.0 [1.6.0]: /../../tree/v1.6.0 +[1.7.0]: /../../tree/v1.7.0 diff --git a/pom.xml b/pom.xml index d6f84623b..42b3f16cc 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ nl.dtls fairdatapoint - 1.6.0 + 1.7.0 jar FairDataPoint diff --git a/src/main/java/nl/dtls/fairdatapoint/config/SwaggerConfig.java b/src/main/java/nl/dtls/fairdatapoint/config/SwaggerConfig.java index 8c9426f4c..159c72a68 100644 --- a/src/main/java/nl/dtls/fairdatapoint/config/SwaggerConfig.java +++ b/src/main/java/nl/dtls/fairdatapoint/config/SwaggerConfig.java @@ -71,7 +71,7 @@ private ApiInfo apiInfo() { " FAIR Data Point Specification" + " " + "
", - "1.6.0", + "1.7.0", "ATO", new Contact("Luiz Bonino", "https://github.com/FAIRDataTeam/FAIRDataPoint",