From 66caeb1edef5af119a76ff21008feb1f41505f87 Mon Sep 17 00:00:00 2001 From: Alex Maltsev Date: Fri, 26 Jul 2024 16:02:21 +0300 Subject: [PATCH 1/9] Added default ttl seconds. --- .../cache/config/ModuleStorageConfig.java | 19 +++++++++++++++++++ .../storage/PostModuleStorageHandler.java | 11 +++++++++-- .../org/prebid/cache/model/ModulePayload.java | 1 - src/main/resources/application.yml | 3 ++- 4 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/prebid/cache/config/ModuleStorageConfig.java diff --git a/src/main/java/org/prebid/cache/config/ModuleStorageConfig.java b/src/main/java/org/prebid/cache/config/ModuleStorageConfig.java new file mode 100644 index 0000000..e1d7b28 --- /dev/null +++ b/src/main/java/org/prebid/cache/config/ModuleStorageConfig.java @@ -0,0 +1,19 @@ +package org.prebid.cache.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; +import org.springframework.validation.annotation.Validated; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Configuration +@Validated +@ConfigurationProperties(prefix = "module.storage") +public class ModuleStorageConfig { + + Long defaultTtlSeconds; +} diff --git a/src/main/java/org/prebid/cache/handlers/storage/PostModuleStorageHandler.java b/src/main/java/org/prebid/cache/handlers/storage/PostModuleStorageHandler.java index 5c8ad9f..401c721 100644 --- a/src/main/java/org/prebid/cache/handlers/storage/PostModuleStorageHandler.java +++ b/src/main/java/org/prebid/cache/handlers/storage/PostModuleStorageHandler.java @@ -5,6 +5,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.prebid.cache.builders.PrebidServerResponseBuilder; +import org.prebid.cache.config.ModuleStorageConfig; import org.prebid.cache.exceptions.BadRequestException; import org.prebid.cache.model.ModulePayload; import org.prebid.cache.model.Payload; @@ -20,6 +21,7 @@ import reactor.core.publisher.SynchronousSink; import reactor.core.scheduler.Schedulers; +import java.util.Optional; import java.util.stream.Collectors; @Slf4j @@ -33,6 +35,7 @@ public class PostModuleStorageHandler { private final ModuleCompositeRepository moduleRepository; private final PrebidServerResponseBuilder responseBuilder; private final ApiConfig apiConfig; + private final ModuleStorageConfig moduleStorageConfig; public Mono save(final ServerRequest request) { if (!isApiKeyValid(request)) { @@ -66,11 +69,15 @@ private void validateModulePayload(final ModulePayload payload, final Synchronou } } - private static PayloadWrapper mapToPayloadWrapper(final ModulePayload payload) { + private PayloadWrapper mapToPayloadWrapper(final ModulePayload payload) { + final long ttlSeconds = Optional.ofNullable(payload.getTtlseconds()) + .map(Integer::longValue) + .orElse(moduleStorageConfig.getDefaultTtlSeconds()); + return PayloadWrapper.builder() .id(payload.getKey()) .prefix(StringUtils.EMPTY) - .expiry(payload.getTtlseconds().longValue()) + .expiry(ttlSeconds) .payload(Payload.of(payload.getType().toString(), payload.getKey(), payload.getValue())) .build(); } diff --git a/src/main/java/org/prebid/cache/model/ModulePayload.java b/src/main/java/org/prebid/cache/model/ModulePayload.java index 193f922..47177a6 100644 --- a/src/main/java/org/prebid/cache/model/ModulePayload.java +++ b/src/main/java/org/prebid/cache/model/ModulePayload.java @@ -23,7 +23,6 @@ public class ModulePayload { @NotEmpty public String application; - @NotNull @Min(0) public Integer ttlseconds; } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 7f1a79d..2f49df3 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -4,7 +4,7 @@ info.app.description: @project.description@ info.app.version: @project.version@ spring.main.banner-mode: "off" api.cache-path: /cache -api.module-storage-path: /module-storage +api.module-storage-path: /pbc-storage api.api-key: API_KEY server.port: 8080 server.compression.enabled: true @@ -30,6 +30,7 @@ cache: module: storage: redis: {} + default-ttl-seconds: 300 # logging logging.level.root: info From 2ceff5b4498ba97b3f77357c7683c939c200e27f Mon Sep 17 00:00:00 2001 From: Alex Maltsev Date: Fri, 26 Jul 2024 16:42:32 +0300 Subject: [PATCH 2/9] Updated naming. --- ...uleStorageConfig.java => StorageConfig.java} | 4 ++-- ...orageHandler.java => GetStorageHandler.java} | 2 +- ...rageHandler.java => PostStorageHandler.java} | 8 ++++---- ...leCompositeRedisConfigurationProperties.java | 2 +- .../org/prebid/cache/routers/ApiConfig.java | 2 +- .../org/prebid/cache/routers/ApiRouter.java | 14 +++++++------- src/main/resources/application.yml | 9 ++++----- ...erTests.java => GetStorageHandlerTests.java} | 10 +++++----- ...rTests.java => PostStorageHandlerTests.java} | 17 +++++++++++------ src/test/resources/application.properties | 2 +- 10 files changed, 37 insertions(+), 33 deletions(-) rename src/main/java/org/prebid/cache/config/{ModuleStorageConfig.java => StorageConfig.java} (82%) rename src/main/java/org/prebid/cache/handlers/storage/{GetModuleStorageHandler.java => GetStorageHandler.java} (98%) rename src/main/java/org/prebid/cache/handlers/storage/{PostModuleStorageHandler.java => PostStorageHandler.java} (93%) rename src/test/java/org/prebid/cache/handlers/{GetModuleStorageHandlerTests.java => GetStorageHandlerTests.java} (91%) rename src/test/java/org/prebid/cache/handlers/{PostModuleStorageHandlerTests.java => PostStorageHandlerTests.java} (90%) diff --git a/src/main/java/org/prebid/cache/config/ModuleStorageConfig.java b/src/main/java/org/prebid/cache/config/StorageConfig.java similarity index 82% rename from src/main/java/org/prebid/cache/config/ModuleStorageConfig.java rename to src/main/java/org/prebid/cache/config/StorageConfig.java index e1d7b28..e002ae5 100644 --- a/src/main/java/org/prebid/cache/config/ModuleStorageConfig.java +++ b/src/main/java/org/prebid/cache/config/StorageConfig.java @@ -12,8 +12,8 @@ @AllArgsConstructor @Configuration @Validated -@ConfigurationProperties(prefix = "module.storage") -public class ModuleStorageConfig { +@ConfigurationProperties(prefix = "storage") +public class StorageConfig { Long defaultTtlSeconds; } diff --git a/src/main/java/org/prebid/cache/handlers/storage/GetModuleStorageHandler.java b/src/main/java/org/prebid/cache/handlers/storage/GetStorageHandler.java similarity index 98% rename from src/main/java/org/prebid/cache/handlers/storage/GetModuleStorageHandler.java rename to src/main/java/org/prebid/cache/handlers/storage/GetStorageHandler.java index 061de8b..3c1c06a 100644 --- a/src/main/java/org/prebid/cache/handlers/storage/GetModuleStorageHandler.java +++ b/src/main/java/org/prebid/cache/handlers/storage/GetStorageHandler.java @@ -17,7 +17,7 @@ @Component @RequiredArgsConstructor -public class GetModuleStorageHandler { +public class GetStorageHandler { private static final String API_KEY_HEADER = "x-pbc-api-key"; diff --git a/src/main/java/org/prebid/cache/handlers/storage/PostModuleStorageHandler.java b/src/main/java/org/prebid/cache/handlers/storage/PostStorageHandler.java similarity index 93% rename from src/main/java/org/prebid/cache/handlers/storage/PostModuleStorageHandler.java rename to src/main/java/org/prebid/cache/handlers/storage/PostStorageHandler.java index 401c721..052552c 100644 --- a/src/main/java/org/prebid/cache/handlers/storage/PostModuleStorageHandler.java +++ b/src/main/java/org/prebid/cache/handlers/storage/PostStorageHandler.java @@ -5,7 +5,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.prebid.cache.builders.PrebidServerResponseBuilder; -import org.prebid.cache.config.ModuleStorageConfig; +import org.prebid.cache.config.StorageConfig; import org.prebid.cache.exceptions.BadRequestException; import org.prebid.cache.model.ModulePayload; import org.prebid.cache.model.Payload; @@ -27,7 +27,7 @@ @Slf4j @Component @RequiredArgsConstructor -public class PostModuleStorageHandler { +public class PostStorageHandler { private static final String API_KEY_HEADER = "x-pbc-api-key"; @@ -35,7 +35,7 @@ public class PostModuleStorageHandler { private final ModuleCompositeRepository moduleRepository; private final PrebidServerResponseBuilder responseBuilder; private final ApiConfig apiConfig; - private final ModuleStorageConfig moduleStorageConfig; + private final StorageConfig storageConfig; public Mono save(final ServerRequest request) { if (!isApiKeyValid(request)) { @@ -72,7 +72,7 @@ private void validateModulePayload(final ModulePayload payload, final Synchronou private PayloadWrapper mapToPayloadWrapper(final ModulePayload payload) { final long ttlSeconds = Optional.ofNullable(payload.getTtlseconds()) .map(Integer::longValue) - .orElse(moduleStorageConfig.getDefaultTtlSeconds()); + .orElse(storageConfig.getDefaultTtlSeconds()); return PayloadWrapper.builder() .id(payload.getKey()) diff --git a/src/main/java/org/prebid/cache/repository/redis/module/storage/ModuleCompositeRedisConfigurationProperties.java b/src/main/java/org/prebid/cache/repository/redis/module/storage/ModuleCompositeRedisConfigurationProperties.java index ac6b595..d7802d7 100644 --- a/src/main/java/org/prebid/cache/repository/redis/module/storage/ModuleCompositeRedisConfigurationProperties.java +++ b/src/main/java/org/prebid/cache/repository/redis/module/storage/ModuleCompositeRedisConfigurationProperties.java @@ -9,7 +9,7 @@ @Data @Component -@ConfigurationProperties(prefix = "module.storage") +@ConfigurationProperties(prefix = "storage") public class ModuleCompositeRedisConfigurationProperties { private Map redis; diff --git a/src/main/java/org/prebid/cache/routers/ApiConfig.java b/src/main/java/org/prebid/cache/routers/ApiConfig.java index 841a1e7..cf5216f 100644 --- a/src/main/java/org/prebid/cache/routers/ApiConfig.java +++ b/src/main/java/org/prebid/cache/routers/ApiConfig.java @@ -20,7 +20,7 @@ public class ApiConfig { private String cachePath; @NotEmpty - private String moduleStoragePath; + private String storagePath; @NotEmpty private String apiKey; diff --git a/src/main/java/org/prebid/cache/routers/ApiRouter.java b/src/main/java/org/prebid/cache/routers/ApiRouter.java index 689f9be..21388fb 100644 --- a/src/main/java/org/prebid/cache/routers/ApiRouter.java +++ b/src/main/java/org/prebid/cache/routers/ApiRouter.java @@ -4,8 +4,8 @@ import org.prebid.cache.handlers.ErrorHandler; import org.prebid.cache.handlers.cache.GetCacheHandler; import org.prebid.cache.handlers.cache.PostCacheHandler; -import org.prebid.cache.handlers.storage.GetModuleStorageHandler; -import org.prebid.cache.handlers.storage.PostModuleStorageHandler; +import org.prebid.cache.handlers.storage.GetStorageHandler; +import org.prebid.cache.handlers.storage.PostStorageHandler; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; @@ -28,8 +28,8 @@ public class ApiRouter { @Bean RouterFunction doRoute(final GetCacheHandler getCacheHandler, final PostCacheHandler postCacheHandler, - final GetModuleStorageHandler getModuleStorageHandler, - final PostModuleStorageHandler postModuleStorageHandler, + final GetStorageHandler getStorageHandler, + final PostStorageHandler postStorageHandler, final ErrorHandler errorHandler, final ApiConfig apiConfig) { @@ -45,9 +45,9 @@ RouterFunction doRoute(final GetCacheHandler getCacheHandler, GET(apiConfig.getCachePath()).and(accept(MediaType.APPLICATION_XML)), getCacheHandler::fetch) .andRoute( - POST(apiConfig.getModuleStoragePath()).and(accept(MediaType.APPLICATION_JSON)), - postModuleStorageHandler::save) - .andRoute(GET(apiConfig.getModuleStoragePath()), getModuleStorageHandler::fetch) + POST(apiConfig.getStoragePath()).and(accept(MediaType.APPLICATION_JSON)), + postStorageHandler::save) + .andRoute(GET(apiConfig.getStoragePath()), getStorageHandler::fetch) .andOther(route(RequestPredicates.all(), errorHandler::invalidRequest)); } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 2f49df3..fc83b1e 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -4,7 +4,7 @@ info.app.description: @project.description@ info.app.version: @project.version@ spring.main.banner-mode: "off" api.cache-path: /cache -api.module-storage-path: /pbc-storage +api.storage-path: /storage api.api-key: API_KEY server.port: 8080 server.compression.enabled: true @@ -27,10 +27,9 @@ cache: clients_cache_size: 1000 host_param_protocol: https -module: - storage: - redis: {} - default-ttl-seconds: 300 +storage: + redis: {} + default-ttl-seconds: 300 # logging logging.level.root: info diff --git a/src/test/java/org/prebid/cache/handlers/GetModuleStorageHandlerTests.java b/src/test/java/org/prebid/cache/handlers/GetStorageHandlerTests.java similarity index 91% rename from src/test/java/org/prebid/cache/handlers/GetModuleStorageHandlerTests.java rename to src/test/java/org/prebid/cache/handlers/GetStorageHandlerTests.java index f59ac31..ed6a1a6 100644 --- a/src/test/java/org/prebid/cache/handlers/GetModuleStorageHandlerTests.java +++ b/src/test/java/org/prebid/cache/handlers/GetStorageHandlerTests.java @@ -6,7 +6,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.prebid.cache.builders.PrebidServerResponseBuilder; -import org.prebid.cache.handlers.storage.GetModuleStorageHandler; +import org.prebid.cache.handlers.storage.GetStorageHandler; import org.prebid.cache.model.Payload; import org.prebid.cache.model.PayloadWrapper; import org.prebid.cache.repository.redis.module.storage.ModuleCompositeRepository; @@ -27,13 +27,13 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = { - GetModuleStorageHandler.class, + GetStorageHandler.class, PrebidServerResponseBuilder.class, ApiConfig.class }) @EnableConfigurationProperties @SpringBootTest -public class GetModuleStorageHandlerTests { +public class GetStorageHandlerTests { @Autowired ApiConfig apiConfig; @@ -44,13 +44,13 @@ public class GetModuleStorageHandlerTests { @MockBean ModuleCompositeRepository moduleCompositeRepository; - GetModuleStorageHandler handler; + GetStorageHandler handler; WireMockServer serverMock; @BeforeEach public void setup() { - handler = new GetModuleStorageHandler(moduleCompositeRepository, responseBuilder, apiConfig); + handler = new GetStorageHandler(moduleCompositeRepository, responseBuilder, apiConfig); serverMock = new WireMockServer(8080); serverMock.start(); } diff --git a/src/test/java/org/prebid/cache/handlers/PostModuleStorageHandlerTests.java b/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java similarity index 90% rename from src/test/java/org/prebid/cache/handlers/PostModuleStorageHandlerTests.java rename to src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java index 4e08481..6808de1 100644 --- a/src/test/java/org/prebid/cache/handlers/PostModuleStorageHandlerTests.java +++ b/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java @@ -7,14 +7,14 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.prebid.cache.builders.PrebidServerResponseBuilder; -import org.prebid.cache.handlers.storage.PostModuleStorageHandler; +import org.prebid.cache.config.StorageConfig; +import org.prebid.cache.handlers.storage.PostStorageHandler; import org.prebid.cache.model.ModulePayload; import org.prebid.cache.model.Payload; import org.prebid.cache.model.PayloadWrapper; import org.prebid.cache.repository.redis.module.storage.ModuleCompositeRepository; import org.prebid.cache.routers.ApiConfig; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; @@ -33,13 +33,13 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = { - PostModuleStorageHandler.class, + PostStorageHandler.class, PrebidServerResponseBuilder.class, ApiConfig.class }) @EnableConfigurationProperties @SpringBootTest -class PostModuleStorageHandlerTests { +class PostStorageHandlerTests { @Autowired ApiConfig apiConfig; @@ -53,13 +53,18 @@ class PostModuleStorageHandlerTests { @MockBean Validator validator; - PostModuleStorageHandler handler; + PostStorageHandler handler; WireMockServer serverMock; @BeforeEach public void setup() { - handler = new PostModuleStorageHandler(validator, moduleCompositeRepository, responseBuilder, apiConfig); + handler = new PostStorageHandler( + validator, + moduleCompositeRepository, + responseBuilder, + apiConfig, + new StorageConfig(9999L)); serverMock = new WireMockServer(8080); serverMock.start(); } diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index 2f4ef22..a135060 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -1,6 +1,6 @@ spring.main.banner-mode=off api.cache-path=/cache -api.module-storage-path=/module-storage +api.storage-path=/module-storage api.api-key=API_KEY server.port=8080 server.compression.enabled=true From 3b00fef1f4084e68fb311004b5a5672bd8de77da Mon Sep 17 00:00:00 2001 From: Alex Maltsev Date: Fri, 26 Jul 2024 16:51:55 +0300 Subject: [PATCH 3/9] Fixed integration test. --- .../org/prebid/cache/handlers/PostStorageHandlerTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java b/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java index 6808de1..3b967e5 100644 --- a/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java +++ b/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java @@ -33,9 +33,9 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = { - PostStorageHandler.class, PrebidServerResponseBuilder.class, - ApiConfig.class + ApiConfig.class, + StorageConfig.class, }) @EnableConfigurationProperties @SpringBootTest From ec0ab6557fbd56f7afca2f1c7229e12a4b5cdd55 Mon Sep 17 00:00:00 2001 From: Alex Maltsev Date: Fri, 26 Jul 2024 16:53:24 +0300 Subject: [PATCH 4/9] More renamings. --- .../cache/handlers/storage/PostStorageHandler.java | 14 +++++++------- .../{ModulePayload.java => StoragePayload.java} | 2 +- .../cache/handlers/PostStorageHandlerTests.java | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) rename src/main/java/org/prebid/cache/model/{ModulePayload.java => StoragePayload.java} (94%) diff --git a/src/main/java/org/prebid/cache/handlers/storage/PostStorageHandler.java b/src/main/java/org/prebid/cache/handlers/storage/PostStorageHandler.java index 052552c..a41c832 100644 --- a/src/main/java/org/prebid/cache/handlers/storage/PostStorageHandler.java +++ b/src/main/java/org/prebid/cache/handlers/storage/PostStorageHandler.java @@ -7,7 +7,7 @@ import org.prebid.cache.builders.PrebidServerResponseBuilder; import org.prebid.cache.config.StorageConfig; import org.prebid.cache.exceptions.BadRequestException; -import org.prebid.cache.model.ModulePayload; +import org.prebid.cache.model.StoragePayload; import org.prebid.cache.model.Payload; import org.prebid.cache.model.PayloadWrapper; import org.prebid.cache.repository.redis.module.storage.ModuleCompositeRepository; @@ -42,12 +42,12 @@ public Mono save(final ServerRequest request) { return ServerResponse.status(HttpStatus.UNAUTHORIZED).build(); } - return request.body(BodyExtractors.toMono(ModulePayload.class)) + return request.body(BodyExtractors.toMono(StoragePayload.class)) .switchIfEmpty(Mono.error(new BadRequestException("Empty body"))) .handle(this::validateModulePayload) - .flatMap(modulePayload -> moduleRepository.save( - modulePayload.getApplication(), - mapToPayloadWrapper(modulePayload))) + .flatMap(storagePayload -> moduleRepository.save( + storagePayload.getApplication(), + mapToPayloadWrapper(storagePayload))) .subscribeOn(Schedulers.parallel()) .flatMap(ignored -> ServerResponse.noContent().build()) .onErrorResume(error -> responseBuilder.error(Mono.just(error), request)); @@ -57,7 +57,7 @@ private boolean isApiKeyValid(final ServerRequest request) { return StringUtils.equals(request.headers().firstHeader(API_KEY_HEADER), apiConfig.getApiKey()); } - private void validateModulePayload(final ModulePayload payload, final SynchronousSink sink) { + private void validateModulePayload(final StoragePayload payload, final SynchronousSink sink) { final var result = validator.validate(payload); if (result.isEmpty()) { sink.next(payload); @@ -69,7 +69,7 @@ private void validateModulePayload(final ModulePayload payload, final Synchronou } } - private PayloadWrapper mapToPayloadWrapper(final ModulePayload payload) { + private PayloadWrapper mapToPayloadWrapper(final StoragePayload payload) { final long ttlSeconds = Optional.ofNullable(payload.getTtlseconds()) .map(Integer::longValue) .orElse(storageConfig.getDefaultTtlSeconds()); diff --git a/src/main/java/org/prebid/cache/model/ModulePayload.java b/src/main/java/org/prebid/cache/model/StoragePayload.java similarity index 94% rename from src/main/java/org/prebid/cache/model/ModulePayload.java rename to src/main/java/org/prebid/cache/model/StoragePayload.java index 47177a6..a9d7e61 100644 --- a/src/main/java/org/prebid/cache/model/ModulePayload.java +++ b/src/main/java/org/prebid/cache/model/StoragePayload.java @@ -9,7 +9,7 @@ @Value @Builder -public class ModulePayload { +public class StoragePayload { @NotEmpty public String key; diff --git a/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java b/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java index 3b967e5..08b2753 100644 --- a/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java +++ b/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java @@ -9,7 +9,7 @@ import org.prebid.cache.builders.PrebidServerResponseBuilder; import org.prebid.cache.config.StorageConfig; import org.prebid.cache.handlers.storage.PostStorageHandler; -import org.prebid.cache.model.ModulePayload; +import org.prebid.cache.model.StoragePayload; import org.prebid.cache.model.Payload; import org.prebid.cache.model.PayloadWrapper; import org.prebid.cache.repository.redis.module.storage.ModuleCompositeRepository; @@ -78,7 +78,7 @@ public void teardown() { void testVerifySave() { given(validator.validate(any())).willReturn(Collections.emptySet()); - final var payload = ModulePayload.builder() + final var payload = StoragePayload.builder() .key("key") .type(PayloadType.TEXT) .application("application") @@ -113,7 +113,7 @@ void testVerifySave() { void testVerifyApiKeyAuthorization() { given(validator.validate(any())).willReturn(Collections.emptySet()); - final var payload = ModulePayload.builder() + final var payload = StoragePayload.builder() .key("key") .type(PayloadType.TEXT) .application("application") From d7e1fe94ed28d04f7f5d3bd661c430c994ba001a Mon Sep 17 00:00:00 2001 From: Alex Maltsev Date: Fri, 26 Jul 2024 16:56:48 +0300 Subject: [PATCH 5/9] Added doc for new property. --- docs/config.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index 5d99b81..a8e7e2e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -9,8 +9,9 @@ The next sections describes how to set up project configuration. ## Application properties -### Server +### Cache - `cache.allowed_proxy_host` - set the allowed proxy host for request with `ch` parameter. -[//]: # (TODO: Add all server config proprties) +### Storage +- `storage.default-ttl-seconds` - set the default ttl for the data From 920b5acfec47e92d5a3c04f672873bec37648968 Mon Sep 17 00:00:00 2001 From: Markiyan Mykush <95693607+marki1an@users.noreply.github.com> Date: Tue, 6 Aug 2024 14:19:29 +0300 Subject: [PATCH 6/9] Test: `Storage-module` (#139) --- .../{ModuleStorageSpec.kt => StorageSpec.kt} | 95 ++++++++++++++----- .../functional/service/PrebidCacheApi.kt | 10 +- .../PrebidCacheContainerConfig.kt | 11 ++- 3 files changed, 81 insertions(+), 35 deletions(-) rename src/test/kotlin/org/prebid/cache/functional/{ModuleStorageSpec.kt => StorageSpec.kt} (73%) diff --git a/src/test/kotlin/org/prebid/cache/functional/ModuleStorageSpec.kt b/src/test/kotlin/org/prebid/cache/functional/StorageSpec.kt similarity index 73% rename from src/test/kotlin/org/prebid/cache/functional/ModuleStorageSpec.kt rename to src/test/kotlin/org/prebid/cache/functional/StorageSpec.kt index e9009a2..09600ca 100644 --- a/src/test/kotlin/org/prebid/cache/functional/ModuleStorageSpec.kt +++ b/src/test/kotlin/org/prebid/cache/functional/StorageSpec.kt @@ -16,8 +16,9 @@ import org.prebid.cache.functional.util.getRandomString import org.springframework.http.HttpStatus.BAD_REQUEST import org.springframework.http.HttpStatus.NOT_FOUND import org.springframework.http.HttpStatus.UNAUTHORIZED +import org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR -class ModuleStorageSpec : ShouldSpec({ +class StorageSpec : ShouldSpec({ lateinit var apiKey: String lateinit var applicationName: String @@ -39,10 +40,10 @@ class ModuleStorageSpec : ShouldSpec({ } // when: POST module-storage endpoint is called - cacheApi.postModuleStorageCache(payloadTransfer, apiKey) + cacheApi.postStorageCache(payloadTransfer, apiKey) // then: recorded payload should contain the same type and value - val savedPayload = cacheApi.getModuleStorageCache(payloadKey, applicationName, apiKey) + val savedPayload = cacheApi.getStorageCache(payloadKey, applicationName, apiKey) savedPayload.type shouldBe payloadTransfer.type savedPayload.value shouldBe payloadTransfer.value @@ -60,10 +61,10 @@ class ModuleStorageSpec : ShouldSpec({ } // when: POST module-storage endpoint is called - cacheApi.postModuleStorageCache(payloadTransfer, apiKey) + cacheApi.postStorageCache(payloadTransfer, apiKey) // then: recorded payload should contain the same type and value - val savedPayload = cacheApi.getModuleStorageCache(payloadKey, applicationName, apiKey) + val savedPayload = cacheApi.getStorageCache(payloadKey, applicationName, apiKey) savedPayload.type shouldBe payloadTransfer.type savedPayload.value shouldBe payloadTransfer.value @@ -81,10 +82,10 @@ class ModuleStorageSpec : ShouldSpec({ } // when: POST module-storage endpoint is called - cacheApi.postModuleStorageCache(payloadTransfer, apiKey) + cacheApi.postStorageCache(payloadTransfer, apiKey) // then: recorded payload should contain the same type and value - val savedPayload = cacheApi.getModuleStorageCache(payloadKey, applicationName, apiKey) + val savedPayload = cacheApi.getStorageCache(payloadKey, applicationName, apiKey) savedPayload.type shouldBe payloadTransfer.type savedPayload.value shouldBe payloadTransfer.value @@ -103,12 +104,12 @@ class ModuleStorageSpec : ShouldSpec({ // when: POST module-storage endpoint is called val exception = shouldThrowExactly { - cacheApi.postModuleStorageCache(payloadTransfer, apiKey) } + cacheApi.postStorageCache(payloadTransfer, apiKey) } // then: Not found exception is thrown assertSoftly { exception.statusCode shouldBe NOT_FOUND.value() - exception.responseBody shouldContain "\"path\":\"/module-storage\"" + exception.responseBody shouldContain "\"path\":\"/storage\"" exception.responseBody shouldContain "\"message\":\"Invalid application: ${randomApplication}\"" } } @@ -122,12 +123,12 @@ class ModuleStorageSpec : ShouldSpec({ } // when: POST module-storage endpoint is called - val exception = shouldThrowExactly { cacheApi.postModuleStorageCache(payloadTransfer, apiKey) } + val exception = shouldThrowExactly { cacheApi.postStorageCache(payloadTransfer, apiKey) } // then: Bad request exception is thrown assertSoftly { exception.statusCode shouldBe BAD_REQUEST.value() - exception.responseBody shouldContain "\"path\":\"/module-storage\"" + exception.responseBody shouldContain "\"path\":\"/storage\"" exception.responseBody shouldContain "application must not be empty" } } @@ -141,12 +142,12 @@ class ModuleStorageSpec : ShouldSpec({ } // when: POST module-storage endpoint is called - val exception = shouldThrowExactly { cacheApi.postModuleStorageCache(payloadTransfer, apiKey) } + val exception = shouldThrowExactly { cacheApi.postStorageCache(payloadTransfer, apiKey) } // then: Bad request exception is thrown assertSoftly { exception.statusCode shouldBe BAD_REQUEST.value() - exception.responseBody shouldContain "\"path\":\"/module-storage\"" + exception.responseBody shouldContain "\"path\":\"/storage\"" exception.responseBody shouldContain "application must not be empty" } } @@ -159,12 +160,12 @@ class ModuleStorageSpec : ShouldSpec({ } // when: POST module-storage endpoint is called - val exception = shouldThrowExactly { cacheApi.postModuleStorageCache(payloadTransfer, apiKey) } + val exception = shouldThrowExactly { cacheApi.postStorageCache(payloadTransfer, apiKey) } // then: Bad request exception is thrown assertSoftly { exception.statusCode shouldBe BAD_REQUEST.value() - exception.responseBody shouldContain "\"path\":\"/module-storage\"" + exception.responseBody shouldContain "\"path\":\"/storage\"" exception.responseBody shouldContain "key must not be empty" } } @@ -177,12 +178,12 @@ class ModuleStorageSpec : ShouldSpec({ } // when: POST module-storage endpoint is called - val exception = shouldThrowExactly { cacheApi.postModuleStorageCache(payloadTransfer, apiKey) } + val exception = shouldThrowExactly { cacheApi.postStorageCache(payloadTransfer, apiKey) } // then: Bad request exception is thrown assertSoftly { exception.statusCode shouldBe BAD_REQUEST.value() - exception.responseBody shouldContain "\"path\":\"/module-storage\"" + exception.responseBody shouldContain "\"path\":\"/storage\"" exception.responseBody shouldContain "key must not be empty" } } @@ -197,7 +198,7 @@ class ModuleStorageSpec : ShouldSpec({ // when: POST module-storage endpoint is called val exception = - shouldThrowExactly { cacheApi.postModuleStorageCache(payloadTransfer, getRandomString()) } + shouldThrowExactly { cacheApi.postStorageCache(payloadTransfer, getRandomString()) } // then: Not found exception is thrown assertSoftly { @@ -214,17 +215,17 @@ class ModuleStorageSpec : ShouldSpec({ } // and: POST module-storage endpoint is called - cacheApi.postModuleStorageCache(payloadTransfer, apiKey) + cacheApi.postStorageCache(payloadTransfer, apiKey) // when: GET module-storage endpoint is called with invalid data val exception = shouldThrowExactly { - cacheApi.getModuleStorageCache(getRandomString(), applicationName, apiKey) + cacheApi.getStorageCache(getRandomString(), applicationName, apiKey) } // then: Not found exception is thrown assertSoftly { exception.statusCode shouldBe NOT_FOUND.value() - exception.responseBody shouldContain "\"path\":\"/module-storage\"" + exception.responseBody shouldContain "\"path\":\"/storage\"" exception.responseBody shouldContain "Invalid application or key" } } @@ -238,20 +239,20 @@ class ModuleStorageSpec : ShouldSpec({ } // and: POST module-storage endpoint is called - cacheApi.postModuleStorageCache(payloadTransfer, apiKey) + cacheApi.postStorageCache(payloadTransfer, apiKey) //and: random application name val randomApplication = getRandomString() // when: GET module-storage endpoint is called with invalid data val exception = shouldThrowExactly { - cacheApi.getModuleStorageCache(payloadKey, randomApplication, apiKey) + cacheApi.getStorageCache(payloadKey, randomApplication, apiKey) } // then: Not found exception is thrown assertSoftly { exception.statusCode shouldBe NOT_FOUND.value() - exception.responseBody shouldContain "\"path\":\"/module-storage\"" + exception.responseBody shouldContain "\"path\":\"/storage\"" exception.responseBody shouldContain "\"message\":\"Invalid application: ${randomApplication}\"" } } @@ -265,11 +266,11 @@ class ModuleStorageSpec : ShouldSpec({ } // and: POST module-storage endpoint is called - cacheApi.postModuleStorageCache(payloadTransfer, apiKey) + cacheApi.postStorageCache(payloadTransfer, apiKey) // when: GET module-storage endpoint is called with invalid data val exception = shouldThrowExactly { - cacheApi.getModuleStorageCache(payloadKey, applicationName, getRandomString()) + cacheApi.getStorageCache(payloadKey, applicationName, getRandomString()) } // then: Not found exception is thrown @@ -278,4 +279,46 @@ class ModuleStorageSpec : ShouldSpec({ exception.responseBody should beEmpty() } } + + should("throw an exception when ttlsecond is zero") { + //given: default json payload with application + val payloadKey = getRandomString() + val payloadTransfer = PayloadTransfer.getDefaultJsonPayloadTransfer().apply { + key = payloadKey + application = applicationName + ttlseconds = 0 + } + + // when: POST module-storage endpoint is called + val exception = shouldThrowExactly { + cacheApi.postStorageCache(payloadTransfer, apiKey) } + + // then: Expire time exception is thrown + assertSoftly { + exception.statusCode shouldBe INTERNAL_SERVER_ERROR.value() + exception.responseBody shouldContain "\"path\":\"/storage\"" + exception.responseBody shouldContain "\"message\":\"ERR invalid expire time in setex" + } + } + + should("not throw an exception when ttlsecond is null and config ttlseconds are present") { + //given: default json payload with application + val payloadKey = getRandomString() + val payloadTransfer = PayloadTransfer.getDefaultJsonPayloadTransfer().apply { + key = payloadKey + application = applicationName + ttlseconds = null + } + + // when: POST module-storage endpoint is called + cacheApi.postStorageCache(payloadTransfer, apiKey) + + // then: recorded payload should contain the same type and value + val savedPayload = cacheApi.getStorageCache(payloadKey, applicationName, apiKey) + savedPayload.type shouldBe payloadTransfer.type + savedPayload.value shouldBe payloadTransfer.value + + // and: shouldn't contain information about application + savedPayload.application?.should(beNull()) + } }) diff --git a/src/test/kotlin/org/prebid/cache/functional/service/PrebidCacheApi.kt b/src/test/kotlin/org/prebid/cache/functional/service/PrebidCacheApi.kt index 67c21f0..59c6390 100644 --- a/src/test/kotlin/org/prebid/cache/functional/service/PrebidCacheApi.kt +++ b/src/test/kotlin/org/prebid/cache/functional/service/PrebidCacheApi.kt @@ -30,20 +30,20 @@ class PrebidCacheApi(prebidCacheHost: String, prebidCachePort: Int) { suspend fun postCache(requestObject: RequestObject, secondaryCache: String? = null): ResponseObject = post(CACHE_ENDPOINT, requestObject, mapOf(SECONDARY_CACHE_QUERY_PARAMETER to secondaryCache)).body() - suspend fun getModuleStorageCache( + suspend fun getStorageCache( payloadTransferKey: String?, application: String?, apiKey: String? ): PayloadTransfer = get( - MODULE_STORAGE_ENDPOINT, + STORAGE_ENDPOINT, mapOf(KEY_PARAMETER to payloadTransferKey, APPLICATION_PARAMETER to application), mapOf(API_KEY_PARAMETER to apiKey) ).body() - suspend fun postModuleStorageCache(requestObject: PayloadTransfer, apiKey: String? = null): Boolean = + suspend fun postStorageCache(requestObject: PayloadTransfer, apiKey: String? = null): Boolean = post( - MODULE_STORAGE_ENDPOINT, + STORAGE_ENDPOINT, requestObject, headers = mapOf(API_KEY_PARAMETER to apiKey) ).status == HttpStatusCode.NoContent @@ -102,7 +102,7 @@ class PrebidCacheApi(prebidCacheHost: String, prebidCachePort: Int) { private const val PROXY_CACHE_HOST_QUERY_PARAMETER = "ch" private const val SECONDARY_CACHE_QUERY_PARAMETER = "secondaryCache" - private const val MODULE_STORAGE_ENDPOINT = "/module-storage" + private const val STORAGE_ENDPOINT = "/storage" private const val API_KEY_PARAMETER = "x-pbc-api-key" private const val KEY_PARAMETER = "k" private const val APPLICATION_PARAMETER = "a" diff --git a/src/test/kotlin/org/prebid/cache/functional/testcontainers/PrebidCacheContainerConfig.kt b/src/test/kotlin/org/prebid/cache/functional/testcontainers/PrebidCacheContainerConfig.kt index 9fb389f..c54346b 100644 --- a/src/test/kotlin/org/prebid/cache/functional/testcontainers/PrebidCacheContainerConfig.kt +++ b/src/test/kotlin/org/prebid/cache/functional/testcontainers/PrebidCacheContainerConfig.kt @@ -63,13 +63,16 @@ class PrebidCacheContainerConfig(private val redisHost: String, private val aero private fun getModuleStorageRedisConfig( apiKey: String, applicationName: String, - timeoutMs: Long = 9999L + timeoutMs: Long = 9999L, + endpoint: String = "/storage" ): Map = mapOf( "api.api-key" to apiKey, - "module.storage.redis.${applicationName}.port" to RedisContainer.PORT.toString(), - "module.storage.redis.${applicationName}.host" to redisHost, - "module.storage.redis.${applicationName}.timeout" to timeoutMs.toString(), + "api.storage-path" to endpoint, + "storage.redis.${applicationName}.port" to RedisContainer.PORT.toString(), + "storage.redis.${applicationName}.host" to redisHost, + "storage.redis.${applicationName}.timeout" to timeoutMs.toString(), + "storage.default-ttl-seconds" to 1000L.toString() ) private fun getBaseConfig(allowExternalUuid: String): Map = From af9e226401ba6c2a32f12327b04b17268679aeb3 Mon Sep 17 00:00:00 2001 From: Alex Maltsev Date: Tue, 6 Aug 2024 16:56:35 +0300 Subject: [PATCH 7/9] Added unit test. --- .../handlers/PostStorageHandlerTests.java | 43 +++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java b/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java index 08b2753..fe83cc3 100644 --- a/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java +++ b/src/test/java/org/prebid/cache/handlers/PostStorageHandlerTests.java @@ -34,8 +34,7 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = { PrebidServerResponseBuilder.class, - ApiConfig.class, - StorageConfig.class, + ApiConfig.class }) @EnableConfigurationProperties @SpringBootTest @@ -47,6 +46,9 @@ class PostStorageHandlerTests { @Autowired PrebidServerResponseBuilder responseBuilder; + @MockBean + StorageConfig storageConfig; + @MockBean ModuleCompositeRepository moduleCompositeRepository; @@ -64,7 +66,7 @@ public void setup() { moduleCompositeRepository, responseBuilder, apiConfig, - new StorageConfig(9999L)); + storageConfig); serverMock = new WireMockServer(8080); serverMock.start(); } @@ -109,6 +111,41 @@ void testVerifySave() { .verify(); } + @Test + void testVerifyDefaultTtl() { + given(validator.validate(any())).willReturn(Collections.emptySet()); + given(storageConfig.getDefaultTtlSeconds()).willReturn(999L); + + final var payload = StoragePayload.builder() + .key("key") + .type(PayloadType.TEXT) + .application("application") + .value("value") + .build(); + + final var payloadWrapper = PayloadWrapper.builder() + .id("key") + .prefix("") + .payload(Payload.of("text", "key", "value")) + .expiry(999L) + .build(); + + given(moduleCompositeRepository.save("application", payloadWrapper)) + .willReturn(Mono.just(payloadWrapper)); + + final var serverRequest = MockServerRequest.builder() + .method(HttpMethod.POST) + .header("x-pbc-api-key", apiConfig.getApiKey()) + .body(Mono.just(payload)); + + final var responseMono = handler.save(serverRequest); + + StepVerifier.create(responseMono) + .consumeNextWith(serverResponse -> assertEquals(204, serverResponse.statusCode().value())) + .expectComplete() + .verify(); + } + @Test void testVerifyApiKeyAuthorization() { given(validator.validate(any())).willReturn(Collections.emptySet()); From ad5bc01556b87e73615be5c28c2a46e258b6264a Mon Sep 17 00:00:00 2001 From: Alex Maltsev Date: Tue, 6 Aug 2024 17:03:32 +0300 Subject: [PATCH 8/9] Updated storage config. --- src/main/java/org/prebid/cache/config/StorageConfig.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/prebid/cache/config/StorageConfig.java b/src/main/java/org/prebid/cache/config/StorageConfig.java index e002ae5..601502d 100644 --- a/src/main/java/org/prebid/cache/config/StorageConfig.java +++ b/src/main/java/org/prebid/cache/config/StorageConfig.java @@ -1,5 +1,7 @@ package org.prebid.cache.config; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -15,5 +17,7 @@ @ConfigurationProperties(prefix = "storage") public class StorageConfig { + @NotNull + @Min(0) Long defaultTtlSeconds; } From 0ca41e710579b0f306d33c2c552b0b8c1b341376 Mon Sep 17 00:00:00 2001 From: Alex Maltsev Date: Wed, 7 Aug 2024 15:43:59 +0300 Subject: [PATCH 9/9] Added information about redis storage configuration. --- docs/config.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/config.md b/docs/config.md index 201285c..174632f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -42,3 +42,11 @@ The next sections describes how to set up project configuration. ### Storage - `storage.default-ttl-seconds` - set the default ttl for the data + +#### Redis +- `storage.redis.{application-name}.port` - redis port. +- `storage.redis.{application-name}.host` - redis host. +- `storage.redis.{application-name}.password` - redis password, leave empty if no password required. +- `storage.redis.{application-name}.cluster.nodes` - list of node uris, set when using clustered redis. +- `storage.redis.{application-name}.cluster.enable_topology_refresh` - toggle for topology refresh support, set when using clustered redis. +- `storage.redis.{application-name}.cluster.topology_periodic_refresh_period` - refresh period of clustered redis topology, used when `storage.redis.{application-name}.cluster.enable_topology_refresh` is set to true.