From 2a00d480a7cfb57e2eeeb96403aec811153b1499 Mon Sep 17 00:00:00 2001 From: Martin Lowe <martin.lowe@eclipse-foundation.org> Date: Tue, 19 Nov 2019 15:56:47 -0500 Subject: [PATCH] Implement solution for deduplication of random results #7 Added Cache-Control and Etag header values for responses. Added no-store to random sort order requests, as well as a cache bypass mechanism. This mechanism currently only triggers when sort order is set to random. Added calls to cache service to maintain expiration times as well as accessor for max age. Change-Id: If7b4b57c7e265fe69ef4fdefec4249ea55bcdf5d Signed-off-by: Martin Lowe <martin.lowe@eclipse-foundation.org> --- src/main/docker/Dockerfile.jvm | 15 +-- .../marketplace/helper/ResponseHelper.java | 94 +++++++++++++++++++ .../marketplace/model/MongoQuery.java | 17 +--- .../marketplace/model/RequestWrapper.java | 14 ++- .../marketplace/model/SortOrder.java | 20 ++++ .../request/CacheBypassFilter.java | 60 ++++++++++++ .../marketplace/resource/CatalogResource.java | 15 +-- .../resource/CategoryResource.java | 15 +-- .../resource/ErrorReportResource.java | 13 ++- .../marketplace/resource/InstallResource.java | 21 +++-- .../marketplace/resource/ListingResource.java | 15 +-- .../resource/ListingVersionResource.java | 11 ++- .../marketplace/resource/MarketResource.java | 16 ++-- .../marketplace/service/CachingService.java | 25 ++++- .../service/impl/GuavaCachingService.java | 55 ++++++++--- .../service/impl/GuavaCachingServiceTest.java | 19 ++-- src/test/resources/application.properties | 4 +- 17 files changed, 330 insertions(+), 99 deletions(-) create mode 100644 src/main/java/org/eclipsefoundation/marketplace/helper/ResponseHelper.java create mode 100644 src/main/java/org/eclipsefoundation/marketplace/request/CacheBypassFilter.java diff --git a/src/main/docker/Dockerfile.jvm b/src/main/docker/Dockerfile.jvm index 4a1b9f6..985ce5f 100644 --- a/src/main/docker/Dockerfile.jvm +++ b/src/main/docker/Dockerfile.jvm @@ -3,27 +3,18 @@ # # Before building the docker image run: # -# mvn package +# mvn package -Dconfig.secret.path=<full path to secret file> # # Then, build the image with: # -# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/sample-jvm . +# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/mpc-rest-api-jvm . # # Then run the container using: # -# docker run -i --rm -p 8080:8080 quarkus/sample-jvm +# docker run -i --rm -p 8090:8090 quarkus/mpc-rest-api-jvm # ### FROM fabric8/java-alpine-openjdk8-jre - -## Where to source the cert file -ARG LOCAL_CRT=config/local.crt -ENV LOCAL_CRT ${LOCAL_CRT} - -## copy to a temp ssl dir for container usage -WORKDIR /tmp -RUN mkdir ssl -COPY $LOCAL_CRT ssl/local.crt ## Where to copy the secret file, default to tmp ARG SECRET_LOCATION=/tmp diff --git a/src/main/java/org/eclipsefoundation/marketplace/helper/ResponseHelper.java b/src/main/java/org/eclipsefoundation/marketplace/helper/ResponseHelper.java new file mode 100644 index 0000000..d991e7c --- /dev/null +++ b/src/main/java/org/eclipsefoundation/marketplace/helper/ResponseHelper.java @@ -0,0 +1,94 @@ +/* Copyright (c) 2019 Eclipse Foundation and others. + * This program and the accompanying materials are made available + * under the terms of the Eclipse Public License 2.0 + * which is available at http://www.eclipse.org/legal/epl-v20.html, + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipsefoundation.marketplace.helper; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Date; +import java.util.Objects; +import java.util.Optional; + +import javax.enterprise.context.ApplicationScoped; +import javax.inject.Inject; +import javax.json.bind.Jsonb; +import javax.ws.rs.core.CacheControl; +import javax.ws.rs.core.Response; +import javax.xml.bind.DatatypeConverter; + +import org.eclipsefoundation.marketplace.model.RequestWrapper; +import org.eclipsefoundation.marketplace.service.CachingService; + +/** + * Helper class that transforms data into a response usable for the RESTeasy + * container. Uses injected JSON-B serializer and caching service to get current + * information on cache data. + * + * @author Martin Lowe + * + */ +@ApplicationScoped +public class ResponseHelper { + + private static final MessageDigest DIGEST; + static { + try { + DIGEST = MessageDigest.getInstance("md5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("Could not create an MD5 hash digest"); + } + } + + @Inject + Jsonb jsonb; + @Inject + CachingService<?> cachingService; + + /** + * Builds a response using passed data. Uses references to the caching service + * and the current request to add information about ETags and Cache-Control + * headers. + * + * @param id the ID of the object to be stored in cache + * @param wrapper the query parameters for the current request + * @param data the data to attach to the response + * @return a complete response object for the given data and request. + */ + public Response build(String id, RequestWrapper wrapper, Object data) { + // set default cache control flags for API responses + CacheControl cc = new CacheControl(); + cc.setNoStore(wrapper.isCacheBypass()); + + if (!cc.isNoStore()) { + cc.setMaxAge((int) cachingService.getMaxAge()); + // get the TTL for the current entry + Optional<Long> ttl = cachingService.getExpiration(id, wrapper); + if (!ttl.isPresent()) { + return Response.serverError().build(); + } + + // serialize the data to get an etag + String content = jsonb.toJson(Objects.requireNonNull(data)); + // ingest the content and hash to create an etag for current content + String hash; + synchronized (this) { + DIGEST.update(content.getBytes(StandardCharsets.UTF_8)); + hash = DatatypeConverter.printHexBinary(DIGEST.digest()); + DIGEST.reset(); + } + + // check if etag matches + String etag = wrapper.getHeader("Etag"); + if (hash.equals(etag)) { + return Response.notModified(etag).cacheControl(cc).expires(new Date(ttl.get())).build(); + } + // return a response w/ the generated etag + return Response.ok(data).tag(hash).cacheControl(cc).expires(new Date(ttl.get())).build(); + } + return Response.ok(data).cacheControl(cc).build(); + } +} diff --git a/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java b/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java index 1b48732..a392fb6 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java +++ b/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java @@ -17,7 +17,6 @@ import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; import org.eclipsefoundation.marketplace.helper.SortableHelper; import org.eclipsefoundation.marketplace.helper.SortableHelper.Sortable; import org.eclipsefoundation.marketplace.namespace.UrlParameterNames; -import org.eclipsefoundation.marketplace.service.CachingService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +34,6 @@ import com.mongodb.client.model.Filters; public class MongoQuery<T> { private static final Logger LOGGER = LoggerFactory.getLogger(MongoQuery.class); - private CachingService<List<T>> cache; private RequestWrapper wrapper; private DtoFilter<T> dtoFilter; @@ -44,10 +42,9 @@ public class MongoQuery<T> { private SortOrder order; private List<Bson> aggregates; - public MongoQuery(RequestWrapper wrapper, DtoFilter<T> dtoFilter, CachingService<List<T>> cache) { + public MongoQuery(RequestWrapper wrapper, DtoFilter<T> dtoFilter) { this.wrapper = wrapper; this.dtoFilter = dtoFilter; - this.cache = cache; this.aggregates = new ArrayList<>(); init(); } @@ -72,12 +69,13 @@ public class MongoQuery<T> { Optional<String> sortOpt = wrapper.getFirstParam(UrlParameterNames.SORT); if (sortOpt.isPresent()) { String sortVal = sortOpt.get(); + SortOrder ord = SortOrder.getOrderFromValue(sortOpt.get()); // split sort string of `<fieldName> <SortOrder>` int idx = sortVal.indexOf(' '); // check if the sort string matches the RANDOM sort order - if (SortOrder.RANDOM.equals(SortOrder.getOrderByName(sortVal))) { + if (SortOrder.RANDOM.equals(ord)) { this.order = SortOrder.RANDOM; - } else if (idx > 0) { + } else if (ord != SortOrder.NONE) { setSort(sortVal.substring(0, idx), sortVal.substring(idx + 1), filters); } } @@ -143,13 +141,6 @@ public class MongoQuery<T> { this.order = SortOrder.getOrderByName(sortOrder); // add sorting query if the sortOrder matches a defined order switch (order) { - case RANDOM: - // TODO support for random, implement the following (in this order) - // 1. Add not in clause that checks Cache for previously read objects - // 2. Set useAggregate flag to true to signal to DAO to use aggregate selection - // rather than traditional find - - break; case ASCENDING: // if last seen is set, add a filter to shift the results if (lastOpt.isPresent()) { diff --git a/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java b/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java index aec134a..43fc346 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java +++ b/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java @@ -23,6 +23,7 @@ import javax.ws.rs.core.UriInfo; import org.apache.commons.lang3.StringUtils; import org.eclipsefoundation.marketplace.namespace.DeprecatedHeader; import org.eclipsefoundation.marketplace.namespace.RequestHeaderNames; +import org.eclipsefoundation.marketplace.request.CacheBypassFilter; import org.jboss.resteasy.core.ResteasyContext; /** @@ -168,7 +169,18 @@ public class RequestWrapper { } /** - * Retrieve a request header value as an optional value. + * Check whether the current request should bypass caching + * + * @return true if cache should be bypassed, otherwise false + */ + public boolean isCacheBypass() { + Object attr = request.getAttribute(CacheBypassFilter.ATTRIBUTE_NAME); + // if we have the attribute set on the request, return it. otherwise, false. + return attr instanceof Boolean ? (boolean) attr : Boolean.FALSE; + } + + /** + * Retrieve a request header value. * * @param key the headers key value * @return the value, or an empty optional if missing. diff --git a/src/main/java/org/eclipsefoundation/marketplace/model/SortOrder.java b/src/main/java/org/eclipsefoundation/marketplace/model/SortOrder.java index 86c250e..f850948 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/model/SortOrder.java +++ b/src/main/java/org/eclipsefoundation/marketplace/model/SortOrder.java @@ -53,4 +53,24 @@ public enum SortOrder { } return NONE; } + + /** + * Gets the SortOrder value associated with a sort parameter value if one + * exists. + * + * @param value the value of the sort parameter + * @return the SortOrder associated with the request, or + * {@linkplain SortOrder.NONE} + */ + public static SortOrder getOrderFromValue(String value) { + // get the index of the space separator + int idx = value.indexOf(' '); + // check if the sort string matches the RANDOM sort order + if (SortOrder.RANDOM.equals(SortOrder.getOrderByName(value))) { + return SortOrder.RANDOM; + } else if (idx > 0) { + return SortOrder.getOrderByName(value.substring(idx + 1)); + } + return SortOrder.NONE; + } } \ No newline at end of file diff --git a/src/main/java/org/eclipsefoundation/marketplace/request/CacheBypassFilter.java b/src/main/java/org/eclipsefoundation/marketplace/request/CacheBypassFilter.java new file mode 100644 index 0000000..636c40b --- /dev/null +++ b/src/main/java/org/eclipsefoundation/marketplace/request/CacheBypassFilter.java @@ -0,0 +1,60 @@ +/* Copyright (c) 2019 Eclipse Foundation and others. + * This program and the accompanying materials are made available + * under the terms of the Eclipse Public License 2.0 + * which is available at http://www.eclipse.org/legal/epl-v20.html, + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipsefoundation.marketplace.request; + +import java.io.IOException; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerRequestFilter; +import javax.ws.rs.core.Context; +import javax.ws.rs.ext.Provider; + +import org.eclipsefoundation.marketplace.model.SortOrder; +import org.eclipsefoundation.marketplace.namespace.UrlParameterNames; + +/** + * Checks passed parameters and if any match one of the criteria for bypassing + * caching, an attribute will be set to the request to skip cache requests and + * instead directly return results. + * + * @author Martin Lowe + * + */ +@Provider +public class CacheBypassFilter implements ContainerRequestFilter { + public static final String ATTRIBUTE_NAME = "bypass-cache"; + + @Context + HttpServletRequest request; + + @Context + HttpServletResponse response; + + @Override + public void filter(ContainerRequestContext requestContext) throws IOException { + // check for random sort order, which always bypasses cache + String[] sortVals = request.getParameterValues(UrlParameterNames.SORT); + if (sortVals != null) { + for (String sortVal : sortVals) { + // check if the sort order for request matches RANDOM + if (SortOrder.RANDOM.equals(SortOrder.getOrderFromValue(sortVal))) { + setBypass(); + return; + } + } + } + request.setAttribute(ATTRIBUTE_NAME, Boolean.FALSE); + } + + private void setBypass() { + request.setAttribute(ATTRIBUTE_NAME, Boolean.TRUE); + // no-store should be used as cache bypass should not return + response.setHeader("Cache-Control", "no-store"); + } +} diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/CatalogResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/CatalogResource.java index 84cb41b..0aa35a0 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/CatalogResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/CatalogResource.java @@ -27,6 +27,7 @@ import javax.ws.rs.core.Response.Status; import org.eclipsefoundation.marketplace.dao.MongoDao; import org.eclipsefoundation.marketplace.dto.Catalog; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.Error; import org.eclipsefoundation.marketplace.model.MongoQuery; @@ -58,11 +59,13 @@ public class CatalogResource { RequestWrapper params; @Inject DtoFilter<Catalog> dtoFilter; + @Inject + ResponseHelper responseBuider; @GET @PermitAll public Response select() { - MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter); // retrieve the possible cached object Optional<List<Catalog>> cachedResults = cachingService.get("all", params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -72,7 +75,7 @@ public class CatalogResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build("all", params, cachedResults.get()); } /** @@ -84,7 +87,7 @@ public class CatalogResource { @PUT @RolesAllowed({ "marketplace_catalog_put", "marketplace_admin_access" }) public Response putCatalog(Catalog catalog) { - MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(catalog))); @@ -104,7 +107,7 @@ public class CatalogResource { public Response select(@PathParam("catalogId") String catalogId) { params.addParam(UrlParameterNames.ID, catalogId); - MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter); // retrieve a cached version of the value for the current listing Optional<List<Catalog>> cachedResults = cachingService.get(catalogId, params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -114,7 +117,7 @@ public class CatalogResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(catalogId, params, cachedResults.get()); } /** @@ -130,7 +133,7 @@ public class CatalogResource { public Response delete(@PathParam("catalogId") String catalogId) { params.addParam(UrlParameterNames.ID, catalogId); - MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter); // delete the currently selected asset DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q)); if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) { diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/CategoryResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/CategoryResource.java index ed73074..b343c31 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/CategoryResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/CategoryResource.java @@ -27,6 +27,7 @@ import javax.ws.rs.core.Response.Status; import org.eclipsefoundation.marketplace.dao.MongoDao; import org.eclipsefoundation.marketplace.dto.Category; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.Error; import org.eclipsefoundation.marketplace.model.MongoQuery; @@ -58,11 +59,13 @@ public class CategoryResource { RequestWrapper params; @Inject DtoFilter<Category> dtoFilter; + @Inject + ResponseHelper responseBuider; @GET @PermitAll public Response select() { - MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter); // retrieve the possible cached object Optional<List<Category>> cachedResults = cachingService.get("all", params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -72,7 +75,7 @@ public class CategoryResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build("all", params, cachedResults.get()); } /** @@ -84,7 +87,7 @@ public class CategoryResource { @PUT @RolesAllowed({"marketplace_category_put", "marketplace_admin_access"}) public Response putCategory(Category category) { - MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(category))); @@ -104,7 +107,7 @@ public class CategoryResource { public Response select(@PathParam("categoryId") String categoryId) { params.addParam(UrlParameterNames.ID, categoryId); - MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter); // retrieve a cached version of the value for the current listing Optional<List<Category>> cachedResults = cachingService.get(categoryId, params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -114,7 +117,7 @@ public class CategoryResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(categoryId, params, cachedResults.get()); } /** @@ -130,7 +133,7 @@ public class CategoryResource { public Response delete(@PathParam("categoryId") String categoryId) { params.addParam(UrlParameterNames.ID, categoryId); - MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter); // delete the currently selected asset DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q)); if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) { diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/ErrorReportResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/ErrorReportResource.java index 80d5e8d..5c8b1ad 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/ErrorReportResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/ErrorReportResource.java @@ -25,6 +25,7 @@ import javax.ws.rs.core.Response; import org.eclipsefoundation.marketplace.dao.MongoDao; import org.eclipsefoundation.marketplace.dto.ErrorReport; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.MongoQuery; import org.eclipsefoundation.marketplace.model.RequestWrapper; @@ -54,6 +55,8 @@ public class ErrorReportResource { RequestWrapper params; @Inject DtoFilter<ErrorReport> dtoFilter; + @Inject + ResponseHelper responseBuider; /** * Endpoint for /error/ to retrieve all ErrorReports from the database along with @@ -65,7 +68,7 @@ public class ErrorReportResource { @GET @PermitAll public Response select() { - MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter); // retrieve the possible cached object Optional<List<ErrorReport>> cachedResults = cachingService.get("all", params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -75,7 +78,7 @@ public class ErrorReportResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build("all", params, cachedResults.get()); } /** @@ -87,7 +90,7 @@ public class ErrorReportResource { @PUT @RolesAllowed("error_put") public Response putErrorReport(ErrorReport errorReport) { - MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(errorReport))); @@ -109,7 +112,7 @@ public class ErrorReportResource { public Response select(@PathParam("errorReportId") String errorReportId) { params.addParam(UrlParameterNames.ID, errorReportId); - MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter); // retrieve a cached version of the value for the current ErrorReport Optional<List<ErrorReport>> cachedResults = cachingService.get(errorReportId, params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -119,6 +122,6 @@ public class ErrorReportResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(errorReportId, params, cachedResults.get()); } } diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java index 8c1562d..9519ff4 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java @@ -37,6 +37,7 @@ import org.eclipsefoundation.marketplace.dto.InstallMetrics; import org.eclipsefoundation.marketplace.dto.MetricPeriod; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; import org.eclipsefoundation.marketplace.helper.DateTimeHelper; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.Error; import org.eclipsefoundation.marketplace.model.MongoQuery; @@ -63,6 +64,8 @@ public class InstallResource { MongoDao dao; @Inject RequestWrapper wrapper; + @Inject + ResponseHelper responseBuider; // insert required filters for different objects + states @Inject @@ -90,7 +93,7 @@ public class InstallResource { @Path("/{listingId}") public Response selectInstallCount(@PathParam("listingId") String listingId) { wrapper.addParam(UrlParameterNames.ID, listingId); - MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, null); + MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter); Optional<Long> cachedResults = countCache.get(listingId, wrapper, () -> StreamHelper.awaitCompletionStage(dao.count(q))); if (!cachedResults.isPresent()) { @@ -99,7 +102,7 @@ public class InstallResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(listingId, wrapper, cachedResults.get()); } /** @@ -117,7 +120,7 @@ public class InstallResource { public Response selectInstallCount(@PathParam("listingId") String listingId, @PathParam("version") String version) { wrapper.addParam(UrlParameterNames.ID, listingId); wrapper.addParam(UrlParameterNames.VERSION, version); - MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, null); + MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter); Optional<Long> cachedResults = countCache.get(getCompositeKey(listingId, version), wrapper, () -> StreamHelper.awaitCompletionStage(dao.count(q))); if (!cachedResults.isPresent()) { @@ -126,7 +129,7 @@ public class InstallResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(getCompositeKey(listingId, version), wrapper, cachedResults.get()); } /** @@ -141,7 +144,7 @@ public class InstallResource { @Path("/{listingId}/metrics") public Response selectInstallMetrics(@PathParam("listingId") String listingId) { wrapper.addParam(UrlParameterNames.ID, listingId); - MongoQuery<InstallMetrics> q = new MongoQuery<>(wrapper, metricFilter, null); + MongoQuery<InstallMetrics> q = new MongoQuery<>(wrapper, metricFilter); Optional<List<InstallMetrics>> cachedResults = installCache.get(listingId, wrapper, () -> StreamHelper.awaitCompletionStage(dao.get(q))); if (!cachedResults.isPresent()) { @@ -188,7 +191,7 @@ public class InstallResource { record.setVersion(version); // create the query wrapper to pass to DB dao - MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, null); + MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(record))); @@ -215,7 +218,7 @@ public class InstallResource { List<CompletionStage<List<MetricPeriod>>> stages = new ArrayList<>(); // get total install count for all listings available Map<String, Integer> overallCounts = new HashMap<>(); - CompletionStage<List<MetricPeriod>> stage = dao.get(new MongoQuery<>(wrapper, periodFilter, null)); + CompletionStage<List<MetricPeriod>> stage = dao.get(new MongoQuery<>(wrapper, periodFilter)); stage.whenComplete((metrics, e) -> { // if theres an error, immediately stop processing if (e != null) { @@ -242,7 +245,7 @@ public class InstallResource { // create the query wrapper to pass to DB dao. No cache needed as this info // won't be cached - MongoQuery<MetricPeriod> q = new MongoQuery<>(wrapper, periodFilter, null); + MongoQuery<MetricPeriod> q = new MongoQuery<>(wrapper, periodFilter); // run query, and set up a completion activity to record data CompletionStage<List<MetricPeriod>> statStage = dao.get(q); statStage.whenComplete((metrics, e) -> { @@ -266,7 +269,7 @@ public class InstallResource { entry.getValue(), overallCounts.getOrDefault(entry.getKey(), 0))).collect(Collectors.toList()); // push the content to the database, and await for it to finish - StreamHelper.awaitCompletionStage(dao.add(new MongoQuery<>(wrapper, metricFilter, null), installMetrics)); + StreamHelper.awaitCompletionStage(dao.add(new MongoQuery<>(wrapper, metricFilter), installMetrics)); // return the results as a response return Response.ok().build(); } diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/ListingResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/ListingResource.java index 996d5da..d31a615 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/ListingResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/ListingResource.java @@ -30,6 +30,7 @@ import javax.ws.rs.core.Response.Status; import org.eclipsefoundation.marketplace.dao.MongoDao; import org.eclipsefoundation.marketplace.dto.Listing; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.Error; import org.eclipsefoundation.marketplace.model.MongoQuery; @@ -62,6 +63,8 @@ public class ListingResource { RequestWrapper params; @Inject DtoFilter<Listing> dtoFilter; + @Inject + ResponseHelper responseBuider; /** * Endpoint for /listing/ to retrieve all listings from the database along with @@ -73,7 +76,7 @@ public class ListingResource { @GET @PermitAll public Response select() { - MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter); // retrieve the possible cached object Optional<List<Listing>> cachedResults = cachingService.get("all", params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -83,7 +86,7 @@ public class ListingResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build("all", params, cachedResults.get()); } /** @@ -95,7 +98,7 @@ public class ListingResource { @PUT @RolesAllowed({ "marketplace_listing_put", "marketplace_admin_access" }) public Response putListing(Listing listing) { - MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(listing))); @@ -117,7 +120,7 @@ public class ListingResource { public Response select(@PathParam("listingId") String listingId) { params.addParam(UrlParameterNames.ID, listingId); - MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter); // retrieve a cached version of the value for the current listing Optional<List<Listing>> cachedResults = cachingService.get(listingId, params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -127,7 +130,7 @@ public class ListingResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(listingId, params, cachedResults.get()); } /** @@ -142,7 +145,7 @@ public class ListingResource { @Path("/{listingId}") public Response delete(@PathParam("listingId") String listingId) { params.addParam(UrlParameterNames.ID, listingId); - MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter); // delete the currently selected asset DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q)); if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) { diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/ListingVersionResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/ListingVersionResource.java index cbd82c8..9b074b8 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/ListingVersionResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/ListingVersionResource.java @@ -25,6 +25,7 @@ import javax.ws.rs.core.Response.Status; import org.eclipsefoundation.marketplace.dao.MongoDao; import org.eclipsefoundation.marketplace.dto.ListingVersion; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.Error; import org.eclipsefoundation.marketplace.model.MongoQuery; @@ -58,10 +59,12 @@ public class ListingVersionResource { RequestWrapper params; @Inject DtoFilter<ListingVersion> dtoFilter; + @Inject + ResponseHelper responseBuider; @GET public Response select() { - MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter); // retrieve the possible cached object Optional<List<ListingVersion>> cachedResults = cachingService.get("all", params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -82,7 +85,7 @@ public class ListingVersionResource { */ @PUT public Response putListingVersion(ListingVersion listingVersion) { - MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(listingVersion))); @@ -102,7 +105,7 @@ public class ListingVersionResource { public Response select(@PathParam("listingVersionId") String listingVersionId) { params.addParam(UrlParameterNames.ID, listingVersionId); - MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter); // retrieve a cached version of the value for the current listing Optional<List<ListingVersion>> cachedResults = cachingService.get(listingVersionId, params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -127,7 +130,7 @@ public class ListingVersionResource { public Response delete(@PathParam("listingVersionId") String listingVersionId) { params.addParam(UrlParameterNames.ID, listingVersionId); - MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter); // delete the currently selected asset DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q)); if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) { diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/MarketResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/MarketResource.java index 4fc2662..44c7101 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/MarketResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/MarketResource.java @@ -27,6 +27,7 @@ import javax.ws.rs.core.Response.Status; import org.eclipsefoundation.marketplace.dao.MongoDao; import org.eclipsefoundation.marketplace.dto.Market; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.Error; import org.eclipsefoundation.marketplace.model.MongoQuery; @@ -58,12 +59,13 @@ public class MarketResource { RequestWrapper params; @Inject DtoFilter<Market> dtoFilter; + @Inject + ResponseHelper responseBuider; - @GET @PermitAll public Response select() { - MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter); // retrieve the possible cached object Optional<List<Market>> cachedResults = cachingService.get("all", params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -73,7 +75,7 @@ public class MarketResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build("all", params, cachedResults.get()); } /** @@ -85,7 +87,7 @@ public class MarketResource { @PUT @RolesAllowed("market_put") public Response putMarket(Market market) { - MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(market))); @@ -107,7 +109,7 @@ public class MarketResource { public Response select(@PathParam("marketId") String marketId) { params.addParam(UrlParameterNames.ID, marketId); - MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter); // retrieve a cached version of the value for the current listing Optional<List<Market>> cachedResults = cachingService.get(marketId, params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -117,7 +119,7 @@ public class MarketResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(marketId, params, cachedResults.get()); } /** @@ -132,7 +134,7 @@ public class MarketResource { public Response delete(@PathParam("marketId") String marketId) { params.addParam(UrlParameterNames.ID, marketId); - MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter); // delete the currently selected asset DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q)); if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) { diff --git a/src/main/java/org/eclipsefoundation/marketplace/service/CachingService.java b/src/main/java/org/eclipsefoundation/marketplace/service/CachingService.java index 74a99f9..4821cab 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/service/CachingService.java +++ b/src/main/java/org/eclipsefoundation/marketplace/service/CachingService.java @@ -33,6 +33,21 @@ public interface CachingService<T> { */ Optional<T> get(String id, RequestWrapper params, Callable<? extends T> callable); + /** + * Returns the expiration date in millis since epoch. + * + * @param id the ID of the object to be stored in cache + * @param params the query parameters for the current request + * @return an Optional expiration date for the current object if its set. If + * there is no underlying data, then empty would be returned + */ + Optional<Long> getExpiration(String id, RequestWrapper params); + + /** + * @return the max age of cache entries + */ + long getMaxAge(); + /** * Retrieves a set of cache keys available to the current cache. * @@ -56,18 +71,18 @@ public interface CachingService<T> { * Generates a unique key based on the id of the item/set of items to be stored, * as well as any passed parameters. * - * @param id identity string of the item to cache - * @param qps parameters associated with the request for information + * @param id identity string of the item to cache + * @param wrapper parameters associated with the request for information * @return the unique cache key for the request. */ - default String getCacheKey(String id, RequestWrapper qps) { + default String getCacheKey(String id, RequestWrapper wrapper) { StringBuilder sb = new StringBuilder(); - sb.append('[').append(qps.getEndpoint()).append(']'); + sb.append('[').append(wrapper.getEndpoint()).append(']'); sb.append("id:").append(id); // join all the non-empty params to the key to create distinct entries for // filtered values - qps.asMap().entrySet().stream().filter(e -> !e.getValue().isEmpty()) + wrapper.asMap().entrySet().stream().filter(e -> !e.getValue().isEmpty()) .map(e -> e.getKey() + '=' + StringUtils.join(e.getValue(), ',')) .forEach(s -> sb.append('|').append(s)); diff --git a/src/main/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingService.java b/src/main/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingService.java index 2d71b3a..d3940a7 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingService.java +++ b/src/main/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingService.java @@ -6,11 +6,12 @@ */ package org.eclipsefoundation.marketplace.service.impl; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import javax.annotation.PostConstruct; @@ -47,24 +48,26 @@ import com.google.common.util.concurrent.UncheckedExecutionException; public class GuavaCachingService<T> implements CachingService<T> { private static final Logger LOGGER = LoggerFactory.getLogger(GuavaCachingService.class); - @ConfigProperty(name = "cache.max", defaultValue = "2500") + @ConfigProperty(name = "cache.max.size", defaultValue = "10000") long maxSize; - @ConfigProperty(name = "cache.ttl.access", defaultValue = "21600") - long ttlAccess; - @ConfigProperty(name = "cache.ttl.write", defaultValue = "86400") + @ConfigProperty(name = "cache.ttl.write.seconds", defaultValue = "900") long ttlWrite; // actual cache object Cache<String, T> cache = null; + Map<String, Long> ttl; @PostConstruct public void init() { + this.ttl = new HashMap<>(); + // create cache with configured settings that maintains a TTL map cache = CacheBuilder - .newBuilder() - .maximumSize(maxSize) - .expireAfterAccess(ttlAccess, TimeUnit.SECONDS) - .expireAfterWrite(ttlWrite, TimeUnit.SECONDS) - .build(); + .newBuilder() + .maximumSize(maxSize) + .expireAfterWrite(ttlWrite, TimeUnit.SECONDS) + .removalListener(not -> ttl.remove(not.getKey())) + .build(); + } @Override @@ -72,18 +75,38 @@ public class GuavaCachingService<T> implements CachingService<T> { Objects.requireNonNull(id); Objects.requireNonNull(params); Objects.requireNonNull(callable); - - String cacheKey = getCacheKey(id, params); + + String cacheKey = getCacheKey(id, params); try { + // check if the cache is bypassed for the request + if (params.isCacheBypass()) { + T result = callable.call(); + // if the cache has a value for key, update it + if (cache.asMap().containsKey(cacheKey)) { + cache.put(cacheKey, result); + } + return Optional.of(result); + } + + // get entry, and enter a ttl as soon as it returns + T data = cache.get(cacheKey, callable); + if (data != null) { + ttl.putIfAbsent(cacheKey, System.currentTimeMillis() + TimeUnit.MILLISECONDS.convert(ttlWrite, TimeUnit.SECONDS)); + } return Optional.of(cache.get(cacheKey, callable)); - } catch (ExecutionException e) { - LOGGER.error("Error while retrieving value of callback", e); } catch (InvalidCacheLoadException | UncheckedExecutionException e) { LOGGER.error("Error while retrieving fresh value for cachekey: {}", cacheKey, e); + } catch (Exception e) { + LOGGER.error("Error while retrieving value of callback", e); } return Optional.empty(); } + @Override + public Optional<Long> getExpiration(String id, RequestWrapper params) { + return Optional.ofNullable(ttl.get(getCacheKey(Objects.requireNonNull(id), Objects.requireNonNull(params)))); + } + @Override public Set<String> getCacheKeys() { return cache.asMap().keySet(); @@ -99,4 +122,8 @@ public class GuavaCachingService<T> implements CachingService<T> { cache.invalidateAll(); } + @Override + public long getMaxAge() { + return ttlWrite; + } } diff --git a/src/test/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingServiceTest.java b/src/test/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingServiceTest.java index 7a25b44..798c1d7 100644 --- a/src/test/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingServiceTest.java +++ b/src/test/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingServiceTest.java @@ -40,10 +40,10 @@ public class GuavaCachingServiceTest { @BeforeEach public void pre() { // inject empty objects into the Request context before creating a mock object - ResteasyContext.pushContext(UriInfo.class, new ResteasyUriInfo("","")); - + ResteasyContext.pushContext(UriInfo.class, new ResteasyUriInfo("", "")); + ResteasyContext.pushContext(HttpServletRequest.class, new HttpServletRequestImpl(null, null)); - + this.sample = new RequestWrapperMock(); // expire all active key values gcs.removeAll(); @@ -60,18 +60,19 @@ public class GuavaCachingServiceTest { // without post construct init via javax management, cache will not be properly // set - Assertions.assertThrows(NullPointerException.class, () -> { - gcsManual.get("sampleKey", sample, Object::new); - }); + Assertions.assertTrue(!gcsManual.get("sampleKey", sample, Object::new).isPresent(), + "Object should not be generated when there is no cache initialized"); // initialize the cache w/ configs gcsManual.init(); // run a command to interact with cache - gcsManual.get("sampleKey", sample, Object::new); + Assertions.assertTrue(gcsManual.get("sampleKey", sample, Object::new).isPresent(), + "Object should be generated once cache is instantiated"); // test the injected cache service (which is the normal use case) - gcs.get("sampleKey", sample, Object::new); + Assertions.assertTrue(gcs.get("sampleKey", sample, Object::new).isPresent(), + "Object should be generated once cache is instantiated"); } @Test @@ -98,7 +99,7 @@ public class GuavaCachingServiceTest { Optional<Object> emptyObj = gcs.get("failure key", sample, () -> null); Assertions.assertFalse(emptyObj.isPresent()); } - + @Test public void testGetExceptionalCallable() { Optional<Object> emptyObj = gcs.get("k", sample, () -> { diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index 2784ee0..9c5f91e 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -1,6 +1,6 @@ ## OAUTH CONFIG quarkus.oauth2.enabled=true -quarkus.oauth2.introspection-url=https://accounts.php56.dev.docker/oauth2/introspect +quarkus.oauth2.introspection-url=http://accounts.php56.dev.docker/oauth2/introspect ## LOGGER CONFIG quarkus.log.file.enable=true @@ -9,7 +9,7 @@ quarkus.log.level=TRACE quarkus.log.file.path=/tmp/logs/quarkus.log ## DATASOURCE CONFIG -quarkus.mongodb.connection-string=mongodb://localhost:27017 +quarkus.mongodb.connection-string=mongodb://192.168.1.178:27017 mongodb.database=mpc mongodb.default.limit=25 mongodb.default.limit.max=100 -- GitLab