diff --git a/src/main/docker/Dockerfile.jvm b/src/main/docker/Dockerfile.jvm index 4a1b9f647d026fe9771cb38bf1b21b2503f99eae..985ce5feac6d3da7889330c39585debefc384aa4 100644 --- a/src/main/docker/Dockerfile.jvm +++ b/src/main/docker/Dockerfile.jvm @@ -3,27 +3,18 @@ # # Before building the docker image run: # -# mvn package +# mvn package -Dconfig.secret.path=<full path to secret file> # # Then, build the image with: # -# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/sample-jvm . +# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/mpc-rest-api-jvm . # # Then run the container using: # -# docker run -i --rm -p 8080:8080 quarkus/sample-jvm +# docker run -i --rm -p 8090:8090 quarkus/mpc-rest-api-jvm # ### FROM fabric8/java-alpine-openjdk8-jre - -## Where to source the cert file -ARG LOCAL_CRT=config/local.crt -ENV LOCAL_CRT ${LOCAL_CRT} - -## copy to a temp ssl dir for container usage -WORKDIR /tmp -RUN mkdir ssl -COPY $LOCAL_CRT ssl/local.crt ## Where to copy the secret file, default to tmp ARG SECRET_LOCATION=/tmp diff --git a/src/main/java/org/eclipsefoundation/marketplace/helper/ResponseHelper.java b/src/main/java/org/eclipsefoundation/marketplace/helper/ResponseHelper.java new file mode 100644 index 0000000000000000000000000000000000000000..d991e7c015e7a902cf6af093fb468f5267997ace --- /dev/null +++ b/src/main/java/org/eclipsefoundation/marketplace/helper/ResponseHelper.java @@ -0,0 +1,94 @@ +/* Copyright (c) 2019 Eclipse Foundation and others. + * This program and the accompanying materials are made available + * under the terms of the Eclipse Public License 2.0 + * which is available at http://www.eclipse.org/legal/epl-v20.html, + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipsefoundation.marketplace.helper; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Date; +import java.util.Objects; +import java.util.Optional; + +import javax.enterprise.context.ApplicationScoped; +import javax.inject.Inject; +import javax.json.bind.Jsonb; +import javax.ws.rs.core.CacheControl; +import javax.ws.rs.core.Response; +import javax.xml.bind.DatatypeConverter; + +import org.eclipsefoundation.marketplace.model.RequestWrapper; +import org.eclipsefoundation.marketplace.service.CachingService; + +/** + * Helper class that transforms data into a response usable for the RESTeasy + * container. Uses injected JSON-B serializer and caching service to get current + * information on cache data. + * + * @author Martin Lowe + * + */ +@ApplicationScoped +public class ResponseHelper { + + private static final MessageDigest DIGEST; + static { + try { + DIGEST = MessageDigest.getInstance("md5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("Could not create an MD5 hash digest"); + } + } + + @Inject + Jsonb jsonb; + @Inject + CachingService<?> cachingService; + + /** + * Builds a response using passed data. Uses references to the caching service + * and the current request to add information about ETags and Cache-Control + * headers. + * + * @param id the ID of the object to be stored in cache + * @param wrapper the query parameters for the current request + * @param data the data to attach to the response + * @return a complete response object for the given data and request. + */ + public Response build(String id, RequestWrapper wrapper, Object data) { + // set default cache control flags for API responses + CacheControl cc = new CacheControl(); + cc.setNoStore(wrapper.isCacheBypass()); + + if (!cc.isNoStore()) { + cc.setMaxAge((int) cachingService.getMaxAge()); + // get the TTL for the current entry + Optional<Long> ttl = cachingService.getExpiration(id, wrapper); + if (!ttl.isPresent()) { + return Response.serverError().build(); + } + + // serialize the data to get an etag + String content = jsonb.toJson(Objects.requireNonNull(data)); + // ingest the content and hash to create an etag for current content + String hash; + synchronized (this) { + DIGEST.update(content.getBytes(StandardCharsets.UTF_8)); + hash = DatatypeConverter.printHexBinary(DIGEST.digest()); + DIGEST.reset(); + } + + // check if etag matches + String etag = wrapper.getHeader("Etag"); + if (hash.equals(etag)) { + return Response.notModified(etag).cacheControl(cc).expires(new Date(ttl.get())).build(); + } + // return a response w/ the generated etag + return Response.ok(data).tag(hash).cacheControl(cc).expires(new Date(ttl.get())).build(); + } + return Response.ok(data).cacheControl(cc).build(); + } +} diff --git a/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java b/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java index 1b487325947c97c789dc3f6fb21fa5b1d793956e..a392fb61948b1b5151ec2c77b68c3b8a32631c04 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java +++ b/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java @@ -17,7 +17,6 @@ import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; import org.eclipsefoundation.marketplace.helper.SortableHelper; import org.eclipsefoundation.marketplace.helper.SortableHelper.Sortable; import org.eclipsefoundation.marketplace.namespace.UrlParameterNames; -import org.eclipsefoundation.marketplace.service.CachingService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +34,6 @@ import com.mongodb.client.model.Filters; public class MongoQuery<T> { private static final Logger LOGGER = LoggerFactory.getLogger(MongoQuery.class); - private CachingService<List<T>> cache; private RequestWrapper wrapper; private DtoFilter<T> dtoFilter; @@ -44,10 +42,9 @@ public class MongoQuery<T> { private SortOrder order; private List<Bson> aggregates; - public MongoQuery(RequestWrapper wrapper, DtoFilter<T> dtoFilter, CachingService<List<T>> cache) { + public MongoQuery(RequestWrapper wrapper, DtoFilter<T> dtoFilter) { this.wrapper = wrapper; this.dtoFilter = dtoFilter; - this.cache = cache; this.aggregates = new ArrayList<>(); init(); } @@ -72,12 +69,13 @@ public class MongoQuery<T> { Optional<String> sortOpt = wrapper.getFirstParam(UrlParameterNames.SORT); if (sortOpt.isPresent()) { String sortVal = sortOpt.get(); + SortOrder ord = SortOrder.getOrderFromValue(sortOpt.get()); // split sort string of `<fieldName> <SortOrder>` int idx = sortVal.indexOf(' '); // check if the sort string matches the RANDOM sort order - if (SortOrder.RANDOM.equals(SortOrder.getOrderByName(sortVal))) { + if (SortOrder.RANDOM.equals(ord)) { this.order = SortOrder.RANDOM; - } else if (idx > 0) { + } else if (ord != SortOrder.NONE) { setSort(sortVal.substring(0, idx), sortVal.substring(idx + 1), filters); } } @@ -143,13 +141,6 @@ public class MongoQuery<T> { this.order = SortOrder.getOrderByName(sortOrder); // add sorting query if the sortOrder matches a defined order switch (order) { - case RANDOM: - // TODO support for random, implement the following (in this order) - // 1. Add not in clause that checks Cache for previously read objects - // 2. Set useAggregate flag to true to signal to DAO to use aggregate selection - // rather than traditional find - - break; case ASCENDING: // if last seen is set, add a filter to shift the results if (lastOpt.isPresent()) { diff --git a/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java b/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java index aec134a4a2ebf5f92680293a873c5e6c9f66b2eb..43fc3469c2b916fc78933073cc350bddcf709a25 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java +++ b/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java @@ -23,6 +23,7 @@ import javax.ws.rs.core.UriInfo; import org.apache.commons.lang3.StringUtils; import org.eclipsefoundation.marketplace.namespace.DeprecatedHeader; import org.eclipsefoundation.marketplace.namespace.RequestHeaderNames; +import org.eclipsefoundation.marketplace.request.CacheBypassFilter; import org.jboss.resteasy.core.ResteasyContext; /** @@ -168,7 +169,18 @@ public class RequestWrapper { } /** - * Retrieve a request header value as an optional value. + * Check whether the current request should bypass caching + * + * @return true if cache should be bypassed, otherwise false + */ + public boolean isCacheBypass() { + Object attr = request.getAttribute(CacheBypassFilter.ATTRIBUTE_NAME); + // if we have the attribute set on the request, return it. otherwise, false. + return attr instanceof Boolean ? (boolean) attr : Boolean.FALSE; + } + + /** + * Retrieve a request header value. * * @param key the headers key value * @return the value, or an empty optional if missing. diff --git a/src/main/java/org/eclipsefoundation/marketplace/model/SortOrder.java b/src/main/java/org/eclipsefoundation/marketplace/model/SortOrder.java index 86c250ef72586c784bbc6a47fb15585a6a722f75..f850948d0dd2c3d97015973cbd03ebd0ea33f338 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/model/SortOrder.java +++ b/src/main/java/org/eclipsefoundation/marketplace/model/SortOrder.java @@ -53,4 +53,24 @@ public enum SortOrder { } return NONE; } + + /** + * Gets the SortOrder value associated with a sort parameter value if one + * exists. + * + * @param value the value of the sort parameter + * @return the SortOrder associated with the request, or + * {@linkplain SortOrder.NONE} + */ + public static SortOrder getOrderFromValue(String value) { + // get the index of the space separator + int idx = value.indexOf(' '); + // check if the sort string matches the RANDOM sort order + if (SortOrder.RANDOM.equals(SortOrder.getOrderByName(value))) { + return SortOrder.RANDOM; + } else if (idx > 0) { + return SortOrder.getOrderByName(value.substring(idx + 1)); + } + return SortOrder.NONE; + } } \ No newline at end of file diff --git a/src/main/java/org/eclipsefoundation/marketplace/request/CacheBypassFilter.java b/src/main/java/org/eclipsefoundation/marketplace/request/CacheBypassFilter.java new file mode 100644 index 0000000000000000000000000000000000000000..636c40b5e8cdd6fb6cd35a5cd8142c799f1651e3 --- /dev/null +++ b/src/main/java/org/eclipsefoundation/marketplace/request/CacheBypassFilter.java @@ -0,0 +1,60 @@ +/* Copyright (c) 2019 Eclipse Foundation and others. + * This program and the accompanying materials are made available + * under the terms of the Eclipse Public License 2.0 + * which is available at http://www.eclipse.org/legal/epl-v20.html, + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipsefoundation.marketplace.request; + +import java.io.IOException; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerRequestFilter; +import javax.ws.rs.core.Context; +import javax.ws.rs.ext.Provider; + +import org.eclipsefoundation.marketplace.model.SortOrder; +import org.eclipsefoundation.marketplace.namespace.UrlParameterNames; + +/** + * Checks passed parameters and if any match one of the criteria for bypassing + * caching, an attribute will be set to the request to skip cache requests and + * instead directly return results. + * + * @author Martin Lowe + * + */ +@Provider +public class CacheBypassFilter implements ContainerRequestFilter { + public static final String ATTRIBUTE_NAME = "bypass-cache"; + + @Context + HttpServletRequest request; + + @Context + HttpServletResponse response; + + @Override + public void filter(ContainerRequestContext requestContext) throws IOException { + // check for random sort order, which always bypasses cache + String[] sortVals = request.getParameterValues(UrlParameterNames.SORT); + if (sortVals != null) { + for (String sortVal : sortVals) { + // check if the sort order for request matches RANDOM + if (SortOrder.RANDOM.equals(SortOrder.getOrderFromValue(sortVal))) { + setBypass(); + return; + } + } + } + request.setAttribute(ATTRIBUTE_NAME, Boolean.FALSE); + } + + private void setBypass() { + request.setAttribute(ATTRIBUTE_NAME, Boolean.TRUE); + // no-store should be used as cache bypass should not return + response.setHeader("Cache-Control", "no-store"); + } +} diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/CatalogResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/CatalogResource.java index 84cb41b7d8a870019fe3c7fe8ee9b52a3a48c8a2..0aa35a0823ad6d47f990754710457918400dbf63 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/CatalogResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/CatalogResource.java @@ -27,6 +27,7 @@ import javax.ws.rs.core.Response.Status; import org.eclipsefoundation.marketplace.dao.MongoDao; import org.eclipsefoundation.marketplace.dto.Catalog; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.Error; import org.eclipsefoundation.marketplace.model.MongoQuery; @@ -58,11 +59,13 @@ public class CatalogResource { RequestWrapper params; @Inject DtoFilter<Catalog> dtoFilter; + @Inject + ResponseHelper responseBuider; @GET @PermitAll public Response select() { - MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter); // retrieve the possible cached object Optional<List<Catalog>> cachedResults = cachingService.get("all", params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -72,7 +75,7 @@ public class CatalogResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build("all", params, cachedResults.get()); } /** @@ -84,7 +87,7 @@ public class CatalogResource { @PUT @RolesAllowed({ "marketplace_catalog_put", "marketplace_admin_access" }) public Response putCatalog(Catalog catalog) { - MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(catalog))); @@ -104,7 +107,7 @@ public class CatalogResource { public Response select(@PathParam("catalogId") String catalogId) { params.addParam(UrlParameterNames.ID, catalogId); - MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter); // retrieve a cached version of the value for the current listing Optional<List<Catalog>> cachedResults = cachingService.get(catalogId, params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -114,7 +117,7 @@ public class CatalogResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(catalogId, params, cachedResults.get()); } /** @@ -130,7 +133,7 @@ public class CatalogResource { public Response delete(@PathParam("catalogId") String catalogId) { params.addParam(UrlParameterNames.ID, catalogId); - MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter); // delete the currently selected asset DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q)); if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) { diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/CategoryResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/CategoryResource.java index ed7307476d2734a48f191d99bad03bdfc53e8167..b343c314fe57312992e7b575cc7822a3bf5f9c4f 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/CategoryResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/CategoryResource.java @@ -27,6 +27,7 @@ import javax.ws.rs.core.Response.Status; import org.eclipsefoundation.marketplace.dao.MongoDao; import org.eclipsefoundation.marketplace.dto.Category; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.Error; import org.eclipsefoundation.marketplace.model.MongoQuery; @@ -58,11 +59,13 @@ public class CategoryResource { RequestWrapper params; @Inject DtoFilter<Category> dtoFilter; + @Inject + ResponseHelper responseBuider; @GET @PermitAll public Response select() { - MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter); // retrieve the possible cached object Optional<List<Category>> cachedResults = cachingService.get("all", params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -72,7 +75,7 @@ public class CategoryResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build("all", params, cachedResults.get()); } /** @@ -84,7 +87,7 @@ public class CategoryResource { @PUT @RolesAllowed({"marketplace_category_put", "marketplace_admin_access"}) public Response putCategory(Category category) { - MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(category))); @@ -104,7 +107,7 @@ public class CategoryResource { public Response select(@PathParam("categoryId") String categoryId) { params.addParam(UrlParameterNames.ID, categoryId); - MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter); // retrieve a cached version of the value for the current listing Optional<List<Category>> cachedResults = cachingService.get(categoryId, params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -114,7 +117,7 @@ public class CategoryResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(categoryId, params, cachedResults.get()); } /** @@ -130,7 +133,7 @@ public class CategoryResource { public Response delete(@PathParam("categoryId") String categoryId) { params.addParam(UrlParameterNames.ID, categoryId); - MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter); // delete the currently selected asset DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q)); if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) { diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/ErrorReportResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/ErrorReportResource.java index 80d5e8d0e439a70385dce4a417355bc67d6201ca..5c8b1adc9d6c4417b5ead3ff67be41f4678fd879 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/ErrorReportResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/ErrorReportResource.java @@ -25,6 +25,7 @@ import javax.ws.rs.core.Response; import org.eclipsefoundation.marketplace.dao.MongoDao; import org.eclipsefoundation.marketplace.dto.ErrorReport; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.MongoQuery; import org.eclipsefoundation.marketplace.model.RequestWrapper; @@ -54,6 +55,8 @@ public class ErrorReportResource { RequestWrapper params; @Inject DtoFilter<ErrorReport> dtoFilter; + @Inject + ResponseHelper responseBuider; /** * Endpoint for /error/ to retrieve all ErrorReports from the database along with @@ -65,7 +68,7 @@ public class ErrorReportResource { @GET @PermitAll public Response select() { - MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter); // retrieve the possible cached object Optional<List<ErrorReport>> cachedResults = cachingService.get("all", params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -75,7 +78,7 @@ public class ErrorReportResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build("all", params, cachedResults.get()); } /** @@ -87,7 +90,7 @@ public class ErrorReportResource { @PUT @RolesAllowed("error_put") public Response putErrorReport(ErrorReport errorReport) { - MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(errorReport))); @@ -109,7 +112,7 @@ public class ErrorReportResource { public Response select(@PathParam("errorReportId") String errorReportId) { params.addParam(UrlParameterNames.ID, errorReportId); - MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter); // retrieve a cached version of the value for the current ErrorReport Optional<List<ErrorReport>> cachedResults = cachingService.get(errorReportId, params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -119,6 +122,6 @@ public class ErrorReportResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(errorReportId, params, cachedResults.get()); } } diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java index 8c1562d81d84b1df9593b4dbe56a3652bef9bdff..9519ff493f5807e40fff9e1d296ee928ac4b8b9a 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java @@ -37,6 +37,7 @@ import org.eclipsefoundation.marketplace.dto.InstallMetrics; import org.eclipsefoundation.marketplace.dto.MetricPeriod; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; import org.eclipsefoundation.marketplace.helper.DateTimeHelper; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.Error; import org.eclipsefoundation.marketplace.model.MongoQuery; @@ -63,6 +64,8 @@ public class InstallResource { MongoDao dao; @Inject RequestWrapper wrapper; + @Inject + ResponseHelper responseBuider; // insert required filters for different objects + states @Inject @@ -90,7 +93,7 @@ public class InstallResource { @Path("/{listingId}") public Response selectInstallCount(@PathParam("listingId") String listingId) { wrapper.addParam(UrlParameterNames.ID, listingId); - MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, null); + MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter); Optional<Long> cachedResults = countCache.get(listingId, wrapper, () -> StreamHelper.awaitCompletionStage(dao.count(q))); if (!cachedResults.isPresent()) { @@ -99,7 +102,7 @@ public class InstallResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(listingId, wrapper, cachedResults.get()); } /** @@ -117,7 +120,7 @@ public class InstallResource { public Response selectInstallCount(@PathParam("listingId") String listingId, @PathParam("version") String version) { wrapper.addParam(UrlParameterNames.ID, listingId); wrapper.addParam(UrlParameterNames.VERSION, version); - MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, null); + MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter); Optional<Long> cachedResults = countCache.get(getCompositeKey(listingId, version), wrapper, () -> StreamHelper.awaitCompletionStage(dao.count(q))); if (!cachedResults.isPresent()) { @@ -126,7 +129,7 @@ public class InstallResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(getCompositeKey(listingId, version), wrapper, cachedResults.get()); } /** @@ -141,7 +144,7 @@ public class InstallResource { @Path("/{listingId}/metrics") public Response selectInstallMetrics(@PathParam("listingId") String listingId) { wrapper.addParam(UrlParameterNames.ID, listingId); - MongoQuery<InstallMetrics> q = new MongoQuery<>(wrapper, metricFilter, null); + MongoQuery<InstallMetrics> q = new MongoQuery<>(wrapper, metricFilter); Optional<List<InstallMetrics>> cachedResults = installCache.get(listingId, wrapper, () -> StreamHelper.awaitCompletionStage(dao.get(q))); if (!cachedResults.isPresent()) { @@ -188,7 +191,7 @@ public class InstallResource { record.setVersion(version); // create the query wrapper to pass to DB dao - MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, null); + MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(record))); @@ -215,7 +218,7 @@ public class InstallResource { List<CompletionStage<List<MetricPeriod>>> stages = new ArrayList<>(); // get total install count for all listings available Map<String, Integer> overallCounts = new HashMap<>(); - CompletionStage<List<MetricPeriod>> stage = dao.get(new MongoQuery<>(wrapper, periodFilter, null)); + CompletionStage<List<MetricPeriod>> stage = dao.get(new MongoQuery<>(wrapper, periodFilter)); stage.whenComplete((metrics, e) -> { // if theres an error, immediately stop processing if (e != null) { @@ -242,7 +245,7 @@ public class InstallResource { // create the query wrapper to pass to DB dao. No cache needed as this info // won't be cached - MongoQuery<MetricPeriod> q = new MongoQuery<>(wrapper, periodFilter, null); + MongoQuery<MetricPeriod> q = new MongoQuery<>(wrapper, periodFilter); // run query, and set up a completion activity to record data CompletionStage<List<MetricPeriod>> statStage = dao.get(q); statStage.whenComplete((metrics, e) -> { @@ -266,7 +269,7 @@ public class InstallResource { entry.getValue(), overallCounts.getOrDefault(entry.getKey(), 0))).collect(Collectors.toList()); // push the content to the database, and await for it to finish - StreamHelper.awaitCompletionStage(dao.add(new MongoQuery<>(wrapper, metricFilter, null), installMetrics)); + StreamHelper.awaitCompletionStage(dao.add(new MongoQuery<>(wrapper, metricFilter), installMetrics)); // return the results as a response return Response.ok().build(); } diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/ListingResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/ListingResource.java index 996d5da2258d583b541f45bdcc236f5045a14e59..d31a6153f8d4c65988e235df26f35fcf7e53d665 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/ListingResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/ListingResource.java @@ -30,6 +30,7 @@ import javax.ws.rs.core.Response.Status; import org.eclipsefoundation.marketplace.dao.MongoDao; import org.eclipsefoundation.marketplace.dto.Listing; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.Error; import org.eclipsefoundation.marketplace.model.MongoQuery; @@ -62,6 +63,8 @@ public class ListingResource { RequestWrapper params; @Inject DtoFilter<Listing> dtoFilter; + @Inject + ResponseHelper responseBuider; /** * Endpoint for /listing/ to retrieve all listings from the database along with @@ -73,7 +76,7 @@ public class ListingResource { @GET @PermitAll public Response select() { - MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter); // retrieve the possible cached object Optional<List<Listing>> cachedResults = cachingService.get("all", params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -83,7 +86,7 @@ public class ListingResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build("all", params, cachedResults.get()); } /** @@ -95,7 +98,7 @@ public class ListingResource { @PUT @RolesAllowed({ "marketplace_listing_put", "marketplace_admin_access" }) public Response putListing(Listing listing) { - MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(listing))); @@ -117,7 +120,7 @@ public class ListingResource { public Response select(@PathParam("listingId") String listingId) { params.addParam(UrlParameterNames.ID, listingId); - MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter); // retrieve a cached version of the value for the current listing Optional<List<Listing>> cachedResults = cachingService.get(listingId, params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -127,7 +130,7 @@ public class ListingResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(listingId, params, cachedResults.get()); } /** @@ -142,7 +145,7 @@ public class ListingResource { @Path("/{listingId}") public Response delete(@PathParam("listingId") String listingId) { params.addParam(UrlParameterNames.ID, listingId); - MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter); // delete the currently selected asset DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q)); if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) { diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/ListingVersionResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/ListingVersionResource.java index cbd82c896110e612af408ea37551e20638220748..9b074b875081187bd65edf3c5ccc7d840e3811f3 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/ListingVersionResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/ListingVersionResource.java @@ -25,6 +25,7 @@ import javax.ws.rs.core.Response.Status; import org.eclipsefoundation.marketplace.dao.MongoDao; import org.eclipsefoundation.marketplace.dto.ListingVersion; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.Error; import org.eclipsefoundation.marketplace.model.MongoQuery; @@ -58,10 +59,12 @@ public class ListingVersionResource { RequestWrapper params; @Inject DtoFilter<ListingVersion> dtoFilter; + @Inject + ResponseHelper responseBuider; @GET public Response select() { - MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter); // retrieve the possible cached object Optional<List<ListingVersion>> cachedResults = cachingService.get("all", params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -82,7 +85,7 @@ public class ListingVersionResource { */ @PUT public Response putListingVersion(ListingVersion listingVersion) { - MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(listingVersion))); @@ -102,7 +105,7 @@ public class ListingVersionResource { public Response select(@PathParam("listingVersionId") String listingVersionId) { params.addParam(UrlParameterNames.ID, listingVersionId); - MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter); // retrieve a cached version of the value for the current listing Optional<List<ListingVersion>> cachedResults = cachingService.get(listingVersionId, params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -127,7 +130,7 @@ public class ListingVersionResource { public Response delete(@PathParam("listingVersionId") String listingVersionId) { params.addParam(UrlParameterNames.ID, listingVersionId); - MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter); // delete the currently selected asset DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q)); if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) { diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/MarketResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/MarketResource.java index 4fc266221e3c1c01f6b97d76f3090cf91f876b11..44c710117eea26ae9eefe267d001b85a6ba82f95 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/resource/MarketResource.java +++ b/src/main/java/org/eclipsefoundation/marketplace/resource/MarketResource.java @@ -27,6 +27,7 @@ import javax.ws.rs.core.Response.Status; import org.eclipsefoundation.marketplace.dao.MongoDao; import org.eclipsefoundation.marketplace.dto.Market; import org.eclipsefoundation.marketplace.dto.filter.DtoFilter; +import org.eclipsefoundation.marketplace.helper.ResponseHelper; import org.eclipsefoundation.marketplace.helper.StreamHelper; import org.eclipsefoundation.marketplace.model.Error; import org.eclipsefoundation.marketplace.model.MongoQuery; @@ -58,12 +59,13 @@ public class MarketResource { RequestWrapper params; @Inject DtoFilter<Market> dtoFilter; + @Inject + ResponseHelper responseBuider; - @GET @PermitAll public Response select() { - MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter); // retrieve the possible cached object Optional<List<Market>> cachedResults = cachingService.get("all", params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -73,7 +75,7 @@ public class MarketResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build("all", params, cachedResults.get()); } /** @@ -85,7 +87,7 @@ public class MarketResource { @PUT @RolesAllowed("market_put") public Response putMarket(Market market) { - MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter); // add the object, and await the result StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(market))); @@ -107,7 +109,7 @@ public class MarketResource { public Response select(@PathParam("marketId") String marketId) { params.addParam(UrlParameterNames.ID, marketId); - MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter); // retrieve a cached version of the value for the current listing Optional<List<Market>> cachedResults = cachingService.get(marketId, params, () -> StreamHelper.awaitCompletionStage(dao.get(q))); @@ -117,7 +119,7 @@ public class MarketResource { } // return the results as a response - return Response.ok(cachedResults.get()).build(); + return responseBuider.build(marketId, params, cachedResults.get()); } /** @@ -132,7 +134,7 @@ public class MarketResource { public Response delete(@PathParam("marketId") String marketId) { params.addParam(UrlParameterNames.ID, marketId); - MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter, cachingService); + MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter); // delete the currently selected asset DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q)); if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) { diff --git a/src/main/java/org/eclipsefoundation/marketplace/service/CachingService.java b/src/main/java/org/eclipsefoundation/marketplace/service/CachingService.java index 74a99f9ec3b00b2b7380137d27665a1ebb53d5a2..4821cab5aa58db293363723caaef12504f1901ae 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/service/CachingService.java +++ b/src/main/java/org/eclipsefoundation/marketplace/service/CachingService.java @@ -33,6 +33,21 @@ public interface CachingService<T> { */ Optional<T> get(String id, RequestWrapper params, Callable<? extends T> callable); + /** + * Returns the expiration date in millis since epoch. + * + * @param id the ID of the object to be stored in cache + * @param params the query parameters for the current request + * @return an Optional expiration date for the current object if its set. If + * there is no underlying data, then empty would be returned + */ + Optional<Long> getExpiration(String id, RequestWrapper params); + + /** + * @return the max age of cache entries + */ + long getMaxAge(); + /** * Retrieves a set of cache keys available to the current cache. * @@ -56,18 +71,18 @@ public interface CachingService<T> { * Generates a unique key based on the id of the item/set of items to be stored, * as well as any passed parameters. * - * @param id identity string of the item to cache - * @param qps parameters associated with the request for information + * @param id identity string of the item to cache + * @param wrapper parameters associated with the request for information * @return the unique cache key for the request. */ - default String getCacheKey(String id, RequestWrapper qps) { + default String getCacheKey(String id, RequestWrapper wrapper) { StringBuilder sb = new StringBuilder(); - sb.append('[').append(qps.getEndpoint()).append(']'); + sb.append('[').append(wrapper.getEndpoint()).append(']'); sb.append("id:").append(id); // join all the non-empty params to the key to create distinct entries for // filtered values - qps.asMap().entrySet().stream().filter(e -> !e.getValue().isEmpty()) + wrapper.asMap().entrySet().stream().filter(e -> !e.getValue().isEmpty()) .map(e -> e.getKey() + '=' + StringUtils.join(e.getValue(), ',')) .forEach(s -> sb.append('|').append(s)); diff --git a/src/main/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingService.java b/src/main/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingService.java index 2d71b3a9b8cc7a20e09f7ed0d4cda5563b3d260a..d3940a77d970a8c9925dfa25fe2662b68c483dc3 100644 --- a/src/main/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingService.java +++ b/src/main/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingService.java @@ -6,11 +6,12 @@ */ package org.eclipsefoundation.marketplace.service.impl; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import javax.annotation.PostConstruct; @@ -47,24 +48,26 @@ import com.google.common.util.concurrent.UncheckedExecutionException; public class GuavaCachingService<T> implements CachingService<T> { private static final Logger LOGGER = LoggerFactory.getLogger(GuavaCachingService.class); - @ConfigProperty(name = "cache.max", defaultValue = "2500") + @ConfigProperty(name = "cache.max.size", defaultValue = "10000") long maxSize; - @ConfigProperty(name = "cache.ttl.access", defaultValue = "21600") - long ttlAccess; - @ConfigProperty(name = "cache.ttl.write", defaultValue = "86400") + @ConfigProperty(name = "cache.ttl.write.seconds", defaultValue = "900") long ttlWrite; // actual cache object Cache<String, T> cache = null; + Map<String, Long> ttl; @PostConstruct public void init() { + this.ttl = new HashMap<>(); + // create cache with configured settings that maintains a TTL map cache = CacheBuilder - .newBuilder() - .maximumSize(maxSize) - .expireAfterAccess(ttlAccess, TimeUnit.SECONDS) - .expireAfterWrite(ttlWrite, TimeUnit.SECONDS) - .build(); + .newBuilder() + .maximumSize(maxSize) + .expireAfterWrite(ttlWrite, TimeUnit.SECONDS) + .removalListener(not -> ttl.remove(not.getKey())) + .build(); + } @Override @@ -72,18 +75,38 @@ public class GuavaCachingService<T> implements CachingService<T> { Objects.requireNonNull(id); Objects.requireNonNull(params); Objects.requireNonNull(callable); - - String cacheKey = getCacheKey(id, params); + + String cacheKey = getCacheKey(id, params); try { + // check if the cache is bypassed for the request + if (params.isCacheBypass()) { + T result = callable.call(); + // if the cache has a value for key, update it + if (cache.asMap().containsKey(cacheKey)) { + cache.put(cacheKey, result); + } + return Optional.of(result); + } + + // get entry, and enter a ttl as soon as it returns + T data = cache.get(cacheKey, callable); + if (data != null) { + ttl.putIfAbsent(cacheKey, System.currentTimeMillis() + TimeUnit.MILLISECONDS.convert(ttlWrite, TimeUnit.SECONDS)); + } return Optional.of(cache.get(cacheKey, callable)); - } catch (ExecutionException e) { - LOGGER.error("Error while retrieving value of callback", e); } catch (InvalidCacheLoadException | UncheckedExecutionException e) { LOGGER.error("Error while retrieving fresh value for cachekey: {}", cacheKey, e); + } catch (Exception e) { + LOGGER.error("Error while retrieving value of callback", e); } return Optional.empty(); } + @Override + public Optional<Long> getExpiration(String id, RequestWrapper params) { + return Optional.ofNullable(ttl.get(getCacheKey(Objects.requireNonNull(id), Objects.requireNonNull(params)))); + } + @Override public Set<String> getCacheKeys() { return cache.asMap().keySet(); @@ -99,4 +122,8 @@ public class GuavaCachingService<T> implements CachingService<T> { cache.invalidateAll(); } + @Override + public long getMaxAge() { + return ttlWrite; + } } diff --git a/src/test/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingServiceTest.java b/src/test/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingServiceTest.java index 7a25b44366f8dbf52edd2984b8f73c6111545b14..798c1d7ef3e640eb1d876ab96eecc9124efa6dfb 100644 --- a/src/test/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingServiceTest.java +++ b/src/test/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingServiceTest.java @@ -40,10 +40,10 @@ public class GuavaCachingServiceTest { @BeforeEach public void pre() { // inject empty objects into the Request context before creating a mock object - ResteasyContext.pushContext(UriInfo.class, new ResteasyUriInfo("","")); - + ResteasyContext.pushContext(UriInfo.class, new ResteasyUriInfo("", "")); + ResteasyContext.pushContext(HttpServletRequest.class, new HttpServletRequestImpl(null, null)); - + this.sample = new RequestWrapperMock(); // expire all active key values gcs.removeAll(); @@ -60,18 +60,19 @@ public class GuavaCachingServiceTest { // without post construct init via javax management, cache will not be properly // set - Assertions.assertThrows(NullPointerException.class, () -> { - gcsManual.get("sampleKey", sample, Object::new); - }); + Assertions.assertTrue(!gcsManual.get("sampleKey", sample, Object::new).isPresent(), + "Object should not be generated when there is no cache initialized"); // initialize the cache w/ configs gcsManual.init(); // run a command to interact with cache - gcsManual.get("sampleKey", sample, Object::new); + Assertions.assertTrue(gcsManual.get("sampleKey", sample, Object::new).isPresent(), + "Object should be generated once cache is instantiated"); // test the injected cache service (which is the normal use case) - gcs.get("sampleKey", sample, Object::new); + Assertions.assertTrue(gcs.get("sampleKey", sample, Object::new).isPresent(), + "Object should be generated once cache is instantiated"); } @Test @@ -98,7 +99,7 @@ public class GuavaCachingServiceTest { Optional<Object> emptyObj = gcs.get("failure key", sample, () -> null); Assertions.assertFalse(emptyObj.isPresent()); } - + @Test public void testGetExceptionalCallable() { Optional<Object> emptyObj = gcs.get("k", sample, () -> { diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index 2784ee044c1b30c233d699b84e12897ca3330f55..9c5f91e89329cd818f9bf6a1fb1d238b28cd365b 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -1,6 +1,6 @@ ## OAUTH CONFIG quarkus.oauth2.enabled=true -quarkus.oauth2.introspection-url=https://accounts.php56.dev.docker/oauth2/introspect +quarkus.oauth2.introspection-url=http://accounts.php56.dev.docker/oauth2/introspect ## LOGGER CONFIG quarkus.log.file.enable=true @@ -9,7 +9,7 @@ quarkus.log.level=TRACE quarkus.log.file.path=/tmp/logs/quarkus.log ## DATASOURCE CONFIG -quarkus.mongodb.connection-string=mongodb://localhost:27017 +quarkus.mongodb.connection-string=mongodb://192.168.1.178:27017 mongodb.database=mpc mongodb.default.limit=25 mongodb.default.limit.max=100