Implement solution for deduplication of random results #7

Added Cache-Control and Etag header values for responses. Added no-store
to random sort order requests, as well as a cache bypass mechanism. This
mechanism currently only triggers when sort order is set to random.
Added calls to cache service to maintain expiration times as well as
accessor for max age.

Change-Id: If7b4b57c7e265fe69ef4fdefec4249ea55bcdf5d
Signed-off-by: Martin Lowe <martin.lowe@eclipse-foundation.org>

src/main/docker/Dockerfile.jvm

@@ -3,27 +3,18 @@
 #
 # Before building the docker image run:
 #
-# mvn package
+# mvn package -Dconfig.secret.path=<full path to secret file>
 #
 # Then, build the image with:
 #
-# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/sample-jvm .
+# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/mpc-rest-api-jvm .
 #
 # Then run the container using:
 #
-# docker run -i --rm -p 8080:8080 quarkus/sample-jvm
+# docker run -i --rm -p 8090:8090 quarkus/mpc-rest-api-jvm
 #
 ###
 FROM fabric8/java-alpine-openjdk8-jre
-  
-## Where to source the cert file
-ARG LOCAL_CRT=config/local.crt
-ENV LOCAL_CRT ${LOCAL_CRT}
-
-## copy to a temp ssl dir for container usage
-WORKDIR /tmp
-RUN mkdir ssl
-COPY $LOCAL_CRT ssl/local.crt
 
 ## Where to copy the secret file, default to tmp
 ARG SECRET_LOCATION=/tmp

src/main/java/org/eclipsefoundation/marketplace/helper/ResponseHelper.java

+/* Copyright (c) 2019 Eclipse Foundation and others.
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Public License 2.0
+ * which is available at http://www.eclipse.org/legal/epl-v20.html,
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.marketplace.helper;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Date;
+import java.util.Objects;
+import java.util.Optional;
+
+import javax.enterprise.context.ApplicationScoped;
+import javax.inject.Inject;
+import javax.json.bind.Jsonb;
+import javax.ws.rs.core.CacheControl;
+import javax.ws.rs.core.Response;
+import javax.xml.bind.DatatypeConverter;
+
+import org.eclipsefoundation.marketplace.model.RequestWrapper;
+import org.eclipsefoundation.marketplace.service.CachingService;
+
+/**
+ * Helper class that transforms data into a response usable for the RESTeasy
+ * container. Uses injected JSON-B serializer and caching service to get current
+ * information on cache data.
+ * 
+ * @author Martin Lowe
+ *
+ */
+@ApplicationScoped
+public class ResponseHelper {
+
+	private static final MessageDigest DIGEST;
+	static {
+		try {
+			DIGEST = MessageDigest.getInstance("md5");
+		} catch (NoSuchAlgorithmException e) {
+			throw new RuntimeException("Could not create an MD5 hash digest");
+		}
+	}
+	
+	@Inject
+	Jsonb jsonb;
+	@Inject
+	CachingService<?> cachingService;
+
+	/**
+	 * Builds a response using passed data. Uses references to the caching service
+	 * and the current request to add information about ETags and Cache-Control
+	 * headers.
+	 * 
+	 * @param id      the ID of the object to be stored in cache
+	 * @param wrapper the query parameters for the current request
+	 * @param data    the data to attach to the response
+	 * @return a complete response object for the given data and request.
+	 */
+	public Response build(String id, RequestWrapper wrapper, Object data) {
+		// set default cache control flags for API responses
+		CacheControl cc = new CacheControl();
+		cc.setNoStore(wrapper.isCacheBypass());
+
+		if (!cc.isNoStore()) {
+			cc.setMaxAge((int) cachingService.getMaxAge());
+			// get the TTL for the current entry
+			Optional<Long> ttl = cachingService.getExpiration(id, wrapper);
+			if (!ttl.isPresent()) {
+				return Response.serverError().build();
+			}
+
+			// serialize the data to get an etag
+			String content = jsonb.toJson(Objects.requireNonNull(data));
+			// ingest the content and hash to create an etag for current content
+			String hash;
+			synchronized (this) {
+				DIGEST.update(content.getBytes(StandardCharsets.UTF_8));
+				hash = DatatypeConverter.printHexBinary(DIGEST.digest());
+				DIGEST.reset();
+			}
+			
+			// check if etag matches
+			String etag = wrapper.getHeader("Etag");
+			if (hash.equals(etag)) {
+				return Response.notModified(etag).cacheControl(cc).expires(new Date(ttl.get())).build();
+			}
+			// return a response w/ the generated etag
+			return Response.ok(data).tag(hash).cacheControl(cc).expires(new Date(ttl.get())).build();
+		}
+		return Response.ok(data).cacheControl(cc).build();
+	}
+}

src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java

@@ -17,7 +17,6 @@ import org.eclipsefoundation.marketplace.dto.filter.DtoFilter;
 import org.eclipsefoundation.marketplace.helper.SortableHelper;
 import org.eclipsefoundation.marketplace.helper.SortableHelper.Sortable;
 import org.eclipsefoundation.marketplace.namespace.UrlParameterNames;
-import org.eclipsefoundation.marketplace.service.CachingService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,7 +34,6 @@ import com.mongodb.client.model.Filters;
 public class MongoQuery<T> {
 	private static final Logger LOGGER = LoggerFactory.getLogger(MongoQuery.class);
 
-	private CachingService<List<T>> cache;
 	private RequestWrapper wrapper;
 	private DtoFilter<T> dtoFilter;
 
@@ -44,10 +42,9 @@ public class MongoQuery<T> {
 	private SortOrder order;
 	private List<Bson> aggregates;
 
-	public MongoQuery(RequestWrapper wrapper, DtoFilter<T> dtoFilter, CachingService<List<T>> cache) {
+	public MongoQuery(RequestWrapper wrapper, DtoFilter<T> dtoFilter) {
 		this.wrapper = wrapper;
 		this.dtoFilter = dtoFilter;
-		this.cache = cache;
 		this.aggregates = new ArrayList<>();
 		init();
 	}
@@ -72,12 +69,13 @@ public class MongoQuery<T> {
 		Optional<String> sortOpt = wrapper.getFirstParam(UrlParameterNames.SORT);
 		if (sortOpt.isPresent()) {
 			String sortVal = sortOpt.get();
+			SortOrder ord = SortOrder.getOrderFromValue(sortOpt.get());
 			// split sort string of `<fieldName> <SortOrder>`
 			int idx = sortVal.indexOf(' ');
 			// check if the sort string matches the RANDOM sort order
-			if (SortOrder.RANDOM.equals(SortOrder.getOrderByName(sortVal))) {
+			if (SortOrder.RANDOM.equals(ord)) {
 				this.order = SortOrder.RANDOM;
-			} else if (idx > 0) {
+			} else if (ord != SortOrder.NONE) {
 				setSort(sortVal.substring(0, idx), sortVal.substring(idx + 1), filters);
 			}
 		}
@@ -143,13 +141,6 @@ public class MongoQuery<T> {
 			this.order = SortOrder.getOrderByName(sortOrder);
 			// add sorting query if the sortOrder matches a defined order
 			switch (order) {
-			case RANDOM:
-				// TODO support for random, implement the following (in this order)
-				// 1. Add not in clause that checks Cache for previously read objects
-				// 2. Set useAggregate flag to true to signal to DAO to use aggregate selection
-				// rather than traditional find
-
-				break;
 			case ASCENDING:
 				// if last seen is set, add a filter to shift the results
 				if (lastOpt.isPresent()) {

src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java

@@ -23,6 +23,7 @@ import javax.ws.rs.core.UriInfo;
 import org.apache.commons.lang3.StringUtils;
 import org.eclipsefoundation.marketplace.namespace.DeprecatedHeader;
 import org.eclipsefoundation.marketplace.namespace.RequestHeaderNames;
+import org.eclipsefoundation.marketplace.request.CacheBypassFilter;
 import org.jboss.resteasy.core.ResteasyContext;
 
 /**
@@ -168,7 +169,18 @@ public class RequestWrapper {
 	}
 
 	/**
-	 * Retrieve a request header value as an optional value.
+	 * Check whether the current request should bypass caching
+	 * 
+	 * @return true if cache should be bypassed, otherwise false
+	 */
+	public boolean isCacheBypass() {
+		Object attr = request.getAttribute(CacheBypassFilter.ATTRIBUTE_NAME);
+		// if we have the attribute set on the request, return it. otherwise, false.
+		return attr instanceof Boolean ? (boolean) attr : Boolean.FALSE;
+	}
+
+	/**
+	 * Retrieve a request header value.
 	 * 
 	 * @param key the headers key value
 	 * @return the value, or an empty optional if missing.

src/main/java/org/eclipsefoundation/marketplace/model/SortOrder.java

@@ -53,4 +53,24 @@ public enum SortOrder {
 		}
 		return NONE;
 	}
+
+	/**
+	 * Gets the SortOrder value associated with a sort parameter value if one
+	 * exists.
+	 * 
+	 * @param value the value of the sort parameter
+	 * @return the SortOrder associated with the request, or
+	 *         {@linkplain SortOrder.NONE}
+	 */
+	public static SortOrder getOrderFromValue(String value) {
+		// get the index of the space separator 
+		int idx = value.indexOf(' ');
+		// check if the sort string matches the RANDOM sort order
+		if (SortOrder.RANDOM.equals(SortOrder.getOrderByName(value))) {
+			return SortOrder.RANDOM;
+		} else if (idx > 0) {
+			return SortOrder.getOrderByName(value.substring(idx + 1));
+		}
+		return SortOrder.NONE;
+	}
 }
\ No newline at end of file

src/main/java/org/eclipsefoundation/marketplace/request/CacheBypassFilter.java

+/* Copyright (c) 2019 Eclipse Foundation and others.
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Public License 2.0
+ * which is available at http://www.eclipse.org/legal/epl-v20.html,
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.marketplace.request;
+
+import java.io.IOException;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.ext.Provider;
+
+import org.eclipsefoundation.marketplace.model.SortOrder;
+import org.eclipsefoundation.marketplace.namespace.UrlParameterNames;
+
+/**
+ * Checks passed parameters and if any match one of the criteria for bypassing
+ * caching, an attribute will be set to the request to skip cache requests and
+ * instead directly return results.
+ * 
+ * @author Martin Lowe
+ *
+ */
+@Provider
+public class CacheBypassFilter implements ContainerRequestFilter {
+	public static final String ATTRIBUTE_NAME = "bypass-cache";
+
+	@Context
+	HttpServletRequest request;
+
+	@Context
+	HttpServletResponse response;
+
+	@Override
+	public void filter(ContainerRequestContext requestContext) throws IOException {
+		// check for random sort order, which always bypasses cache
+		String[] sortVals = request.getParameterValues(UrlParameterNames.SORT);
+		if (sortVals != null) {
+			for (String sortVal : sortVals) {
+				// check if the sort order for request matches RANDOM
+				if (SortOrder.RANDOM.equals(SortOrder.getOrderFromValue(sortVal))) {
+					setBypass();
+					return;
+				}
+			}
+		}
+		request.setAttribute(ATTRIBUTE_NAME, Boolean.FALSE);
+	}
+
+	private void setBypass() {
+		request.setAttribute(ATTRIBUTE_NAME, Boolean.TRUE);
+		// no-store should be used as cache bypass should not return
+		response.setHeader("Cache-Control", "no-store");
+	}
+}

src/main/java/org/eclipsefoundation/marketplace/resource/CatalogResource.java

@@ -27,6 +27,7 @@ import javax.ws.rs.core.Response.Status;
 import org.eclipsefoundation.marketplace.dao.MongoDao;
 import org.eclipsefoundation.marketplace.dto.Catalog;
 import org.eclipsefoundation.marketplace.dto.filter.DtoFilter;
+import org.eclipsefoundation.marketplace.helper.ResponseHelper;
 import org.eclipsefoundation.marketplace.helper.StreamHelper;
 import org.eclipsefoundation.marketplace.model.Error;
 import org.eclipsefoundation.marketplace.model.MongoQuery;
@@ -58,11 +59,13 @@ public class CatalogResource {
 	RequestWrapper params;
 	@Inject
 	DtoFilter<Catalog> dtoFilter;
+	@Inject
+	ResponseHelper responseBuider;
 
 	@GET
 	@PermitAll
 	public Response select() {
-		MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter);
 		// retrieve the possible cached object
 		Optional<List<Catalog>> cachedResults = cachingService.get("all", params,
 				() -> StreamHelper.awaitCompletionStage(dao.get(q)));
@@ -72,7 +75,7 @@ public class CatalogResource {
 		}
 
 		// return the results as a response
-		return Response.ok(cachedResults.get()).build();
+		return responseBuider.build("all", params, cachedResults.get());
 	}
 
 	/**
@@ -84,7 +87,7 @@ public class CatalogResource {
 	@PUT
 	@RolesAllowed({ "marketplace_catalog_put", "marketplace_admin_access" })
 	public Response putCatalog(Catalog catalog) {
-		MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter);
 		// add the object, and await the result
 		StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(catalog)));
 
@@ -104,7 +107,7 @@ public class CatalogResource {
 	public Response select(@PathParam("catalogId") String catalogId) {
 		params.addParam(UrlParameterNames.ID, catalogId);
 
-		MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter);
 		// retrieve a cached version of the value for the current listing
 		Optional<List<Catalog>> cachedResults = cachingService.get(catalogId, params,
 				() -> StreamHelper.awaitCompletionStage(dao.get(q)));
@@ -114,7 +117,7 @@ public class CatalogResource {
 		}
 
 		// return the results as a response
-		return Response.ok(cachedResults.get()).build();
+		return responseBuider.build(catalogId, params, cachedResults.get());
 	}
 
 	/**
@@ -130,7 +133,7 @@ public class CatalogResource {
 	public Response delete(@PathParam("catalogId") String catalogId) {
 		params.addParam(UrlParameterNames.ID, catalogId);
 
-		MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Catalog> q = new MongoQuery<>(params, dtoFilter);
 		// delete the currently selected asset
 		DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q));
 		if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) {

src/main/java/org/eclipsefoundation/marketplace/resource/CategoryResource.java

@@ -27,6 +27,7 @@ import javax.ws.rs.core.Response.Status;
 import org.eclipsefoundation.marketplace.dao.MongoDao;
 import org.eclipsefoundation.marketplace.dto.Category;
 import org.eclipsefoundation.marketplace.dto.filter.DtoFilter;
+import org.eclipsefoundation.marketplace.helper.ResponseHelper;
 import org.eclipsefoundation.marketplace.helper.StreamHelper;
 import org.eclipsefoundation.marketplace.model.Error;
 import org.eclipsefoundation.marketplace.model.MongoQuery;
@@ -58,11 +59,13 @@ public class CategoryResource {
 	RequestWrapper params;
 	@Inject
 	DtoFilter<Category> dtoFilter;
+	@Inject
+	ResponseHelper responseBuider;
 
 	@GET
 	@PermitAll
 	public Response select() {
-		MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter);
 		// retrieve the possible cached object
 		Optional<List<Category>> cachedResults = cachingService.get("all", params,
 				() -> StreamHelper.awaitCompletionStage(dao.get(q)));
@@ -72,7 +75,7 @@ public class CategoryResource {
 		}
 
 		// return the results as a response
-		return Response.ok(cachedResults.get()).build();
+		return responseBuider.build("all", params, cachedResults.get());
 	}
 
 	/**
@@ -84,7 +87,7 @@ public class CategoryResource {
 	@PUT
 	@RolesAllowed({"marketplace_category_put", "marketplace_admin_access"})
 	public Response putCategory(Category category) {
-		MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter);
 		// add the object, and await the result
 		StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(category)));
 
@@ -104,7 +107,7 @@ public class CategoryResource {
 	public Response select(@PathParam("categoryId") String categoryId) {
 		params.addParam(UrlParameterNames.ID, categoryId);
 
-		MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter);
 		// retrieve a cached version of the value for the current listing
 		Optional<List<Category>> cachedResults = cachingService.get(categoryId, params,
 				() -> StreamHelper.awaitCompletionStage(dao.get(q)));
@@ -114,7 +117,7 @@ public class CategoryResource {
 		}
 
 		// return the results as a response
-		return Response.ok(cachedResults.get()).build();
+		return responseBuider.build(categoryId, params, cachedResults.get());
 	}
 
 	/**
@@ -130,7 +133,7 @@ public class CategoryResource {
 	public Response delete(@PathParam("categoryId") String categoryId) {
 		params.addParam(UrlParameterNames.ID, categoryId);
 
-		MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Category> q = new MongoQuery<>(params, dtoFilter);
 		// delete the currently selected asset
 		DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q));
 		if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) {

src/main/java/org/eclipsefoundation/marketplace/resource/ErrorReportResource.java

@@ -25,6 +25,7 @@ import javax.ws.rs.core.Response;
 import org.eclipsefoundation.marketplace.dao.MongoDao;
 import org.eclipsefoundation.marketplace.dto.ErrorReport;
 import org.eclipsefoundation.marketplace.dto.filter.DtoFilter;
+import org.eclipsefoundation.marketplace.helper.ResponseHelper;
 import org.eclipsefoundation.marketplace.helper.StreamHelper;
 import org.eclipsefoundation.marketplace.model.MongoQuery;
 import org.eclipsefoundation.marketplace.model.RequestWrapper;
@@ -54,6 +55,8 @@ public class ErrorReportResource {
 	RequestWrapper params;
 	@Inject
 	DtoFilter<ErrorReport> dtoFilter;
+	@Inject
+	ResponseHelper responseBuider;
 
 	/**
 	 * Endpoint for /error/ to retrieve all ErrorReports from the database along with
@@ -65,7 +68,7 @@ public class ErrorReportResource {
 	@GET
 	@PermitAll
 	public Response select() {
-		MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter);
 		// retrieve the possible cached object
 		Optional<List<ErrorReport>> cachedResults = cachingService.get("all", params,
 				() -> StreamHelper.awaitCompletionStage(dao.get(q)));
@@ -75,7 +78,7 @@ public class ErrorReportResource {
 		}
 
 		// return the results as a response
-		return Response.ok(cachedResults.get()).build();
+		return responseBuider.build("all", params, cachedResults.get());
 	}
 
 	/**
@@ -87,7 +90,7 @@ public class ErrorReportResource {
 	@PUT
 	@RolesAllowed("error_put")
 	public Response putErrorReport(ErrorReport errorReport) {
-		MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter);
 
 		// add the object, and await the result
 		StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(errorReport)));
@@ -109,7 +112,7 @@ public class ErrorReportResource {
 	public Response select(@PathParam("errorReportId") String errorReportId) {
 		params.addParam(UrlParameterNames.ID, errorReportId);
 
-		MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<ErrorReport> q = new MongoQuery<>(params, dtoFilter);
 		// retrieve a cached version of the value for the current ErrorReport
 		Optional<List<ErrorReport>> cachedResults = cachingService.get(errorReportId, params,
 				() -> StreamHelper.awaitCompletionStage(dao.get(q)));
@@ -119,6 +122,6 @@ public class ErrorReportResource {
 		}
 
 		// return the results as a response
-		return Response.ok(cachedResults.get()).build();
+		return responseBuider.build(errorReportId, params, cachedResults.get());
 	}
 }

src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java

@@ -37,6 +37,7 @@ import org.eclipsefoundation.marketplace.dto.InstallMetrics;
 import org.eclipsefoundation.marketplace.dto.MetricPeriod;
 import org.eclipsefoundation.marketplace.dto.filter.DtoFilter;
 import org.eclipsefoundation.marketplace.helper.DateTimeHelper;
+import org.eclipsefoundation.marketplace.helper.ResponseHelper;
 import org.eclipsefoundation.marketplace.helper.StreamHelper;
 import org.eclipsefoundation.marketplace.model.Error;
 import org.eclipsefoundation.marketplace.model.MongoQuery;
@@ -63,6 +64,8 @@ public class InstallResource {
 	MongoDao dao;
 	@Inject
 	RequestWrapper wrapper;
+	@Inject
+	ResponseHelper responseBuider;
 
 	// insert required filters for different objects + states
 	@Inject
@@ -90,7 +93,7 @@ public class InstallResource {
 	@Path("/{listingId}")
 	public Response selectInstallCount(@PathParam("listingId") String listingId) {
 		wrapper.addParam(UrlParameterNames.ID, listingId);
-		MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, null);
+		MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter);
 		Optional<Long> cachedResults = countCache.get(listingId, wrapper,
 				() -> StreamHelper.awaitCompletionStage(dao.count(q)));
 		if (!cachedResults.isPresent()) {
@@ -99,7 +102,7 @@ public class InstallResource {
 		}
 
 		// return the results as a response
-		return Response.ok(cachedResults.get()).build();
+		return responseBuider.build(listingId, wrapper, cachedResults.get());
 	}
 
 	/**
@@ -117,7 +120,7 @@ public class InstallResource {
 	public Response selectInstallCount(@PathParam("listingId") String listingId, @PathParam("version") String version) {
 		wrapper.addParam(UrlParameterNames.ID, listingId);
 		wrapper.addParam(UrlParameterNames.VERSION, version);
-		MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, null);
+		MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter);
 		Optional<Long> cachedResults = countCache.get(getCompositeKey(listingId, version), wrapper,
 				() -> StreamHelper.awaitCompletionStage(dao.count(q)));
 		if (!cachedResults.isPresent()) {
@@ -126,7 +129,7 @@ public class InstallResource {
 		}
 
 		// return the results as a response
-		return Response.ok(cachedResults.get()).build();
+		return responseBuider.build(getCompositeKey(listingId, version), wrapper, cachedResults.get());
 	}
 
 	/**
@@ -141,7 +144,7 @@ public class InstallResource {
 	@Path("/{listingId}/metrics")
 	public Response selectInstallMetrics(@PathParam("listingId") String listingId) {
 		wrapper.addParam(UrlParameterNames.ID, listingId);
-		MongoQuery<InstallMetrics> q = new MongoQuery<>(wrapper, metricFilter, null);
+		MongoQuery<InstallMetrics> q = new MongoQuery<>(wrapper, metricFilter);
 		Optional<List<InstallMetrics>> cachedResults = installCache.get(listingId, wrapper,
 				() -> StreamHelper.awaitCompletionStage(dao.get(q)));
 		if (!cachedResults.isPresent()) {
@@ -188,7 +191,7 @@ public class InstallResource {
 		record.setVersion(version);
 
 		// create the query wrapper to pass to DB dao
-		MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, null);
+		MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter);
 
 		// add the object, and await the result
 		StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(record)));
@@ -215,7 +218,7 @@ public class InstallResource {
 		List<CompletionStage<List<MetricPeriod>>> stages = new ArrayList<>();
 		// get total install count for all listings available
 		Map<String, Integer> overallCounts = new HashMap<>();
-		CompletionStage<List<MetricPeriod>> stage = dao.get(new MongoQuery<>(wrapper, periodFilter, null));
+		CompletionStage<List<MetricPeriod>> stage = dao.get(new MongoQuery<>(wrapper, periodFilter));
 		stage.whenComplete((metrics, e) -> {
 			// if theres an error, immediately stop processing
 			if (e != null) {
@@ -242,7 +245,7 @@ public class InstallResource {
 
 			// create the query wrapper to pass to DB dao. No cache needed as this info
 			// won't be cached
-			MongoQuery<MetricPeriod> q = new MongoQuery<>(wrapper, periodFilter, null);
+			MongoQuery<MetricPeriod> q = new MongoQuery<>(wrapper, periodFilter);
 			// run query, and set up a completion activity to record data
 			CompletionStage<List<MetricPeriod>> statStage = dao.get(q);
 			statStage.whenComplete((metrics, e) -> {
@@ -266,7 +269,7 @@ public class InstallResource {
 				entry.getValue(), overallCounts.getOrDefault(entry.getKey(), 0))).collect(Collectors.toList());
 
 		// push the content to the database, and await for it to finish
-		StreamHelper.awaitCompletionStage(dao.add(new MongoQuery<>(wrapper, metricFilter, null), installMetrics));
+		StreamHelper.awaitCompletionStage(dao.add(new MongoQuery<>(wrapper, metricFilter), installMetrics));
 		// return the results as a response
 		return Response.ok().build();
 	}

src/main/java/org/eclipsefoundation/marketplace/resource/ListingResource.java

@@ -30,6 +30,7 @@ import javax.ws.rs.core.Response.Status;
 import org.eclipsefoundation.marketplace.dao.MongoDao;
 import org.eclipsefoundation.marketplace.dto.Listing;
 import org.eclipsefoundation.marketplace.dto.filter.DtoFilter;
+import org.eclipsefoundation.marketplace.helper.ResponseHelper;
 import org.eclipsefoundation.marketplace.helper.StreamHelper;
 import org.eclipsefoundation.marketplace.model.Error;
 import org.eclipsefoundation.marketplace.model.MongoQuery;
@@ -62,6 +63,8 @@ public class ListingResource {
 	RequestWrapper params;
 	@Inject
 	DtoFilter<Listing> dtoFilter;
+	@Inject
+	ResponseHelper responseBuider;
 
 	/**
 	 * Endpoint for /listing/ to retrieve all listings from the database along with
@@ -73,7 +76,7 @@ public class ListingResource {
 	@GET
 	@PermitAll
 	public Response select() {
-		MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter);
 		// retrieve the possible cached object
 		Optional<List<Listing>> cachedResults = cachingService.get("all", params,
 				() -> StreamHelper.awaitCompletionStage(dao.get(q)));
@@ -83,7 +86,7 @@ public class ListingResource {
 		}
 
 		// return the results as a response
-		return Response.ok(cachedResults.get()).build();
+		return responseBuider.build("all", params, cachedResults.get());
 	}
 
 	/**
@@ -95,7 +98,7 @@ public class ListingResource {
 	@PUT
 	@RolesAllowed({ "marketplace_listing_put", "marketplace_admin_access" })
 	public Response putListing(Listing listing) {
-		MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter);
 
 		// add the object, and await the result
 		StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(listing)));
@@ -117,7 +120,7 @@ public class ListingResource {
 	public Response select(@PathParam("listingId") String listingId) {
 		params.addParam(UrlParameterNames.ID, listingId);
 
-		MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter);
 		// retrieve a cached version of the value for the current listing
 		Optional<List<Listing>> cachedResults = cachingService.get(listingId, params,
 				() -> StreamHelper.awaitCompletionStage(dao.get(q)));
@@ -127,7 +130,7 @@ public class ListingResource {
 		}
 
 		// return the results as a response
-		return Response.ok(cachedResults.get()).build();
+		return responseBuider.build(listingId, params, cachedResults.get());
 	}
 
 	/**
@@ -142,7 +145,7 @@ public class ListingResource {
 	@Path("/{listingId}")
 	public Response delete(@PathParam("listingId") String listingId) {
 		params.addParam(UrlParameterNames.ID, listingId);
-		MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Listing> q = new MongoQuery<>(params, dtoFilter);
 		// delete the currently selected asset
 		DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q));
 		if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) {

src/main/java/org/eclipsefoundation/marketplace/resource/ListingVersionResource.java

@@ -25,6 +25,7 @@ import javax.ws.rs.core.Response.Status;
 import org.eclipsefoundation.marketplace.dao.MongoDao;
 import org.eclipsefoundation.marketplace.dto.ListingVersion;
 import org.eclipsefoundation.marketplace.dto.filter.DtoFilter;
+import org.eclipsefoundation.marketplace.helper.ResponseHelper;
 import org.eclipsefoundation.marketplace.helper.StreamHelper;
 import org.eclipsefoundation.marketplace.model.Error;
 import org.eclipsefoundation.marketplace.model.MongoQuery;
@@ -58,10 +59,12 @@ public class ListingVersionResource {
 	RequestWrapper params;
 	@Inject
 	DtoFilter<ListingVersion> dtoFilter;
+	@Inject
+	ResponseHelper responseBuider;
 
 	@GET
 	public Response select() {
-		MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter);
 		// retrieve the possible cached object
 		Optional<List<ListingVersion>> cachedResults = cachingService.get("all", params,
 				() -> StreamHelper.awaitCompletionStage(dao.get(q)));
@@ -82,7 +85,7 @@ public class ListingVersionResource {
 	 */
 	@PUT
 	public Response putListingVersion(ListingVersion listingVersion) {
-		MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter);
 		// add the object, and await the result
 		StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(listingVersion)));
 
@@ -102,7 +105,7 @@ public class ListingVersionResource {
 	public Response select(@PathParam("listingVersionId") String listingVersionId) {
 		params.addParam(UrlParameterNames.ID, listingVersionId);
 
-		MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter);
 		// retrieve a cached version of the value for the current listing
 		Optional<List<ListingVersion>> cachedResults = cachingService.get(listingVersionId, params,
 				() -> StreamHelper.awaitCompletionStage(dao.get(q)));
@@ -127,7 +130,7 @@ public class ListingVersionResource {
 	public Response delete(@PathParam("listingVersionId") String listingVersionId) {
 		params.addParam(UrlParameterNames.ID, listingVersionId);
 
-		MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<ListingVersion> q = new MongoQuery<>(params, dtoFilter);
 		// delete the currently selected asset
 		DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q));
 		if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) {

src/main/java/org/eclipsefoundation/marketplace/resource/MarketResource.java

@@ -27,6 +27,7 @@ import javax.ws.rs.core.Response.Status;
 import org.eclipsefoundation.marketplace.dao.MongoDao;
 import org.eclipsefoundation.marketplace.dto.Market;
 import org.eclipsefoundation.marketplace.dto.filter.DtoFilter;
+import org.eclipsefoundation.marketplace.helper.ResponseHelper;
 import org.eclipsefoundation.marketplace.helper.StreamHelper;
 import org.eclipsefoundation.marketplace.model.Error;
 import org.eclipsefoundation.marketplace.model.MongoQuery;
@@ -58,12 +59,13 @@ public class MarketResource {
 	RequestWrapper params;
 	@Inject
 	DtoFilter<Market> dtoFilter;
+	@Inject
+	ResponseHelper responseBuider;
 
-	
 	@GET
 	@PermitAll
 	public Response select() {
-		MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter);
 		// retrieve the possible cached object
 		Optional<List<Market>> cachedResults = cachingService.get("all", params,
 				() -> StreamHelper.awaitCompletionStage(dao.get(q)));
@@ -73,7 +75,7 @@ public class MarketResource {
 		}
 
 		// return the results as a response
-		return Response.ok(cachedResults.get()).build();
+		return responseBuider.build("all", params, cachedResults.get());
 	}
 
 	/**
@@ -85,7 +87,7 @@ public class MarketResource {
 	@PUT
 	@RolesAllowed("market_put")
 	public Response putMarket(Market market) {
-		MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter);
 
 		// add the object, and await the result
 		StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(market)));
@@ -107,7 +109,7 @@ public class MarketResource {
 	public Response select(@PathParam("marketId") String marketId) {
 		params.addParam(UrlParameterNames.ID, marketId);
 
-		MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter);
 		// retrieve a cached version of the value for the current listing
 		Optional<List<Market>> cachedResults = cachingService.get(marketId, params,
 				() -> StreamHelper.awaitCompletionStage(dao.get(q)));
@@ -117,7 +119,7 @@ public class MarketResource {
 		}
 
 		// return the results as a response
-		return Response.ok(cachedResults.get()).build();
+		return responseBuider.build(marketId, params, cachedResults.get());
 	}
 
 	/**
@@ -132,7 +134,7 @@ public class MarketResource {
 	public Response delete(@PathParam("marketId") String marketId) {
 		params.addParam(UrlParameterNames.ID, marketId);
 
-		MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter, cachingService);
+		MongoQuery<Market> q = new MongoQuery<>(params, dtoFilter);
 		// delete the currently selected asset
 		DeleteResult result = StreamHelper.awaitCompletionStage(dao.delete(q));
 		if (result.getDeletedCount() == 0 || !result.wasAcknowledged()) {

src/main/java/org/eclipsefoundation/marketplace/service/CachingService.java

@@ -33,6 +33,21 @@ public interface CachingService<T> {
 	 */
 	Optional<T> get(String id, RequestWrapper params, Callable<? extends T> callable);
 
+	/**
+	 * Returns the expiration date in millis since epoch.
+	 * 
+	 * @param id     the ID of the object to be stored in cache
+	 * @param params the query parameters for the current request
+	 * @return an Optional expiration date for the current object if its set. If
+	 *         there is no underlying data, then empty would be returned
+	 */
+	Optional<Long> getExpiration(String id, RequestWrapper params);
+
+	/**
+	 * @return the max age of cache entries
+	 */
+	long getMaxAge();
+
 	/**
 	 * Retrieves a set of cache keys available to the current cache.
 	 * 
@@ -56,18 +71,18 @@ public interface CachingService<T> {
 	 * Generates a unique key based on the id of the item/set of items to be stored,
 	 * as well as any passed parameters.
 	 * 
-	 * @param id  identity string of the item to cache
-	 * @param qps parameters associated with the request for information
+	 * @param id      identity string of the item to cache
+	 * @param wrapper parameters associated with the request for information
 	 * @return the unique cache key for the request.
 	 */
-	default String getCacheKey(String id, RequestWrapper qps) {
+	default String getCacheKey(String id, RequestWrapper wrapper) {
 		StringBuilder sb = new StringBuilder();
-		sb.append('[').append(qps.getEndpoint()).append(']');
+		sb.append('[').append(wrapper.getEndpoint()).append(']');
 		sb.append("id:").append(id);
 
 		// join all the non-empty params to the key to create distinct entries for
 		// filtered values
-		qps.asMap().entrySet().stream().filter(e -> !e.getValue().isEmpty())
+		wrapper.asMap().entrySet().stream().filter(e -> !e.getValue().isEmpty())
 				.map(e -> e.getKey() + '=' + StringUtils.join(e.getValue(), ','))
 				.forEach(s -> sb.append('|').append(s));
 

src/main/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingService.java

@@ -6,11 +6,12 @@
  */
 package org.eclipsefoundation.marketplace.service.impl;
 
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
 import javax.annotation.PostConstruct;
@@ -47,24 +48,26 @@ import com.google.common.util.concurrent.UncheckedExecutionException;
 public class GuavaCachingService<T> implements CachingService<T> {
 	private static final Logger LOGGER = LoggerFactory.getLogger(GuavaCachingService.class);
 
-	@ConfigProperty(name = "cache.max", defaultValue = "2500")
+	@ConfigProperty(name = "cache.max.size", defaultValue = "10000")
 	long maxSize;
-	@ConfigProperty(name = "cache.ttl.access", defaultValue = "21600")
-	long ttlAccess;
-	@ConfigProperty(name = "cache.ttl.write", defaultValue = "86400")
+	@ConfigProperty(name = "cache.ttl.write.seconds", defaultValue = "900")
 	long ttlWrite;
 
 	// actual cache object
 	Cache<String, T> cache = null;
+	Map<String, Long> ttl;
 
 	@PostConstruct
 	public void init() {
+		this.ttl = new HashMap<>();
+		// create cache with configured settings that maintains a TTL map
 		cache = CacheBuilder
-				.newBuilder()
-				.maximumSize(maxSize)
-				.expireAfterAccess(ttlAccess, TimeUnit.SECONDS)
-				.expireAfterWrite(ttlWrite, TimeUnit.SECONDS)
-				.build();
+					.newBuilder()
+					.maximumSize(maxSize)
+					.expireAfterWrite(ttlWrite, TimeUnit.SECONDS)
+					.removalListener(not -> ttl.remove(not.getKey()))
+					.build();
+
 	}
 
 	@Override
@@ -72,18 +75,38 @@ public class GuavaCachingService<T> implements CachingService<T> {
 		Objects.requireNonNull(id);
 		Objects.requireNonNull(params);
 		Objects.requireNonNull(callable);
-		
-		String cacheKey = getCacheKey(id, params);		
+
+		String cacheKey = getCacheKey(id, params);
 		try {
+			// check if the cache is bypassed for the request
+			if (params.isCacheBypass()) {
+				T result = callable.call();
+				// if the cache has a value for key, update it
+				if (cache.asMap().containsKey(cacheKey)) {
+					cache.put(cacheKey, result);
+				}
+				return Optional.of(result);
+			}
+			
+			// get entry, and enter a ttl as soon as it returns
+			T data = cache.get(cacheKey, callable);
+			if (data != null) {
+				ttl.putIfAbsent(cacheKey, System.currentTimeMillis() + TimeUnit.MILLISECONDS.convert(ttlWrite, TimeUnit.SECONDS));
+			}
 			return Optional.of(cache.get(cacheKey, callable));
-		} catch (ExecutionException e) {
-			LOGGER.error("Error while retrieving value of callback", e);
 		} catch (InvalidCacheLoadException | UncheckedExecutionException e) {
 			LOGGER.error("Error while retrieving fresh value for cachekey: {}", cacheKey, e);
+		} catch (Exception e) {
+			LOGGER.error("Error while retrieving value of callback", e);
 		}
 		return Optional.empty();
 	}
 
+	@Override
+	public Optional<Long> getExpiration(String id, RequestWrapper params) {
+		return Optional.ofNullable(ttl.get(getCacheKey(Objects.requireNonNull(id), Objects.requireNonNull(params))));
+	}
+	
 	@Override
 	public Set<String> getCacheKeys() {
 		return cache.asMap().keySet();
@@ -99,4 +122,8 @@ public class GuavaCachingService<T> implements CachingService<T> {
 		cache.invalidateAll();
 	}
 
+	@Override
+	public long getMaxAge() {
+		return ttlWrite;
+	}
 }

src/test/java/org/eclipsefoundation/marketplace/service/impl/GuavaCachingServiceTest.java

@@ -40,10 +40,10 @@ public class GuavaCachingServiceTest {
 	@BeforeEach
 	public void pre() {
 		// inject empty objects into the Request context before creating a mock object
-		ResteasyContext.pushContext(UriInfo.class, new ResteasyUriInfo("",""));
-		
+		ResteasyContext.pushContext(UriInfo.class, new ResteasyUriInfo("", ""));
+
 		ResteasyContext.pushContext(HttpServletRequest.class, new HttpServletRequestImpl(null, null));
-		
+
 		this.sample = new RequestWrapperMock();
 		// expire all active key values
 		gcs.removeAll();
@@ -60,18 +60,19 @@ public class GuavaCachingServiceTest {
 
 		// without post construct init via javax management, cache will not be properly
 		// set
-		Assertions.assertThrows(NullPointerException.class, () -> {
-			gcsManual.get("sampleKey", sample, Object::new);
-		});
+		Assertions.assertTrue(!gcsManual.get("sampleKey", sample, Object::new).isPresent(),
+				"Object should not be generated when there is no cache initialized");
 
 		// initialize the cache w/ configs
 		gcsManual.init();
 
 		// run a command to interact with cache
-		gcsManual.get("sampleKey", sample, Object::new);
+		Assertions.assertTrue(gcsManual.get("sampleKey", sample, Object::new).isPresent(),
+				"Object should be generated once cache is instantiated");
 
 		// test the injected cache service (which is the normal use case)
-		gcs.get("sampleKey", sample, Object::new);
+		Assertions.assertTrue(gcs.get("sampleKey", sample, Object::new).isPresent(),
+				"Object should be generated once cache is instantiated");
 	}
 
 	@Test
@@ -98,7 +99,7 @@ public class GuavaCachingServiceTest {
 		Optional<Object> emptyObj = gcs.get("failure key", sample, () -> null);
 		Assertions.assertFalse(emptyObj.isPresent());
 	}
-	
+
 	@Test
 	public void testGetExceptionalCallable() {
 		Optional<Object> emptyObj = gcs.get("k", sample, () -> {

src/test/resources/application.properties

 ## OAUTH CONFIG
 quarkus.oauth2.enabled=true
-quarkus.oauth2.introspection-url=https://accounts.php56.dev.docker/oauth2/introspect
+quarkus.oauth2.introspection-url=http://accounts.php56.dev.docker/oauth2/introspect
 
 ## LOGGER CONFIG
 quarkus.log.file.enable=true
@@ -9,7 +9,7 @@ quarkus.log.level=TRACE
 quarkus.log.file.path=/tmp/logs/quarkus.log
 
 ## DATASOURCE CONFIG
-quarkus.mongodb.connection-string=mongodb://localhost:27017
+quarkus.mongodb.connection-string=mongodb://192.168.1.178:27017
 mongodb.database=mpc
 mongodb.default.limit=25
 mongodb.default.limit.max=100