Skip to content
Snippets Groups Projects
Unverified Commit c5db9019 authored by Martin Lowe's avatar Martin Lowe :flag_ca: Committed by GitHub
Browse files

Merge pull request #23 from autumnfound/malowe/master/11

Install statistics endpoint #11
parents 5653a7c3 9543ecd0
No related branches found
No related tags found
No related merge requests found
Showing
with 460 additions and 66 deletions
......@@ -6,10 +6,7 @@
*/
package org.eclipsefoundation.marketplace.dto;
import java.sql.Date;
import org.eclipsefoundation.marketplace.model.RequestWrapper;
import org.eclipsefoundation.marketplace.namespace.UrlParameterNames;
import java.util.Date;
/**
* Domain object representing the data stored for installs.
......@@ -18,20 +15,29 @@ import org.eclipsefoundation.marketplace.namespace.UrlParameterNames;
*/
public class Install {
private String id;
private Date installDate;
private String os;
private String version;
private String listingId;
private String javaVersion;
private String eclipseVersion;
private String locale;
/**
* @return the id
*/
public String getId() {
return id;
}
public static Install createFromRequest(RequestWrapper wrap) {
Install install = new Install();
install.installDate = new Date(System.currentTimeMillis());
install.listingId = wrap.getFirstParam(UrlParameterNames.ID).get();
return install;
/**
* @param id the id to set
*/
public void setId(String id) {
this.id = id;
}
/**
* @return the installDate
*/
......@@ -101,4 +107,32 @@ public class Install {
public void setJavaVersion(String javaVersion) {
this.javaVersion = javaVersion;
}
/**
* @return the eclipseVersion
*/
public String getEclipseVersion() {
return eclipseVersion;
}
/**
* @param eclipseVersion the eclipseVersion to set
*/
public void setEclipseVersion(String eclipseVersion) {
this.eclipseVersion = eclipseVersion;
}
/**
* @return the locale
*/
public String getLocale() {
return locale;
}
/**
* @param locale the locale to set
*/
public void setLocale(String locale) {
this.locale = locale;
}
}
/* Copyright (c) 2019 Eclipse Foundation and others.
* This program and the accompanying materials are made available
* under the terms of the Eclipse Public License 2.0
* which is available at http://www.eclipse.org/legal/epl-v20.html,
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipsefoundation.marketplace.dto.codecs;
import java.util.UUID;
import org.bson.BsonReader;
import org.bson.BsonString;
import org.bson.BsonValue;
import org.bson.BsonWriter;
import org.bson.Document;
import org.bson.codecs.Codec;
import org.bson.codecs.CollectibleCodec;
import org.bson.codecs.DecoderContext;
import org.bson.codecs.EncoderContext;
import org.eclipsefoundation.marketplace.dto.Install;
import org.eclipsefoundation.marketplace.namespace.DatabaseFieldNames;
import com.mongodb.MongoClient;
/**
* MongoDB codec for transcoding of {@link Install} and {@link Document}
* objects. Used when writing or retrieving objects of given type from the
* database.
*
* @author Martin Lowe
*/
public class InstallCodec implements CollectibleCodec<Install> {
private final Codec<Document> documentCodec;
/**
* Creates the codec and initializes the codecs and converters needed to create
* an install from end to end.
*/
public InstallCodec() {
this.documentCodec = MongoClient.getDefaultCodecRegistry().get(Document.class);
}
@Override
public void encode(BsonWriter writer, Install value, EncoderContext encoderContext) {
Document doc = new Document();
doc.put(DatabaseFieldNames.DOCID, value.getId());
doc.put(DatabaseFieldNames.INSTALL_JAVA_VERSION, value.getJavaVersion());
doc.put(DatabaseFieldNames.INSTALL_VERSION, value.getVersion());
doc.put(DatabaseFieldNames.INSTALL_LISTING_ID, value.getListingId());
doc.put(DatabaseFieldNames.INSTALL_DATE, value.getInstallDate());
doc.put(DatabaseFieldNames.ECLIPSE_VERSION, value.getEclipseVersion());
doc.put(DatabaseFieldNames.OS, value.getOs());
doc.put(DatabaseFieldNames.LOCALE, value.getLocale());
documentCodec.encode(writer, doc, encoderContext);
}
@Override
public Class<Install> getEncoderClass() {
return Install.class;
}
@Override
public Install decode(BsonReader reader, DecoderContext decoderContext) {
Document document = documentCodec.decode(reader, decoderContext);
Install out = new Install();
out.setId(document.getString(DatabaseFieldNames.DOCID));
out.setJavaVersion(document.getString(DatabaseFieldNames.INSTALL_JAVA_VERSION));
out.setVersion(document.getString(DatabaseFieldNames.INSTALL_VERSION));
out.setListingId(document.getString(DatabaseFieldNames.INSTALL_LISTING_ID));
out.setInstallDate(document.getDate(DatabaseFieldNames.INSTALL_DATE));
out.setEclipseVersion(document.getString(DatabaseFieldNames.ECLIPSE_VERSION));
out.setLocale(document.getString(DatabaseFieldNames.LOCALE));
out.setOs(document.getString(DatabaseFieldNames.OS));
return out;
}
@Override
public Install generateIdIfAbsentFromDocument(Install document) {
if (!documentHasId(document)) {
document.setId(UUID.randomUUID().toString());
}
return document;
}
@Override
public boolean documentHasId(Install document) {
return document.getId() != null;
}
@Override
public BsonValue getDocumentId(Install document) {
return new BsonString(document.getId());
}
}
/* Copyright (c) 2019 Eclipse Foundation and others.
* This program and the accompanying materials are made available
* under the terms of the Eclipse Public License 2.0
* which is available at http://www.eclipse.org/legal/epl-v20.html,
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipsefoundation.marketplace.dto.filter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import javax.enterprise.context.ApplicationScoped;
import org.apache.commons.lang3.StringUtils;
import org.bson.conversions.Bson;
import org.eclipsefoundation.marketplace.dto.Install;
import org.eclipsefoundation.marketplace.model.RequestWrapper;
import org.eclipsefoundation.marketplace.namespace.DatabaseFieldNames;
import org.eclipsefoundation.marketplace.namespace.UrlParameterNames;
import com.mongodb.client.model.Filters;
/**
* Filter implementation for the {@linkplain Install} class.
*
* @author Martin Lowe
*/
@ApplicationScoped
public class InstallFilter implements DtoFilter<Install> {
@Override
public List<Bson> getFilters(RequestWrapper wrap) {
List<Bson> filters = new ArrayList<>();
// Listing ID check
Optional<String> id = wrap.getFirstParam(UrlParameterNames.ID);
if (id.isPresent()) {
filters.add(Filters.eq(DatabaseFieldNames.INSTALL_LISTING_ID, id.get()));
}
// version check
Optional<String> version = wrap.getFirstParam(UrlParameterNames.VERSION);
if (version.isPresent()) {
filters.add(Filters.eq(DatabaseFieldNames.INSTALL_VERSION, version.get()));
}
// OS filter
Optional<String> os = wrap.getFirstParam(UrlParameterNames.OS);
if (os.isPresent()) {
filters.add(Filters.eq(DatabaseFieldNames.OS, os.get()));
}
// eclipse version
Optional<String> eclipseVersion = wrap.getFirstParam(UrlParameterNames.ECLIPSE_VERSION);
if (eclipseVersion.isPresent()) {
filters.add(Filters.eq(DatabaseFieldNames.ECLIPSE_VERSION, eclipseVersion.get()));
}
// TODO this sorts by naturally by character rather than by actual number (e.g.
// 1.9 is technically greater than 1.10)
// solution version - Java version
Optional<String> javaVersion = wrap.getFirstParam(UrlParameterNames.JAVA_VERSION);
if (javaVersion.isPresent()) {
filters.add(Filters.gte(DatabaseFieldNames.INSTALL_JAVA_VERSION, javaVersion.get()));
}
Optional<String> date = wrap.getFirstParam(UrlParameterNames.DATE_FROM);
if (date.isPresent() && StringUtils.isNumeric(date.get())) {
filters.add(Filters.gte(DatabaseFieldNames.INSTALL_DATE, new Date(Integer.valueOf(date.get()))));
}
return filters;
}
@Override
public List<Bson> getAggregates(RequestWrapper wrap) {
return Collections.emptyList();
}
@Override
public Class<Install> getType() {
return Install.class;
}
}
......@@ -113,6 +113,7 @@ public class ListingFilter implements DtoFilter<Listing> {
if (!marketIds.isEmpty()) {
aggs.add(Aggregates.match(Filters.in("categories.market_ids", marketIds)));
}
return aggs;
}
......
/* Copyright (c) 2019 Eclipse Foundation and others.
* This program and the accompanying materials are made available
* under the terms of the Eclipse Public License 2.0
* which is available at http://www.eclipse.org/legal/epl-v20.html,
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipsefoundation.marketplace.dto.providers;
import org.bson.codecs.Codec;
import org.bson.codecs.configuration.CodecProvider;
import org.bson.codecs.configuration.CodecRegistry;
import org.eclipsefoundation.marketplace.dto.Catalog;
import org.eclipsefoundation.marketplace.dto.Install;
import org.eclipsefoundation.marketplace.dto.codecs.InstallCodec;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Provides the {@link InstallCodec} to MongoDB for conversions of
* {@link Catalog} objects.
*
* @author Martin Lowe
*/
public class InstallCodecProvider implements CodecProvider {
private static final Logger LOGGER = LoggerFactory.getLogger(InstallCodecProvider.class);
@SuppressWarnings("unchecked")
@Override
public <T> Codec<T> get(Class<T> clazz, CodecRegistry registry) {
if (clazz == Install.class) {
LOGGER.debug("Registering custom Install class MongoDB codec");
return (Codec<T>) new InstallCodec();
}
return null;
}
}
......@@ -69,6 +69,13 @@ public final class DatabaseFieldNames {
public static final String ERROR_IP_ADDRESS = "ip_address";
public static final String ERROR_STATUS_CODE = "status_code";
public static final String ERROR_STATUS_MESSAGE = "status_message";
// installs
public static final String INSTALL_JAVA_VERSION = "java_version";
public static final String INSTALL_VERSION = "version";
public static final String INSTALL_LISTING_ID = "listing_id";
public static final String INSTALL_DATE = "date";
public static final String ECLIPSE_VERSION = "eclipse_version";
public static final String LOCALE = "locale";
private DatabaseFieldNames() {
}
......
......@@ -8,6 +8,7 @@ package org.eclipsefoundation.marketplace.namespace;
import org.eclipsefoundation.marketplace.dto.Catalog;
import org.eclipsefoundation.marketplace.dto.Category;
import org.eclipsefoundation.marketplace.dto.Install;
import org.eclipsefoundation.marketplace.dto.Listing;
import org.eclipsefoundation.marketplace.dto.Market;
import org.eclipsefoundation.marketplace.dto.ErrorReport;
......@@ -21,7 +22,8 @@ import org.eclipsefoundation.marketplace.dto.ErrorReport;
public enum DtoTableNames {
LISTING(Listing.class, "listings"),
CATEGORY(Category.class, "categories"),
CATALOG(Catalog.class, "catalogs"), MARKET(Market.class, "markets"),ERRORREPORT(ErrorReport.class, "errorreports");
CATALOG(Catalog.class, "catalogs"), MARKET(Market.class, "markets"),
ERRORREPORT(ErrorReport.class, "errorreports"), INSTALL(Install.class, "installs");
private Class<?> baseClass;
private String tableName;
......@@ -39,7 +41,7 @@ public enum DtoTableNames {
}
return null;
}
public String getTableName() {
return this.tableName;
}
......
......@@ -30,7 +30,9 @@ public final class UrlParameterNames {
public static final String LISTING_ID = "listing_id";
public static final String READ = "read";
public static final String FEATURE_ID = "feature_id";
public static final String VERSION = "version";
public static final String DATE_FROM = "from";
private UrlParameterNames() {
}
}
/* Copyright (c) 2019 Eclipse Foundation and others.
* This program and the accompanying materials are made available
* under the terms of the Eclipse Public License 2.0
* which is available at http://www.eclipse.org/legal/epl-v20.html,
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipsefoundation.marketplace.resource;
import java.sql.Date;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import javax.enterprise.context.RequestScoped;
import javax.inject.Inject;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.eclipsefoundation.marketplace.dao.MongoDao;
import org.eclipsefoundation.marketplace.dto.Install;
import org.eclipsefoundation.marketplace.dto.filter.DtoFilter;
import org.eclipsefoundation.marketplace.helper.StreamHelper;
import org.eclipsefoundation.marketplace.model.MongoQuery;
import org.eclipsefoundation.marketplace.model.RequestWrapper;
import org.eclipsefoundation.marketplace.model.ResourceDataType;
import org.eclipsefoundation.marketplace.namespace.UrlParameterNames;
import org.eclipsefoundation.marketplace.service.CachingService;
import org.jboss.resteasy.annotations.jaxrs.PathParam;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Resource for retrieving installs + statistics from the MongoDB instance.
*
* @author Martin Lowe
*/
@RequestScoped
@ResourceDataType(Install.class)
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Path("/installs")
public class InstallResource {
private static final Logger LOGGER = LoggerFactory.getLogger(ListingResource.class);
@Inject
MongoDao dao;
@Inject
RequestWrapper params;
@Inject
DtoFilter<Install> dtoFilter;
// Inject 2 caching service references, as we want to cache count results.
@Inject
CachingService<Long> countCache;
@Inject
CachingService<List<Install>> installCache;
/**
* Endpoint for /listing/\<listingId\>/installs to retrieve install metrics for
* a specific listing from the database.
*
* @param listingId int version of the listing ID
* @return response for the browser
*/
@GET
@Path("/{listingId}")
public Response selectInstallMetrics(@PathParam("listingId") String listingId) {
params.addParam(UrlParameterNames.ID, listingId);
MongoQuery<Install> q = new MongoQuery<>(params, dtoFilter, installCache);
Optional<Long> cachedResults = countCache.get(listingId, params,
() -> StreamHelper.awaitCompletionStage(dao.count(q)));
if (!cachedResults.isPresent()) {
LOGGER.error("Error while retrieving cached listing for ID {}", listingId);
return Response.serverError().build();
}
// return the results as a response
return Response.ok(cachedResults.get()).build();
}
/**
*
* Endpoint for /listing/\<listingId\>/installs/\<version\> to retrieve install
* metrics for a specific listing version from the database.
*
* @param listingId int version of the listing ID
* @param version int version of the listing version number
* @return response for the browser
*/
@GET
@Path("/{listingId}/{version}")
public Response selectInstallMetrics(@PathParam("listingId") String listingId, @PathParam("version") String version) {
params.addParam(UrlParameterNames.ID, listingId);
params.addParam(UrlParameterNames.VERSION, version);
MongoQuery<Install> q = new MongoQuery<>(params, dtoFilter, installCache);
Optional<Long> cachedResults = countCache.get(getCompositeKey(listingId, version), params,
() -> StreamHelper.awaitCompletionStage(dao.count(q)));
if (!cachedResults.isPresent()) {
LOGGER.error("Error while retrieving cached listing for ID {}", listingId);
return Response.serverError().build();
}
// return the results as a response
return Response.ok(cachedResults.get()).build();
}
/**
* Endpoint for /listing/\<listingId\>/installs/\<version\> to post install
* metrics for a specific listing version to a database.
*
* @param listingId int version of the listing ID
* @param version int version of the listing version number
* @return response for the browser
*/
@POST
@Path("/{listingId}/{version}")
public Response postInstallMetrics(@PathParam("listingId") String listingId, @PathParam("version") String version,
Install installDetails) {
// update the install details to reflect the current request
installDetails.setInstallDate(new Date(System.currentTimeMillis()));
installDetails.setListingId(listingId);
installDetails.setVersion(version);
// create the query wrapper to pass to DB dao
MongoQuery<Install> q = new MongoQuery<>(params, dtoFilter, installCache);
// add the object, and await the result
StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(installDetails)));
// return the results as a response
return Response.ok().build();
}
private String getCompositeKey(String listingId, String version) {
return listingId + ':' + version;
}
}
......@@ -24,7 +24,6 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.eclipsefoundation.marketplace.dao.MongoDao;
import org.eclipsefoundation.marketplace.dto.Install;
import org.eclipsefoundation.marketplace.dto.Listing;
import org.eclipsefoundation.marketplace.dto.filter.DtoFilter;
import org.eclipsefoundation.marketplace.helper.StreamHelper;
......@@ -122,47 +121,4 @@ public class ListingResource {
// return the results as a response
return Response.ok(cachedResults.get()).build();
}
/**
* Endpoint for /listing/\<listingId\>/installs to retrieve install metrics for
* a specific listing from the database.
*
* @param listingId int version of the listing ID
* @return response for the browser
*/
@GET
@Path("/{listingId}/installs")
public Response selectInstallMetrics(@PathParam("listingId") String listingId) {
throw new UnsupportedOperationException("Getting install statistics is not yet supported");
}
/**
*
* Endpoint for /listing/\<listingId\>/installs/\<version\> to retrieve install
* metrics for a specific listing version from the database.
*
* @param listingId int version of the listing ID
* @param version int version of the listing version number
* @return response for the browser
*/
@GET
@Path("/{listingId}/versions/{version}/installs")
public Response selectInstallMetrics(@PathParam("listingId") String listingId, @PathParam("version") int version) {
throw new UnsupportedOperationException("Getting install statistics is not yet supported");
}
/**
* Endpoint for /listing/\<listingId\>/installs/\<version\> to post install
* metrics for a specific listing version to a database.
*
* @param listingId int version of the listing ID
* @param version int version of the listing version number
* @return response for the browser
*/
@POST
@Path("/{listingId}/versions/{version}/installs")
public Response postInstallMetrics(@PathParam("listingId") String listingId, @PathParam("version") String version,
Install installDetails) {
return Response.ok(installDetails).build();
}
}
\ No newline at end of file
......@@ -5,7 +5,13 @@ const argv = require('yargs')
.option('c', {
description: 'Number of listings to generate',
alias: 'count',
default: 5000,
default: 1000,
nargs: 1
})
.option('i', {
description: 'Number of installs to generate',
alias: 'installs',
default: 0,
nargs: 1
})
.option('s', {
......@@ -59,9 +65,15 @@ function createListing(count) {
if (count >= max) {
return;
}
count++;
axios.post(argv.s+"/listings/", generateJSON(uuid.v4()))
.then(() => createListing(count))
console.log(`Generating listing ${count} of ${max}`);
var json = generateJSON(uuid.v4());
axios.post(argv.s+"/listings/", json)
.then(() => {
var installs = Math.floor(Math.random()*argv.i);
console.log(`Generating ${installs} install records for listing '${json.id}'`);
createInstall(0, installs, json, () => createListing(count+1));
})
.catch(err => console.log(err));
}
......@@ -85,22 +97,33 @@ function createMarket(count) {
.catch(err => console.log(err));
}
function createInstall(curr, max, listing, callback) {
if (curr >= max) {
return callback();
}
var json = generateInstallJSON(listing);
axios.post(`${argv.s}/installs/${json['listing_id']}/${json.version}`, json)
.then(createInstall(curr+1,max,listing,callback))
.catch(err => console.log(err));
}
function generateJSON(id) {
var solutions = [];
var solsCount = Math.floor(Math.random()*5);
var solsCount = Math.floor(Math.random()*5) + 1;
for (var i=0; i < solsCount; i++) {
solutions.push({
"version": i,
"eclipse_versions": splice(eclipseVs),
"min_java_version": javaVs[Math.floor(Math.random()*eclipseVs.length)],
"min_java_version": javaVs[Math.floor(Math.random()*javaVs.length)],
"platforms": splice(platforms)
});
}
return {
"id": id,
"title": "Sample",
"url": "https://jakarta.ee",
"foundation_ember": false,
"foundation_member": false,
"teaser": randomWords({exactly:1, wordsPerString:Math.floor(Math.random()*100)})[0],
"body": randomWords({exactly:1, wordsPerString:Math.floor(Math.random()*300)})[0],
"status": "draft",
......@@ -144,3 +167,17 @@ function generateMarketJSON(id) {
"category_ids": splice(categoryIds).splice(0,Math.ceil(Math.random()*5)+1)
};
}
function generateInstallJSON(listing) {
var version = listing.versions[Math.floor(Math.random()*listing.versions.length)];
var javaVersions = javaVs.splice(javaVs.indexOf(version["min_java_version"]));
var eclipseVersions = eclipseVs.splice(eclipseVs.indexOf(version["eclipse_version"]));
return {
"listing_id": listing.id,
"version": version.version,
"java_version": shuff(javaVersions)[0],
"os": shuff(version.platforms)[0],
"eclipse_version": shuff(eclipseVersions)[0]
};
}
......@@ -23,7 +23,7 @@ public class ListingResourceTest {
@Test
public void testListingIdEndpoint() {
given().when().get("/listings/1").then().statusCode(200);
given().when().get("/listings/abc-123").then().statusCode(200);
}
@Test
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment