Skip to content
Snippets Groups Projects
Commit 5edd8a0f authored by Martin Lowe's avatar Martin Lowe :flag_ca: Committed by Martin Lowe
Browse files

Create an object to extract data from MPC user agents #8 


Finished UserAgent implementation w/ validity checks. Implemented call
to generate install from valid user agents. Fixed issue with generating
dummy install content where java and eclipse version arrays would break
causing issues in posted data. Additionally fixed some badly named
variables that no longer made sense. Added MPC user agent heading to
dummy content script to spoof source.

Change-Id: I32877ad6558edfb04c4d3453cf36e4750ad634c8
Signed-off-by: Martin Lowe's avatarMartin Lowe <martin.lowe@eclipse-foundation.org>
parent 49907fa6
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 245 additions and 87 deletions
src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java
+ 15
− 15
View file @ 5edd8a0f
......@@ -37,7 +37,7 @@ public class MongoQuery<T> {
private static final Logger LOGGER = LoggerFactory.getLogger(MongoQuery.class);
private CachingService<List<T>> cache;
private RequestWrapper qps;
private RequestWrapper wrapper;
private DtoFilter<T> dtoFilter;
private Bson filter;
......@@ -45,8 +45,8 @@ public class MongoQuery<T> {
private SortOrder order;
private List<Bson> aggregates;
public MongoQuery(RequestWrapper qps, DtoFilter<T> dtoFilter, CachingService<List<T>> cache) {
this.qps = qps;
public MongoQuery(RequestWrapper wrapper, DtoFilter<T> dtoFilter, CachingService<List<T>> cache) {
this.wrapper = wrapper;
this.dtoFilter = dtoFilter;
this.cache = cache;
this.aggregates = new ArrayList<>();
......@@ -68,10 +68,10 @@ public class MongoQuery<T> {
// get the filters for the current DTO
List<Bson> filters = new ArrayList<>();
filters.addAll(dtoFilter.getFilters(qps));
filters.addAll(dtoFilter.getFilters(wrapper));
// get fields that make up the required fields to enable pagination and check
Optional<String> sortOpt = qps.getFirstParam(UrlParameterNames.SORT);
Optional<String> sortOpt = wrapper.getFirstParam(UrlParameterNames.SORT);
if (sortOpt.isPresent()) {
String sortVal = sortOpt.get();
// split sort string of `<fieldName> <SortOrder>`
......@@ -86,7 +86,7 @@ public class MongoQuery<T> {
if (!filters.isEmpty()) {
this.filter = Filters.and(filters);
}
this.aggregates = dtoFilter.getAggregates(qps);
this.aggregates = dtoFilter.getAggregates(wrapper);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("MongoDB query initialized with filter: {}", this.filter);
......@@ -129,7 +129,7 @@ public class MongoQuery<T> {
* present and numeric, otherwise returns -1.
*/
public int getLimit() {
Optional<String> limitVal = qps.getFirstParam(UrlParameterNames.LIMIT);
Optional<String> limitVal = wrapper.getFirstParam(UrlParameterNames.LIMIT);
if (limitVal.isPresent() && StringUtils.isNumeric(limitVal.get())) {
return Integer.parseInt(limitVal.get());
}
......@@ -137,7 +137,7 @@ public class MongoQuery<T> {
}
private void setSort(String sortField, String sortOrder, List<Bson> filters) {
Optional<String> lastOpt = qps.getFirstParam(UrlParameterNames.LAST_SEEN);
Optional<String> lastOpt = wrapper.getFirstParam(UrlParameterNames.LAST_SEEN);
List<Sortable<?>> fields = SortableHelper.getSortableFields(getDocType());
Optional<Sortable<?>> fieldContainer = SortableHelper.getSortableFieldByName(fields, sortField);
......@@ -186,21 +186,21 @@ public class MongoQuery<T> {
* @return the docType
*/
public Class<T> getDocType() {
return (Class<T>) qps.getAttribute(AnnotationClassInjectionFilter.ATTRIBUTE_NAME).get();
return (Class<T>) wrapper.getAttribute(AnnotationClassInjectionFilter.ATTRIBUTE_NAME).get();
}
/**
* @return the qps
* @return the wrapper
*/
public RequestWrapper getQps() {
return qps;
public RequestWrapper getWrapper() {
return wrapper;
}
/**
* @param qps the qps to set
* @param wrapper the wrapper to set
*/
public void setQps(RequestWrapper qps) {
this.qps = qps;
public void setWrapper(RequestWrapper wrapper) {
this.wrapper = wrapper;
}
@Override
......
src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java
+ 13
− 3
View file @ 5edd8a0f
......@@ -53,7 +53,7 @@ public class RequestWrapper {
/**
* Retrieves the first value set in a list from the map for a given key.
*
* @param params the parameter map containing the value
* @param wrapper the parameter map containing the value
* @param key the key to retrieve the value for
* @return the first value set in the parameter map for the given key, or null
* if absent.
......@@ -73,7 +73,7 @@ public class RequestWrapper {
/**
* Retrieves the value list from the map for a given key.
*
* @param params the parameter map containing the values
* @param wrapper the parameter map containing the values
* @param key the key to retrieve the values for
* @return the value list for the given key if it exists, or an empty collection
* if none exists.
......@@ -94,7 +94,7 @@ public class RequestWrapper {
* Adds the given value for the given key, preserving previous values if they
* exist.
*
* @param params map containing parameters to update
* @param wrapper map containing parameters to update
* @param key string key to add the value to, must not be null
* @param value the value to add to the key
*/
......@@ -166,4 +166,14 @@ public class RequestWrapper {
}
return this.userAgent;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("RequestWrapper [");
sb.append("ip=").append(request.getRemoteAddr());
sb.append(", uri=").append(request.getRequestURI());
sb.append(", params=").append(getParams());
return sb.toString();
}
}
src/main/java/org/eclipsefoundation/marketplace/model/UserAgent.java
+ 169
− 46
View file @ 5edd8a0f
......@@ -6,12 +6,14 @@
*/
package org.eclipsefoundation.marketplace.model;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.eclipsefoundation.marketplace.dto.Install;
import com.google.common.base.Splitter;
/**
......@@ -21,7 +23,6 @@ import com.google.common.base.Splitter;
* @author Martin Lowe
*/
public class UserAgent {
/**
* Matches a generic form of User-Agent strings with the following sections
*
......@@ -32,16 +33,18 @@ public class UserAgent {
* and version, separated by a slash.
*/
private static final Pattern USER_AGENT_PATTERN = Pattern
.compile("^(\\S+\\/\\S+)\\s?(?:\\(([^\\)]*?)\\)([^\\(]+(?:\\(([^\\)]*?)\\)([^\\(]+)?)?)?)?$");
.compile("^(\\S+\\/\\S+)\\s?(?:\\(([^\\)]*?)\\)(?:([^\\(]+)(?:\\(([^\\)]*+)\\))?)?)?$");
private static final String MPC_CLIENT_AGENT_NAME = "mpc";
private final String name;
private final String version;
private final String base;
private final String agentDeclaration;
private final String systemProperties;
private final String platform;
private final String platformDetails;
private final String enhancements;
private final String application;
private String name;
private String version;
private String javaVersion;
private String javaVendor;
......@@ -49,8 +52,12 @@ public class UserAgent {
private String osVersion;
private String locale;
private String product;
private String productVersion;
private String eclipseVersion;
private boolean valid = true;
public UserAgent(String userAgent) {
Objects.requireNonNull(userAgent);
......@@ -60,49 +67,52 @@ public class UserAgent {
throw new IllegalArgumentException("Passed string does not match an expected user-agent");
}
this.base = userAgent;
// get the name and version of the user agent
String agentDeclaration = m.group(0);
Iterator<String> it = Splitter.on('/').trimResults().split(agentDeclaration).iterator();
this.name = it.next();
this.version = it.next();
this.agentDeclaration = m.group(1);
List<String> agentProperties = Splitter.on('/').trimResults().limit(2).splitToList(agentDeclaration);
if (agentProperties.size() != 2) {
// should never throw as format is promised in regex
throw new IllegalArgumentException("Cannot read User-Agent name and version");
}
this.name = agentProperties.get(0);
this.version = agentProperties.get(1);
this.systemProperties = m.group(1);
this.platform = m.group(2);
this.systemProperties = m.group(2);
this.platformDetails = m.group(3);
this.enhancements = m.group(4);
if (MPC_CLIENT_AGENT_NAME.equalsIgnoreCase(name)) {
handleMpc();
this.application = m.group(4);
if (isFromMPC()) {
consumeSystemProps();
consumePlatformDetails();
}
}
/**
* Breaks down the different MPC properties into explicit properties that can be
* retrieved via getters built into the class. The expected format is defined
* below:
*
* <p>
* {@code mpc/<mpc version> (Java <java version> <java vendor>; <os name> <os version> <os arch>; <locale>) <eclipse product>/<product version> (<eclipse application>)}
* </p>
*/
private void handleMpc() {
private void consumeSystemProps() {
if (this.systemProperties == null) {
this.valid = false;
return;
}
// expected form: (Java <java version> <java vendor>; <os name> <os version> <os
// arch>; <locale>)
List<String> systemPropList = Splitter.on(';').splitToList(systemProperties);
List<String> systemPropList = Splitter.on(';').trimResults().splitToList(systemProperties);
if (systemPropList.size() != 3) {
// TODO throw exception?
this.valid = false;
return;
}
// expected form example: Java <java version> <vendor>
List<String> javaProps = Splitter.on(' ').limit(3).splitToList(systemPropList.get(0));
List<String> javaProps = Splitter.on(' ').trimResults().limit(3).splitToList(systemPropList.get(0));
if (javaProps.size() != 3) {
// TODO throw exception?
this.valid = false;
return;
}
this.javaVersion = javaProps.get(1);
this.javaVendor = javaProps.get(2);
// expected form: <OS name> <OS version> <OS arch>
List<String> systemProps = Splitter.on(' ').limit(3).splitToList(systemPropList.get(1));
List<String> systemProps = Splitter.on(' ').trimResults().limit(3).splitToList(systemPropList.get(1));
if (systemProps.size() != 3) {
// TODO throw exception?
this.valid = false;
return;
}
this.os = systemProps.get(0);
this.osVersion = systemProps.get(1);
......@@ -110,36 +120,75 @@ public class UserAgent {
// get the current locale
this.locale = systemPropList.get(2);
// expected form: <eclipse product>/<product version>
List<String> platformProps = Splitter.on('/').limit(3).splitToList(platform);
// check if any fields are invalid
if (StringUtils.isBlank(javaVersion) || StringUtils.isBlank(javaVendor) || StringUtils.isBlank(os)
|| StringUtils.isBlank(locale)) {
this.valid = false;
}
}
private void consumePlatformDetails() {
if (this.platformDetails == null) {
this.valid = false;
return;
}
// expected form: <eclipse product>/<product version>/<platform version>
List<String> platformProps = Splitter.on('/').trimResults().limit(3).splitToList(platformDetails);
if (platformProps.size() != 3) {
this.valid = false;
return;
}
// get the properties and check if any fields are invalid
this.product = platformProps.get(0);
this.productVersion = platformProps.get(1);
this.eclipseVersion = platformProps.get(2);
if (StringUtils.isBlank(product) || StringUtils.isBlank(productVersion)
|| StringUtils.isBlank(eclipseVersion)) {
this.valid = false;
}
}
/**
* @return the name
* Generates a basic install record based on information based on the user agent
* properties. This can only be used when the agent is detected as an MPC call
* and the object is valid.
*
* @return a basic populated install record without listing information, or null
* if the call doesn't originate from MPC or is missing information.
*/
public String getName() {
return name;
public Install generateInstallRecord() {
// check that agent comes from MPC and is valid before generating
if (!isValid()) {
return null;
}
// generate install record from fields
Install install = new Install();
install.setJavaVersion(javaVersion);
install.setLocale(locale);
install.setOs(os);
install.setEclipseVersion(eclipseVersion);
return install;
}
/**
* @return the version
* @return the base user agent string
*/
public String getVersion() {
return version;
public String getBase() {
return base;
}
/**
* @return the version
* @return the systemProperties
*/
public String getSystemProperties() {
return systemProperties;
}
/**
* @return the platform
* @return the agentDeclaration
*/
public String getPlatform() {
return platform;
public String getAgentDeclaration() {
return agentDeclaration;
}
/**
......@@ -152,8 +201,22 @@ public class UserAgent {
/**
* @return the enhancements
*/
public String getEnhancements() {
return enhancements;
public String getApplication() {
return application;
}
/**
* @return the name
*/
public String getName() {
return name;
}
/**
* @return the version
*/
public String getVersion() {
return version;
}
/**
......@@ -191,4 +254,64 @@ public class UserAgent {
return locale;
}
/**
* @return the product
*/
public String getProduct() {
return product;
}
/**
* @return the productVersion
*/
public String getProductVersion() {
return productVersion;
}
/**
* Checks whether the clients agent name matches expected value for the
* marketplace client {@link MPC_CLIENT_AGENT_NAME}.
*
* @return true if client agent name matches expected value, false otherwise.
*/
public boolean isFromMPC() {
return MPC_CLIENT_AGENT_NAME.equalsIgnoreCase(name);
}
/**
* @return whether the current user agent is a valid MPC user agent
*/
public boolean isValid() {
return valid && isFromMPC();
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("UserAgent [name=");
builder.append(name);
builder.append(", version=");
builder.append(version);
builder.append(", javaVersion=");
builder.append(javaVersion);
builder.append(", javaVendor=");
builder.append(javaVendor);
builder.append(", os=");
builder.append(os);
builder.append(", osVersion=");
builder.append(osVersion);
builder.append(", locale=");
builder.append(locale);
builder.append(", eclipseVersion=");
builder.append(eclipseVersion);
builder.append(", product=");
builder.append(product);
builder.append(", productVersion=");
builder.append(productVersion);
builder.append(", application=");
builder.append(application);
builder.append("]");
return builder.toString();
}
}
src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java
+ 38
− 17
View file @ 5edd8a0f
......@@ -20,11 +20,13 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import org.eclipsefoundation.marketplace.dao.MongoDao;
import org.eclipsefoundation.marketplace.dto.Install;
import org.eclipsefoundation.marketplace.dto.filter.DtoFilter;
import org.eclipsefoundation.marketplace.helper.StreamHelper;
import org.eclipsefoundation.marketplace.model.Error;
import org.eclipsefoundation.marketplace.model.MongoQuery;
import org.eclipsefoundation.marketplace.model.RequestWrapper;
import org.eclipsefoundation.marketplace.model.ResourceDataType;
......@@ -50,7 +52,7 @@ public class InstallResource {
@Inject
MongoDao dao;
@Inject
RequestWrapper params;
RequestWrapper wrapper;
@Inject
DtoFilter<Install> dtoFilter;
......@@ -59,7 +61,7 @@ public class InstallResource {
CachingService<Long> countCache;
@Inject
CachingService<List<Install>> installCache;
/**
* Endpoint for /listing/\<listingId\>/installs to retrieve install metrics for
* a specific listing from the database.
......@@ -70,9 +72,9 @@ public class InstallResource {
@GET
@Path("/{listingId}")
public Response selectInstallMetrics(@PathParam("listingId") String listingId) {
params.addParam(UrlParameterNames.ID, listingId);
MongoQuery<Install> q = new MongoQuery<>(params, dtoFilter, installCache);
Optional<Long> cachedResults = countCache.get(listingId, params,
wrapper.addParam(UrlParameterNames.ID, listingId);
MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, installCache);
Optional<Long> cachedResults = countCache.get(listingId, wrapper,
() -> StreamHelper.awaitCompletionStage(dao.count(q)));
if (!cachedResults.isPresent()) {
LOGGER.error("Error while retrieving cached listing for ID {}", listingId);
......@@ -94,11 +96,12 @@ public class InstallResource {
*/
@GET
@Path("/{listingId}/{version}")
public Response selectInstallMetrics(@PathParam("listingId") String listingId, @PathParam("version") String version) {
params.addParam(UrlParameterNames.ID, listingId);
params.addParam(UrlParameterNames.VERSION, version);
MongoQuery<Install> q = new MongoQuery<>(params, dtoFilter, installCache);
Optional<Long> cachedResults = countCache.get(getCompositeKey(listingId, version), params,
public Response selectInstallMetrics(@PathParam("listingId") String listingId,
@PathParam("version") String version) {
wrapper.addParam(UrlParameterNames.ID, listingId);
wrapper.addParam(UrlParameterNames.VERSION, version);
MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, installCache);
Optional<Long> cachedResults = countCache.get(getCompositeKey(listingId, version), wrapper,
() -> StreamHelper.awaitCompletionStage(dao.count(q)));
if (!cachedResults.isPresent()) {
LOGGER.error("Error while retrieving cached listing for ID {}", listingId);
......@@ -121,21 +124,39 @@ public class InstallResource {
@Path("/{listingId}/{version}")
public Response postInstallMetrics(@PathParam("listingId") String listingId, @PathParam("version") String version,
Install installDetails) {
// update the install details to reflect the current request
installDetails.setInstallDate(new Date(System.currentTimeMillis()));
installDetails.setListingId(listingId);
installDetails.setVersion(version);
Install record = null;
// check that connection was opened by MPC, and check for install information
// from user agent
if (wrapper.getUserAgent().isValid()) {
record = wrapper.getUserAgent().generateInstallRecord();
} else if (wrapper.getUserAgent().isFromMPC()) {
if (installDetails == null) {
return new Error(Status.BAD_REQUEST, "Install data could not be read from request body")
.asResponse();
}
record = installDetails;
} else {
LOGGER.warn("Rebuffed request to post install from request: {}", wrapper);
return new Error(Status.FORBIDDEN, "Installs cannot be posted directly from consumer applications")
.asResponse();
}
// update the install details to reflect the current request
record.setInstallDate(new Date(System.currentTimeMillis()));
record.setListingId(listingId);
record.setVersion(version);
// create the query wrapper to pass to DB dao
MongoQuery<Install> q = new MongoQuery<>(params, dtoFilter, installCache);
MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, installCache);
// add the object, and await the result
StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(installDetails)));
StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(record)));
// return the results as a response
return Response.ok().build();
}
private String getCompositeKey(String listingId, String version) {
return listingId + ':' + version;
}
......
src/main/node/index.js
+ 10
− 6
View file @ 5edd8a0f
const axios = require('axios');
const instance = axios.create({
timeout: 1000,
headers: {'User-Agent': 'mpc/0.0.0'}
});
const randomWords = require('random-words');
const uuid = require('uuid');
const argv = require('yargs')
......@@ -68,7 +72,7 @@ function createListing(count) {
console.log(`Generating listing ${count} of ${max}`);
var json = generateJSON(uuid.v4());
axios.put(argv.s+"/listings/", json)
axios.post(argv.s+"/listings/", json)
.then(() => {
var installs = Math.floor(Math.random()*argv.i);
console.log(`Generating ${installs} install records for listing '${json.id}'`);
......@@ -82,7 +86,7 @@ function createCategory(count) {
return;
}
axios.put(argv.s+"/categories/", generateCategoryJSON(categoryIds[count++]))
instance.put(argv.s+"/categories/", generateCategoryJSON(categoryIds[count++]))
.then(() => createCategory(count))
.catch(err => console.log(err));
}
......@@ -92,7 +96,7 @@ function createMarket(count) {
return;
}
axios.put(argv.s+"/markets/", generateMarketJSON(marketIds[count++]))
instance.put(argv.s+"/markets/", generateMarketJSON(marketIds[count++]))
.then(() => createMarket(count))
.catch(err => console.log(err));
}
......@@ -102,7 +106,7 @@ function createInstall(curr, max, listing, callback) {
return callback();
}
var json = generateInstallJSON(listing);
axios.post(`${argv.s}/installs/${json['listing_id']}/${json.version}`, json)
instance.post(`${argv.s}/installs/${json['listing_id']}/${json.version}`, json)
.then(createInstall(curr+1,max,listing,callback))
.catch(err => console.log(err));
}
......@@ -170,8 +174,8 @@ function generateMarketJSON(id) {
function generateInstallJSON(listing) {
var version = listing.versions[Math.floor(Math.random()*listing.versions.length)];
var javaVersions = javaVs.splice(javaVs.indexOf(version["min_java_version"]));
var eclipseVersions = eclipseVs.splice(eclipseVs.indexOf(version["eclipse_version"]));
var javaVersions = Array.from(javaVs).splice(javaVs.indexOf(version["min_java_version"]));
var eclipseVersions = Array.from(eclipseVs).splice(eclipseVs.indexOf(version["eclipse_version"]));
return {
"listing_id": listing.id,
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Copyright © Eclipse Foundation, Inc. All Rights Reserved.     Privacy Policy | Terms of Use | Copyright Agent