From 5edd8a0fcdd673f76d8941596903a2b08c91bcf7 Mon Sep 17 00:00:00 2001
From: Martin Lowe <martin.lowe@eclipse-foundation.org>
Date: Thu, 17 Oct 2019 13:29:34 -0400
Subject: [PATCH] Create an object to extract data from MPC user agents #8
Finished UserAgent implementation w/ validity checks. Implemented call
to generate install from valid user agents. Fixed issue with generating
dummy install content where java and eclipse version arrays would break
causing issues in posted data. Additionally fixed some badly named
variables that no longer made sense. Added MPC user agent heading to
dummy content script to spoof source.
Change-Id: I32877ad6558edfb04c4d3453cf36e4750ad634c8
Signed-off-by: Martin Lowe <martin.lowe@eclipse-foundation.org>
---
.../marketplace/model/MongoQuery.java | 30 +--
.../marketplace/model/RequestWrapper.java | 16 +-
.../marketplace/model/UserAgent.java | 215 ++++++++++++++----
.../marketplace/resource/InstallResource.java | 55 +++--
src/main/node/index.js | 16 +-
5 files changed, 245 insertions(+), 87 deletions(-)
diff --git a/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java b/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java
index c1ae7dc..4972ca6 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java
@@ -37,7 +37,7 @@ public class MongoQuery<T> {
private static final Logger LOGGER = LoggerFactory.getLogger(MongoQuery.class);
private CachingService<List<T>> cache;
- private RequestWrapper qps;
+ private RequestWrapper wrapper;
private DtoFilter<T> dtoFilter;
private Bson filter;
@@ -45,8 +45,8 @@ public class MongoQuery<T> {
private SortOrder order;
private List<Bson> aggregates;
- public MongoQuery(RequestWrapper qps, DtoFilter<T> dtoFilter, CachingService<List<T>> cache) {
- this.qps = qps;
+ public MongoQuery(RequestWrapper wrapper, DtoFilter<T> dtoFilter, CachingService<List<T>> cache) {
+ this.wrapper = wrapper;
this.dtoFilter = dtoFilter;
this.cache = cache;
this.aggregates = new ArrayList<>();
@@ -68,10 +68,10 @@ public class MongoQuery<T> {
// get the filters for the current DTO
List<Bson> filters = new ArrayList<>();
- filters.addAll(dtoFilter.getFilters(qps));
+ filters.addAll(dtoFilter.getFilters(wrapper));
// get fields that make up the required fields to enable pagination and check
- Optional<String> sortOpt = qps.getFirstParam(UrlParameterNames.SORT);
+ Optional<String> sortOpt = wrapper.getFirstParam(UrlParameterNames.SORT);
if (sortOpt.isPresent()) {
String sortVal = sortOpt.get();
// split sort string of `<fieldName> <SortOrder>`
@@ -86,7 +86,7 @@ public class MongoQuery<T> {
if (!filters.isEmpty()) {
this.filter = Filters.and(filters);
}
- this.aggregates = dtoFilter.getAggregates(qps);
+ this.aggregates = dtoFilter.getAggregates(wrapper);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("MongoDB query initialized with filter: {}", this.filter);
@@ -129,7 +129,7 @@ public class MongoQuery<T> {
* present and numeric, otherwise returns -1.
*/
public int getLimit() {
- Optional<String> limitVal = qps.getFirstParam(UrlParameterNames.LIMIT);
+ Optional<String> limitVal = wrapper.getFirstParam(UrlParameterNames.LIMIT);
if (limitVal.isPresent() && StringUtils.isNumeric(limitVal.get())) {
return Integer.parseInt(limitVal.get());
}
@@ -137,7 +137,7 @@ public class MongoQuery<T> {
}
private void setSort(String sortField, String sortOrder, List<Bson> filters) {
- Optional<String> lastOpt = qps.getFirstParam(UrlParameterNames.LAST_SEEN);
+ Optional<String> lastOpt = wrapper.getFirstParam(UrlParameterNames.LAST_SEEN);
List<Sortable<?>> fields = SortableHelper.getSortableFields(getDocType());
Optional<Sortable<?>> fieldContainer = SortableHelper.getSortableFieldByName(fields, sortField);
@@ -186,21 +186,21 @@ public class MongoQuery<T> {
* @return the docType
*/
public Class<T> getDocType() {
- return (Class<T>) qps.getAttribute(AnnotationClassInjectionFilter.ATTRIBUTE_NAME).get();
+ return (Class<T>) wrapper.getAttribute(AnnotationClassInjectionFilter.ATTRIBUTE_NAME).get();
}
/**
- * @return the qps
+ * @return the wrapper
*/
- public RequestWrapper getQps() {
- return qps;
+ public RequestWrapper getWrapper() {
+ return wrapper;
}
/**
- * @param qps the qps to set
+ * @param wrapper the wrapper to set
*/
- public void setQps(RequestWrapper qps) {
- this.qps = qps;
+ public void setWrapper(RequestWrapper wrapper) {
+ this.wrapper = wrapper;
}
@Override
diff --git a/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java b/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java
index b6f103c..9169035 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java
@@ -53,7 +53,7 @@ public class RequestWrapper {
/**
* Retrieves the first value set in a list from the map for a given key.
*
- * @param params the parameter map containing the value
+ * @param wrapper the parameter map containing the value
* @param key the key to retrieve the value for
* @return the first value set in the parameter map for the given key, or null
* if absent.
@@ -73,7 +73,7 @@ public class RequestWrapper {
/**
* Retrieves the value list from the map for a given key.
*
- * @param params the parameter map containing the values
+ * @param wrapper the parameter map containing the values
* @param key the key to retrieve the values for
* @return the value list for the given key if it exists, or an empty collection
* if none exists.
@@ -94,7 +94,7 @@ public class RequestWrapper {
* Adds the given value for the given key, preserving previous values if they
* exist.
*
- * @param params map containing parameters to update
+ * @param wrapper map containing parameters to update
* @param key string key to add the value to, must not be null
* @param value the value to add to the key
*/
@@ -166,4 +166,14 @@ public class RequestWrapper {
}
return this.userAgent;
}
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("RequestWrapper [");
+ sb.append("ip=").append(request.getRemoteAddr());
+ sb.append(", uri=").append(request.getRequestURI());
+ sb.append(", params=").append(getParams());
+ return sb.toString();
+ }
}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/model/UserAgent.java b/src/main/java/org/eclipsefoundation/marketplace/model/UserAgent.java
index c5221f5..31700ed 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/model/UserAgent.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/model/UserAgent.java
@@ -6,12 +6,14 @@
*/
package org.eclipsefoundation.marketplace.model;
-import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.lang3.StringUtils;
+import org.eclipsefoundation.marketplace.dto.Install;
+
import com.google.common.base.Splitter;
/**
@@ -21,7 +23,6 @@ import com.google.common.base.Splitter;
* @author Martin Lowe
*/
public class UserAgent {
-
/**
* Matches a generic form of User-Agent strings with the following sections
*
@@ -32,16 +33,18 @@ public class UserAgent {
* and version, separated by a slash.
*/
private static final Pattern USER_AGENT_PATTERN = Pattern
- .compile("^(\\S+\\/\\S+)\\s?(?:\\(([^\\)]*?)\\)([^\\(]+(?:\\(([^\\)]*?)\\)([^\\(]+)?)?)?)?$");
+ .compile("^(\\S+\\/\\S+)\\s?(?:\\(([^\\)]*?)\\)(?:([^\\(]+)(?:\\(([^\\)]*+)\\))?)?)?$");
private static final String MPC_CLIENT_AGENT_NAME = "mpc";
- private final String name;
- private final String version;
+ private final String base;
+
+ private final String agentDeclaration;
private final String systemProperties;
- private final String platform;
private final String platformDetails;
- private final String enhancements;
+ private final String application;
+ private String name;
+ private String version;
private String javaVersion;
private String javaVendor;
@@ -49,8 +52,12 @@ public class UserAgent {
private String osVersion;
private String locale;
+ private String product;
+ private String productVersion;
private String eclipseVersion;
+ private boolean valid = true;
+
public UserAgent(String userAgent) {
Objects.requireNonNull(userAgent);
@@ -60,49 +67,52 @@ public class UserAgent {
throw new IllegalArgumentException("Passed string does not match an expected user-agent");
}
+ this.base = userAgent;
// get the name and version of the user agent
- String agentDeclaration = m.group(0);
- Iterator<String> it = Splitter.on('/').trimResults().split(agentDeclaration).iterator();
- this.name = it.next();
- this.version = it.next();
+ this.agentDeclaration = m.group(1);
+ List<String> agentProperties = Splitter.on('/').trimResults().limit(2).splitToList(agentDeclaration);
+ if (agentProperties.size() != 2) {
+ // should never throw as format is promised in regex
+ throw new IllegalArgumentException("Cannot read User-Agent name and version");
+ }
+ this.name = agentProperties.get(0);
+ this.version = agentProperties.get(1);
- this.systemProperties = m.group(1);
- this.platform = m.group(2);
+ this.systemProperties = m.group(2);
this.platformDetails = m.group(3);
- this.enhancements = m.group(4);
- if (MPC_CLIENT_AGENT_NAME.equalsIgnoreCase(name)) {
- handleMpc();
+ this.application = m.group(4);
+ if (isFromMPC()) {
+ consumeSystemProps();
+ consumePlatformDetails();
}
}
- /**
- * Breaks down the different MPC properties into explicit properties that can be
- * retrieved via getters built into the class. The expected format is defined
- * below:
- *
- * <p>
- * {@code mpc/<mpc version> (Java <java version> <java vendor>; <os name> <os version> <os arch>; <locale>) <eclipse product>/<product version> (<eclipse application>)}
- * </p>
- */
- private void handleMpc() {
+ private void consumeSystemProps() {
+ if (this.systemProperties == null) {
+ this.valid = false;
+ return;
+ }
// expected form: (Java <java version> <java vendor>; <os name> <os version> <os
// arch>; <locale>)
- List<String> systemPropList = Splitter.on(';').splitToList(systemProperties);
+ List<String> systemPropList = Splitter.on(';').trimResults().splitToList(systemProperties);
if (systemPropList.size() != 3) {
- // TODO throw exception?
+ this.valid = false;
+ return;
}
// expected form example: Java <java version> <vendor>
- List<String> javaProps = Splitter.on(' ').limit(3).splitToList(systemPropList.get(0));
+ List<String> javaProps = Splitter.on(' ').trimResults().limit(3).splitToList(systemPropList.get(0));
if (javaProps.size() != 3) {
- // TODO throw exception?
+ this.valid = false;
+ return;
}
this.javaVersion = javaProps.get(1);
this.javaVendor = javaProps.get(2);
// expected form: <OS name> <OS version> <OS arch>
- List<String> systemProps = Splitter.on(' ').limit(3).splitToList(systemPropList.get(1));
+ List<String> systemProps = Splitter.on(' ').trimResults().limit(3).splitToList(systemPropList.get(1));
if (systemProps.size() != 3) {
- // TODO throw exception?
+ this.valid = false;
+ return;
}
this.os = systemProps.get(0);
this.osVersion = systemProps.get(1);
@@ -110,36 +120,75 @@ public class UserAgent {
// get the current locale
this.locale = systemPropList.get(2);
- // expected form: <eclipse product>/<product version>
- List<String> platformProps = Splitter.on('/').limit(3).splitToList(platform);
+ // check if any fields are invalid
+ if (StringUtils.isBlank(javaVersion) || StringUtils.isBlank(javaVendor) || StringUtils.isBlank(os)
+ || StringUtils.isBlank(locale)) {
+ this.valid = false;
+ }
+ }
+
+ private void consumePlatformDetails() {
+ if (this.platformDetails == null) {
+ this.valid = false;
+ return;
+ }
+ // expected form: <eclipse product>/<product version>/<platform version>
+ List<String> platformProps = Splitter.on('/').trimResults().limit(3).splitToList(platformDetails);
+ if (platformProps.size() != 3) {
+ this.valid = false;
+ return;
+ }
+ // get the properties and check if any fields are invalid
+ this.product = platformProps.get(0);
+ this.productVersion = platformProps.get(1);
+ this.eclipseVersion = platformProps.get(2);
+ if (StringUtils.isBlank(product) || StringUtils.isBlank(productVersion)
+ || StringUtils.isBlank(eclipseVersion)) {
+ this.valid = false;
+ }
}
/**
- * @return the name
+ * Generates a basic install record based on information based on the user agent
+ * properties. This can only be used when the agent is detected as an MPC call
+ * and the object is valid.
+ *
+ * @return a basic populated install record without listing information, or null
+ * if the call doesn't originate from MPC or is missing information.
*/
- public String getName() {
- return name;
+ public Install generateInstallRecord() {
+ // check that agent comes from MPC and is valid before generating
+ if (!isValid()) {
+ return null;
+ }
+ // generate install record from fields
+ Install install = new Install();
+ install.setJavaVersion(javaVersion);
+ install.setLocale(locale);
+ install.setOs(os);
+ install.setEclipseVersion(eclipseVersion);
+ return install;
}
/**
- * @return the version
+ * @return the base user agent string
*/
- public String getVersion() {
- return version;
+ public String getBase() {
+ return base;
}
/**
- * @return the version
+ * @return the systemProperties
*/
public String getSystemProperties() {
return systemProperties;
}
/**
- * @return the platform
+ * @return the agentDeclaration
*/
- public String getPlatform() {
- return platform;
+ public String getAgentDeclaration() {
+ return agentDeclaration;
}
/**
@@ -152,8 +201,22 @@ public class UserAgent {
/**
* @return the enhancements
*/
- public String getEnhancements() {
- return enhancements;
+ public String getApplication() {
+ return application;
+ }
+
+ /**
+ * @return the name
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * @return the version
+ */
+ public String getVersion() {
+ return version;
}
/**
@@ -191,4 +254,64 @@ public class UserAgent {
return locale;
}
+ /**
+ * @return the product
+ */
+ public String getProduct() {
+ return product;
+ }
+
+ /**
+ * @return the productVersion
+ */
+ public String getProductVersion() {
+ return productVersion;
+ }
+
+ /**
+ * Checks whether the clients agent name matches expected value for the
+ * marketplace client {@link MPC_CLIENT_AGENT_NAME}.
+ *
+ * @return true if client agent name matches expected value, false otherwise.
+ */
+ public boolean isFromMPC() {
+ return MPC_CLIENT_AGENT_NAME.equalsIgnoreCase(name);
+ }
+
+ /**
+ * @return whether the current user agent is a valid MPC user agent
+ */
+ public boolean isValid() {
+ return valid && isFromMPC();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("UserAgent [name=");
+ builder.append(name);
+ builder.append(", version=");
+ builder.append(version);
+ builder.append(", javaVersion=");
+ builder.append(javaVersion);
+ builder.append(", javaVendor=");
+ builder.append(javaVendor);
+ builder.append(", os=");
+ builder.append(os);
+ builder.append(", osVersion=");
+ builder.append(osVersion);
+ builder.append(", locale=");
+ builder.append(locale);
+ builder.append(", eclipseVersion=");
+ builder.append(eclipseVersion);
+ builder.append(", product=");
+ builder.append(product);
+ builder.append(", productVersion=");
+ builder.append(productVersion);
+ builder.append(", application=");
+ builder.append(application);
+ builder.append("]");
+ return builder.toString();
+ }
+
}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java
index 9b84085..d5882ed 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java
@@ -20,11 +20,13 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
import org.eclipsefoundation.marketplace.dao.MongoDao;
import org.eclipsefoundation.marketplace.dto.Install;
import org.eclipsefoundation.marketplace.dto.filter.DtoFilter;
import org.eclipsefoundation.marketplace.helper.StreamHelper;
+import org.eclipsefoundation.marketplace.model.Error;
import org.eclipsefoundation.marketplace.model.MongoQuery;
import org.eclipsefoundation.marketplace.model.RequestWrapper;
import org.eclipsefoundation.marketplace.model.ResourceDataType;
@@ -50,7 +52,7 @@ public class InstallResource {
@Inject
MongoDao dao;
@Inject
- RequestWrapper params;
+ RequestWrapper wrapper;
@Inject
DtoFilter<Install> dtoFilter;
@@ -59,7 +61,7 @@ public class InstallResource {
CachingService<Long> countCache;
@Inject
CachingService<List<Install>> installCache;
-
+
/**
* Endpoint for /listing/\<listingId\>/installs to retrieve install metrics for
* a specific listing from the database.
@@ -70,9 +72,9 @@ public class InstallResource {
@GET
@Path("/{listingId}")
public Response selectInstallMetrics(@PathParam("listingId") String listingId) {
- params.addParam(UrlParameterNames.ID, listingId);
- MongoQuery<Install> q = new MongoQuery<>(params, dtoFilter, installCache);
- Optional<Long> cachedResults = countCache.get(listingId, params,
+ wrapper.addParam(UrlParameterNames.ID, listingId);
+ MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, installCache);
+ Optional<Long> cachedResults = countCache.get(listingId, wrapper,
() -> StreamHelper.awaitCompletionStage(dao.count(q)));
if (!cachedResults.isPresent()) {
LOGGER.error("Error while retrieving cached listing for ID {}", listingId);
@@ -94,11 +96,12 @@ public class InstallResource {
*/
@GET
@Path("/{listingId}/{version}")
- public Response selectInstallMetrics(@PathParam("listingId") String listingId, @PathParam("version") String version) {
- params.addParam(UrlParameterNames.ID, listingId);
- params.addParam(UrlParameterNames.VERSION, version);
- MongoQuery<Install> q = new MongoQuery<>(params, dtoFilter, installCache);
- Optional<Long> cachedResults = countCache.get(getCompositeKey(listingId, version), params,
+ public Response selectInstallMetrics(@PathParam("listingId") String listingId,
+ @PathParam("version") String version) {
+ wrapper.addParam(UrlParameterNames.ID, listingId);
+ wrapper.addParam(UrlParameterNames.VERSION, version);
+ MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, installCache);
+ Optional<Long> cachedResults = countCache.get(getCompositeKey(listingId, version), wrapper,
() -> StreamHelper.awaitCompletionStage(dao.count(q)));
if (!cachedResults.isPresent()) {
LOGGER.error("Error while retrieving cached listing for ID {}", listingId);
@@ -121,21 +124,39 @@ public class InstallResource {
@Path("/{listingId}/{version}")
public Response postInstallMetrics(@PathParam("listingId") String listingId, @PathParam("version") String version,
Install installDetails) {
- // update the install details to reflect the current request
- installDetails.setInstallDate(new Date(System.currentTimeMillis()));
- installDetails.setListingId(listingId);
- installDetails.setVersion(version);
+ Install record = null;
+ // check that connection was opened by MPC, and check for install information
+ // from user agent
+ if (wrapper.getUserAgent().isValid()) {
+ record = wrapper.getUserAgent().generateInstallRecord();
+ } else if (wrapper.getUserAgent().isFromMPC()) {
+ if (installDetails == null) {
+ return new Error(Status.BAD_REQUEST, "Install data could not be read from request body")
+ .asResponse();
+ }
+ record = installDetails;
+ } else {
+ LOGGER.warn("Rebuffed request to post install from request: {}", wrapper);
+ return new Error(Status.FORBIDDEN, "Installs cannot be posted directly from consumer applications")
+ .asResponse();
+ }
+
+ // update the install details to reflect the current request
+ record.setInstallDate(new Date(System.currentTimeMillis()));
+ record.setListingId(listingId);
+ record.setVersion(version);
+
// create the query wrapper to pass to DB dao
- MongoQuery<Install> q = new MongoQuery<>(params, dtoFilter, installCache);
+ MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, installCache);
// add the object, and await the result
- StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(installDetails)));
+ StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(record)));
// return the results as a response
return Response.ok().build();
}
-
+
private String getCompositeKey(String listingId, String version) {
return listingId + ':' + version;
}
diff --git a/src/main/node/index.js b/src/main/node/index.js
index e089979..5bbf210 100644
--- a/src/main/node/index.js
+++ b/src/main/node/index.js
@@ -1,4 +1,8 @@
const axios = require('axios');
+const instance = axios.create({
+ timeout: 1000,
+ headers: {'User-Agent': 'mpc/0.0.0'}
+});
const randomWords = require('random-words');
const uuid = require('uuid');
const argv = require('yargs')
@@ -68,7 +72,7 @@ function createListing(count) {
console.log(`Generating listing ${count} of ${max}`);
var json = generateJSON(uuid.v4());
- axios.put(argv.s+"/listings/", json)
+ axios.post(argv.s+"/listings/", json)
.then(() => {
var installs = Math.floor(Math.random()*argv.i);
console.log(`Generating ${installs} install records for listing '${json.id}'`);
@@ -82,7 +86,7 @@ function createCategory(count) {
return;
}
- axios.put(argv.s+"/categories/", generateCategoryJSON(categoryIds[count++]))
+ instance.put(argv.s+"/categories/", generateCategoryJSON(categoryIds[count++]))
.then(() => createCategory(count))
.catch(err => console.log(err));
}
@@ -92,7 +96,7 @@ function createMarket(count) {
return;
}
- axios.put(argv.s+"/markets/", generateMarketJSON(marketIds[count++]))
+ instance.put(argv.s+"/markets/", generateMarketJSON(marketIds[count++]))
.then(() => createMarket(count))
.catch(err => console.log(err));
}
@@ -102,7 +106,7 @@ function createInstall(curr, max, listing, callback) {
return callback();
}
var json = generateInstallJSON(listing);
- axios.post(`${argv.s}/installs/${json['listing_id']}/${json.version}`, json)
+ instance.post(`${argv.s}/installs/${json['listing_id']}/${json.version}`, json)
.then(createInstall(curr+1,max,listing,callback))
.catch(err => console.log(err));
}
@@ -170,8 +174,8 @@ function generateMarketJSON(id) {
function generateInstallJSON(listing) {
var version = listing.versions[Math.floor(Math.random()*listing.versions.length)];
- var javaVersions = javaVs.splice(javaVs.indexOf(version["min_java_version"]));
- var eclipseVersions = eclipseVs.splice(eclipseVs.indexOf(version["eclipse_version"]));
+ var javaVersions = Array.from(javaVs).splice(javaVs.indexOf(version["min_java_version"]));
+ var eclipseVersions = Array.from(eclipseVs).splice(eclipseVs.indexOf(version["eclipse_version"]));
return {
"listing_id": listing.id,
--
GitLab