diff --git a/package.json b/package.json
index 0de67f033cde395d4c5a6f2d7628654ec8d6f34f..16b124ded35c2ff50ff0bede6a79fcdec64c0628 100644
--- a/package.json
+++ b/package.json
@@ -11,6 +11,7 @@
"license": "EPL-2.0",
"dependencies": {
"axios": "^0.19.0",
+ "moment": "^2.24.0",
"moment-timezone": "^0.5.27",
"random-words": "^1.1.0",
"uuid": "^3.3.3",
diff --git a/src/main/java/org/eclipsefoundation/marketplace/config/JsonBConfig.java b/src/main/java/org/eclipsefoundation/marketplace/config/JsonBConfig.java
index 8d8f914cd46f750d39fa737f0ebde03de5f1be15..cb868867ddcffb024a17a5d0c30f87457baf041b 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/config/JsonBConfig.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/config/JsonBConfig.java
@@ -25,9 +25,10 @@ public class JsonBConfig implements ContextResolver<Jsonb> {
@Override
public Jsonb getContext(Class<?> type) {
JsonbConfig config = new JsonbConfig();
-
+
// following strategy is defined as default by internal API guidelines
- config.withPropertyNamingStrategy(PropertyNamingStrategy.LOWER_CASE_WITH_UNDERSCORES);
+ config.withPropertyNamingStrategy(PropertyNamingStrategy.LOWER_CASE_WITH_UNDERSCORES)
+ .withDateFormat("uuuu-MM-dd'T'HH:mm:ssXXX", null);
return JsonbBuilder.create(config);
}
}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dao/impl/DefaultMongoDao.java b/src/main/java/org/eclipsefoundation/marketplace/dao/impl/DefaultMongoDao.java
index d44a93fbc0922126b707b01984202132b1315b7c..892660e0314498ac6cf2869a06a491ea3ed51468 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/dao/impl/DefaultMongoDao.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/dao/impl/DefaultMongoDao.java
@@ -15,6 +15,7 @@ import javax.inject.Inject;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.eclipse.microprofile.health.HealthCheckResponse;
import org.eclipse.microprofile.health.HealthCheckResponseBuilder;
+import org.eclipse.microprofile.reactive.streams.operators.PublisherBuilder;
import org.eclipsefoundation.marketplace.dao.MongoDao;
import org.eclipsefoundation.marketplace.exception.MaintenanceException;
import org.eclipsefoundation.marketplace.model.MongoQuery;
@@ -59,10 +60,16 @@ public class DefaultMongoDao implements MongoDao {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Querying MongoDB using the following query: {}", q);
}
-
+
LOGGER.debug("Getting aggregate results");
- return getCollection(q.getDocType()).aggregate(q.getPipeline(getLimit(q)), q.getDocType()).limit(getLimit(q))
- .distinct().toList().run();
+ // build base query
+ PublisherBuilder<T> builder = getCollection(q.getDocType()).aggregate(q.getPipeline(getLimit(q)), q.getDocType());
+ // check if result set should be limited
+ if (q.getDTOFilter().useLimit()) {
+ builder = builder.limit(getLimit(q));
+ }
+ // run the query
+ return builder.distinct().toList().run();
}
@Override
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/Install.java b/src/main/java/org/eclipsefoundation/marketplace/dto/Install.java
index ab4a7ac675b94f8a12d8d732d604dc0161de205d..8a914f9d37d430dd4be80bc5bad831f43229057b 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/dto/Install.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/Install.java
@@ -135,4 +135,28 @@ public class Install {
public void setLocale(String locale) {
this.locale = locale;
}
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Install [id=");
+ builder.append(id);
+ builder.append(", installDate=");
+ builder.append(installDate);
+ builder.append(", os=");
+ builder.append(os);
+ builder.append(", version=");
+ builder.append(version);
+ builder.append(", listingId=");
+ builder.append(listingId);
+ builder.append(", javaVersion=");
+ builder.append(javaVersion);
+ builder.append(", eclipseVersion=");
+ builder.append(eclipseVersion);
+ builder.append(", locale=");
+ builder.append(locale);
+ builder.append("]");
+ return builder.toString();
+ }
+
}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/InstallMetrics.java b/src/main/java/org/eclipsefoundation/marketplace/dto/InstallMetrics.java
new file mode 100644
index 0000000000000000000000000000000000000000..5f2201d69c0cc7131c7af10ec0e6062459e6ec4a
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/InstallMetrics.java
@@ -0,0 +1,113 @@
+/* Copyright (c) 2019 Eclipse Foundation and others.
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Public License 2.0
+ * which is available at http://www.eclipse.org/legal/epl-v20.html,
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.marketplace.dto;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Holds a set of install metrics for the last year for a given listing.
+ *
+ * @author Martin Lowe
+ *
+ */
+public class InstallMetrics {
+
+ private String listingId;
+ private List<MetricPeriod> periods;
+ private int total;
+
+ public InstallMetrics() {
+ }
+
+ /**
+ * @param listingId
+ * @param periods
+ */
+ public InstallMetrics(String listingId, List<MetricPeriod> periods, int total) {
+ this.listingId = listingId;
+ this.periods = periods;
+ this.total = total;
+ }
+
+ /**
+ * @return the listingId
+ */
+ public String getListingId() {
+ return listingId;
+ }
+
+ /**
+ * @param listingId the listingId to set
+ */
+ public void setListingId(String listingId) {
+ this.listingId = listingId;
+ }
+
+ /**
+ * @return the periods
+ */
+ public List<MetricPeriod> getPeriods() {
+ return periods;
+ }
+
+ /**
+ * @param periods the periods to set
+ */
+ public void setPeriods(List<MetricPeriod> periods) {
+ this.periods = periods;
+ }
+
+ /**
+ * @return the total
+ */
+ public int getTotal() {
+ return total;
+ }
+
+ /**
+ * @param total the total to set
+ */
+ public void setTotal(int total) {
+ this.total = total;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(listingId, periods, total);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ InstallMetrics other = (InstallMetrics) obj;
+ return Objects.equals(listingId, other.listingId) && Objects.equals(periods, other.periods)
+ && Objects.equals(total, other.total);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("InstallMetrics [listingId=");
+ builder.append(listingId);
+ builder.append(", periods=");
+ builder.append(periods);
+ builder.append(", total=");
+ builder.append(total);
+ builder.append("]");
+ return builder.toString();
+ }
+
+}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/Listing.java b/src/main/java/org/eclipsefoundation/marketplace/dto/Listing.java
index 2a76e21563ef2a4932f55073d02067e4eb8b8c85..d52eb5dade2672742918c458b621d5b0dc49e448 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/dto/Listing.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/Listing.java
@@ -38,10 +38,10 @@ public class Listing extends NodeBase {
private boolean foundationMember;
@SortableField(name = "installs_count")
- private long installsTotal;
+ private Integer installsTotal;
@SortableField(name = "installs_count_recent")
- private long installsRecent;
+ private Integer installsRecent;
@SortableField
private long favoriteCount;
@@ -179,7 +179,7 @@ public class Listing extends NodeBase {
* @return the installsTotal
*/
@JsonbProperty("installs_count")
- public long getInstallsTotal() {
+ public Integer getInstallsTotal() {
return installsTotal;
}
@@ -187,7 +187,7 @@ public class Listing extends NodeBase {
* @param installsTotal the installsTotal to set
*/
@JsonbTransient
- public void setInstallsTotal(long installsTotal) {
+ public void setInstallsTotal(Integer installsTotal) {
this.installsTotal = installsTotal;
}
@@ -195,7 +195,7 @@ public class Listing extends NodeBase {
* @return the installsRecent
*/
@JsonbProperty("installs_count_recent")
- public long getInstallsRecent() {
+ public Integer getInstallsRecent() {
return installsRecent;
}
@@ -203,7 +203,7 @@ public class Listing extends NodeBase {
* @param installsRecent the installsRecent to set
*/
@JsonbTransient
- public void setInstallsRecent(long installsRecent) {
+ public void setInstallsRecent(Integer installsRecent) {
this.installsRecent = installsRecent;
}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/MetricPeriod.java b/src/main/java/org/eclipsefoundation/marketplace/dto/MetricPeriod.java
new file mode 100644
index 0000000000000000000000000000000000000000..187f131bdcc0161ff146eb8c707a8a701ef9c684
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/MetricPeriod.java
@@ -0,0 +1,98 @@
+/* Copyright (c) 2019 Eclipse Foundation and others.
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Public License 2.0
+ * which is available at http://www.eclipse.org/legal/epl-v20.html,
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.marketplace.dto;
+
+import java.util.Date;
+
+import javax.json.bind.annotation.JsonbTransient;
+
+/**
+ * Represents a set of install metrics for a given listing in a time period.
+ * This is represented without filters and cannot be filtered down.
+ *
+ * @author Martin Lowe
+ *
+ */
+public class MetricPeriod {
+
+ private String listingId;
+ private Integer count;
+ private Date start;
+ private Date end;
+
+ /**
+ * @return the listingId
+ */
+ @JsonbTransient
+ public String getListingId() {
+ return listingId;
+ }
+
+ /**
+ * @param listingId the listingId to set
+ */
+ public void setListingId(String listingId) {
+ this.listingId = listingId;
+ }
+
+ /**
+ * @return the count
+ */
+ public Integer getCount() {
+ return count;
+ }
+
+ /**
+ * @param count the count to set
+ */
+ public void setCount(Integer count) {
+ this.count = count;
+ }
+
+ /**
+ * @return the start
+ */
+ public Date getStart() {
+ return start;
+ }
+
+ /**
+ * @param start the start to set
+ */
+ public void setStart(Date start) {
+ this.start = start;
+ }
+
+ /**
+ * @return the end
+ */
+ public Date getEnd() {
+ return end;
+ }
+
+ /**
+ * @param end the end to set
+ */
+ public void setEnd(Date end) {
+ this.end = end;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("InstallMetric [listingId=");
+ builder.append(listingId);
+ builder.append(", count=");
+ builder.append(count);
+ builder.append(", start=");
+ builder.append(start);
+ builder.append(", end=");
+ builder.append(end);
+ builder.append("]");
+ return builder.toString();
+ }
+}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/codecs/InstallMetricsCodec.java b/src/main/java/org/eclipsefoundation/marketplace/dto/codecs/InstallMetricsCodec.java
new file mode 100644
index 0000000000000000000000000000000000000000..ef3a33ab4e1175227b6a93ca3025ca6775f8771f
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/codecs/InstallMetricsCodec.java
@@ -0,0 +1,176 @@
+/* Copyright (c) 2019 Eclipse Foundation and others.
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Public License 2.0
+ * which is available at http://www.eclipse.org/legal/epl-v20.html,
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.marketplace.dto.codecs;
+
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.bson.BsonReader;
+import org.bson.BsonString;
+import org.bson.BsonValue;
+import org.bson.BsonWriter;
+import org.bson.Document;
+import org.bson.codecs.Codec;
+import org.bson.codecs.CollectibleCodec;
+import org.bson.codecs.DecoderContext;
+import org.bson.codecs.EncoderContext;
+import org.eclipsefoundation.marketplace.dto.InstallMetrics;
+import org.eclipsefoundation.marketplace.dto.MetricPeriod;
+import org.eclipsefoundation.marketplace.dto.converters.MetricPeriodConverter;
+import org.eclipsefoundation.marketplace.namespace.DatabaseFieldNames;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.mongodb.MongoClient;
+
+/**
+ * Codec for getting and translating {@linkplain InstallMetrics} objects.
+ *
+ * @author Martin Lowe
+ *
+ */
+public class InstallMetricsCodec implements CollectibleCodec<InstallMetrics> {
+ private static final Logger LOGGER = LoggerFactory.getLogger(InstallMetricsCodec.class);
+ private final Codec<Document> documentCodec;
+
+ private MetricPeriodConverter periodConverter;
+
+ /**
+ * Creates the codec and initializes the codecs and converters needed to create
+ * a listing from end to end.
+ */
+ public InstallMetricsCodec() {
+ this.documentCodec = MongoClient.getDefaultCodecRegistry().get(Document.class);
+ this.periodConverter = new MetricPeriodConverter();
+ }
+
+ @Override
+ public void encode(BsonWriter writer, InstallMetrics value, EncoderContext encoderContext) {
+ Document doc = new Document();
+
+ doc.put(DatabaseFieldNames.DOCID, value.getListingId());
+ doc.put(DatabaseFieldNames.PERIOD_COUNT, value.getTotal());
+
+ // get the periods and sort them
+ List<MetricPeriod> mps = value.getPeriods();
+ mps.sort((o1, o2) -> o2.getEnd().compareTo(o1.getEnd()));
+
+ LOGGER.debug("Parsing periods for {}", value.getListingId());
+ // get calendar to check months
+ Calendar c = Calendar.getInstance();
+ int curr = 0;
+ for (int i = 0; i < 12; i++) {
+ // get the next period
+ MetricPeriod period;
+ if (curr < mps.size()) {
+ period = mps.get(curr);
+ LOGGER.debug("Got period: {}", period);
+ // check that the retrieved period is the same month
+ Calendar periodCalendar = Calendar.getInstance();
+ periodCalendar.setTime(period.getEnd());
+ if (periodCalendar.get(Calendar.MONTH) != c.get(Calendar.MONTH)) {
+ LOGGER.debug("Regenerating period, {}:{}", periodCalendar.get(Calendar.MONTH),
+ c.get(Calendar.MONTH));
+ // if the month doesn't match, get a new month
+ period = generatePeriod(value.getListingId(), c);
+ LOGGER.debug("Regenerated period, {}", period);
+ } else {
+ // increment the array index pointer once its used
+ curr++;
+ // increment after we get a period
+ c.add(Calendar.MONTH, -1);
+ }
+ } else {
+ period = generatePeriod(value.getListingId(), c);
+ LOGGER.debug("Generated period: {}", period);
+ }
+ // put the period into the document
+ doc.put(DatabaseFieldNames.MONTH_OFFSET_PREFIX + i, periodConverter.convert(period));
+ }
+ documentCodec.encode(writer, doc, encoderContext);
+ }
+
+ @Override
+ public InstallMetrics decode(BsonReader reader, DecoderContext decoderContext) {
+ Document document = documentCodec.decode(reader, decoderContext);
+
+ InstallMetrics out = new InstallMetrics();
+ out.setListingId(document.getString(DatabaseFieldNames.DOCID));
+ out.setTotal(document.getInteger(DatabaseFieldNames.PERIOD_COUNT));
+
+ // get the base calendar for the documents
+ Calendar c = getBaseCalendar(document);
+ // create a list of periods
+ List<MetricPeriod> periods = new ArrayList<>(12);
+ for (int i = 0; i < 12; i++) {
+ MetricPeriod period = periodConverter.convert(document.get(DatabaseFieldNames.MONTH_OFFSET_PREFIX + i, Document.class));
+ // if there is no period, generate one. otherwise, increment c and continue
+ if (period == null) {
+ period = generatePeriod(out.getListingId(), c);
+ } else {
+ c.add(Calendar.MONTH, -1);
+ }
+ periods.add(period);
+ }
+ out.setPeriods(periods);
+
+ return out;
+ }
+
+ @Override
+ public Class<InstallMetrics> getEncoderClass() {
+ return InstallMetrics.class;
+ }
+
+ @Override
+ public InstallMetrics generateIdIfAbsentFromDocument(InstallMetrics document) {
+ if (!documentHasId(document)) {
+ throw new IllegalArgumentException(
+ "A listing ID must be set to InstallMetrics objects before writing or they are invalid");
+ }
+ return document;
+ }
+
+ @Override
+ public boolean documentHasId(InstallMetrics document) {
+ return !StringUtils.isBlank(document.getListingId());
+ }
+
+ @Override
+ public BsonValue getDocumentId(InstallMetrics document) {
+ return new BsonString(document.getListingId());
+ }
+
+ private Calendar getBaseCalendar(Document d) {
+ for (int i = 0; i < 12; i++) {
+ MetricPeriod period = periodConverter.convert(d.get(DatabaseFieldNames.MONTH_OFFSET_PREFIX + i, Document.class));
+ // if we have a period set, get its date
+ if (period != null) {
+ Calendar out = Calendar.getInstance();
+ out.setTime(period.getEnd());
+ // adjust the calendar to the base date
+ out.add(Calendar.MONTH, i);
+ return out;
+ }
+ }
+ // fall back to now as the base time as there is no date to compare to
+ return Calendar.getInstance();
+ }
+
+ private MetricPeriod generatePeriod(String listingId, Calendar c) {
+ MetricPeriod period = new MetricPeriod();
+ period.setListingId(listingId);
+ period.setCount(0);
+ period.setEnd(c.getTime());
+ c.add(Calendar.MONTH, -1);
+ period.setStart(c.getTime());
+
+ return period;
+ }
+}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/codecs/ListingCodec.java b/src/main/java/org/eclipsefoundation/marketplace/dto/codecs/ListingCodec.java
index c8f956adbfdb9d5d96a26b6ab17579c4cc756daf..9870f5afb0b2121be9a451d77ba3d648c63c9fea 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/dto/codecs/ListingCodec.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/codecs/ListingCodec.java
@@ -72,8 +72,8 @@ public class ListingCodec implements CollectibleCodec<Listing> {
doc.put(DatabaseFieldNames.LISTING_BODY, value.getTeaser());
doc.put(DatabaseFieldNames.LISTING_TEASER, value.getBody());
doc.put(DatabaseFieldNames.MARKETPLACE_FAVORITES, value.getFavoriteCount());
- doc.put(DatabaseFieldNames.RECENT_NSTALLS, value.getInstallsRecent());
- doc.put(DatabaseFieldNames.TOTAL_NSTALLS, value.getInstallsTotal());
+ doc.put(DatabaseFieldNames.RECENT_INSTALLS, value.getInstallsRecent());
+ doc.put(DatabaseFieldNames.TOTAL_INSTALLS, value.getInstallsTotal());
doc.put(DatabaseFieldNames.LICENSE_TYPE, value.getLicense());
doc.put(DatabaseFieldNames.LISTING_STATUS, value.getStatus());
doc.put(DatabaseFieldNames.UPDATE_DATE, DateTimeHelper.toRFC3339(value.getUpdateDate()));
@@ -112,8 +112,8 @@ public class ListingCodec implements CollectibleCodec<Listing> {
out.setTeaser(document.getString(DatabaseFieldNames.LISTING_TEASER));
out.setBody(document.getString(DatabaseFieldNames.LISTING_BODY));
out.setStatus(document.getString(DatabaseFieldNames.LISTING_STATUS));
- out.setInstallsRecent(document.getLong(DatabaseFieldNames.RECENT_NSTALLS));
- out.setInstallsTotal(document.getLong(DatabaseFieldNames.TOTAL_NSTALLS));
+ out.setInstallsRecent(document.getInteger(DatabaseFieldNames.RECENT_INSTALLS));
+ out.setInstallsTotal(document.getInteger(DatabaseFieldNames.TOTAL_INSTALLS));
out.setLicense(document.getString(DatabaseFieldNames.LICENSE_TYPE));
out.setFavoriteCount(document.getLong(DatabaseFieldNames.MARKETPLACE_FAVORITES));
out.setFoundationMember(document.getBoolean(DatabaseFieldNames.FOUNDATION_MEMBER_FLAG));
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/codecs/MetricPeriodCodec.java b/src/main/java/org/eclipsefoundation/marketplace/dto/codecs/MetricPeriodCodec.java
new file mode 100644
index 0000000000000000000000000000000000000000..c4d4905cd63a386add61900e6b9fc614b68b8039
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/codecs/MetricPeriodCodec.java
@@ -0,0 +1,81 @@
+/* Copyright (c) 2019 Eclipse Foundation and others.
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Public License 2.0
+ * which is available at http://www.eclipse.org/legal/epl-v20.html,
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.marketplace.dto.codecs;
+
+import org.apache.commons.lang3.StringUtils;
+import org.bson.BsonReader;
+import org.bson.BsonString;
+import org.bson.BsonValue;
+import org.bson.BsonWriter;
+import org.bson.Document;
+import org.bson.codecs.Codec;
+import org.bson.codecs.CollectibleCodec;
+import org.bson.codecs.DecoderContext;
+import org.bson.codecs.EncoderContext;
+import org.eclipsefoundation.marketplace.dto.InstallMetrics;
+import org.eclipsefoundation.marketplace.dto.MetricPeriod;
+import org.eclipsefoundation.marketplace.dto.converters.MetricPeriodConverter;
+
+import com.mongodb.MongoClient;
+
+/**
+ * Codec for getting and translating {@linkplain MetricPeriod} objects. These do
+ * not represent a table but a section of the {@linkplain InstallMetrics} that
+ * are generated from the install table.
+ *
+ * @author Martin Lowe
+ *
+ */
+public class MetricPeriodCodec implements CollectibleCodec<MetricPeriod> {
+ private final Codec<Document> documentCodec;
+
+ private MetricPeriodConverter periodConverter;
+
+ /**
+ * Creates the codec and initializes the codecs and converters needed to create
+ * a listing from end to end.
+ */
+ public MetricPeriodCodec() {
+ this.documentCodec = MongoClient.getDefaultCodecRegistry().get(Document.class);
+ this.periodConverter = new MetricPeriodConverter();
+ }
+
+ @Override
+ public void encode(BsonWriter writer, MetricPeriod value, EncoderContext encoderContext) {
+ documentCodec.encode(writer, periodConverter.convert(value), encoderContext);
+ }
+
+ @Override
+ public Class<MetricPeriod> getEncoderClass() {
+ return MetricPeriod.class;
+ }
+
+ @Override
+ public MetricPeriod decode(BsonReader reader, DecoderContext decoderContext) {
+ return periodConverter.convert(documentCodec.decode(reader, decoderContext));
+ }
+
+ @Override
+ public MetricPeriod generateIdIfAbsentFromDocument(MetricPeriod document) {
+ if (!documentHasId(document)) {
+ throw new IllegalArgumentException(
+ "A listing ID must be set to MetricPeriod objects before writing or they are invalid");
+ }
+ return document;
+ }
+
+ @Override
+ public boolean documentHasId(MetricPeriod document) {
+ return !StringUtils.isBlank(document.getListingId());
+ }
+
+ @Override
+ public BsonValue getDocumentId(MetricPeriod document) {
+ return new BsonString(document.getListingId());
+ }
+
+}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/converters/MetricPeriodConverter.java b/src/main/java/org/eclipsefoundation/marketplace/dto/converters/MetricPeriodConverter.java
new file mode 100644
index 0000000000000000000000000000000000000000..9a1fe87b11a1e2fe2d5f851bc1745ae747f6f8a0
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/converters/MetricPeriodConverter.java
@@ -0,0 +1,40 @@
+/* Copyright (c) 2019 Eclipse Foundation and others.
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Public License 2.0
+ * which is available at http://www.eclipse.org/legal/epl-v20.html,
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.marketplace.dto.converters;
+
+import org.bson.Document;
+import org.eclipsefoundation.marketplace.dto.MetricPeriod;
+import org.eclipsefoundation.marketplace.namespace.DatabaseFieldNames;
+
+/**
+ * Converter implementation for the {@link MetricPeriod} object.
+ *
+ * @author Martin Lowe
+ */
+public class MetricPeriodConverter implements Converter<MetricPeriod> {
+
+ @Override
+ public MetricPeriod convert(Document src) {
+ MetricPeriod out = new MetricPeriod();
+ out.setListingId(src.getString(DatabaseFieldNames.DOCID));
+ out.setStart(src.getDate(DatabaseFieldNames.PERIOD_START));
+ out.setEnd(src.getDate(DatabaseFieldNames.PERIOD_END));
+ out.setCount(src.getInteger(DatabaseFieldNames.PERIOD_COUNT));
+ return out;
+ }
+
+ @Override
+ public Document convert(MetricPeriod src) {
+ Document doc = new Document();
+ doc.put(DatabaseFieldNames.DOCID, src.getListingId());
+ doc.put(DatabaseFieldNames.PERIOD_START, src.getStart());
+ doc.put(DatabaseFieldNames.PERIOD_END, src.getEnd());
+ doc.put(DatabaseFieldNames.PERIOD_COUNT, src.getCount());
+ return doc;
+ }
+
+}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/filter/DtoFilter.java b/src/main/java/org/eclipsefoundation/marketplace/dto/filter/DtoFilter.java
index c642a7d6de5155e07c6d15df645dd7361f9d3c4b..c2b2448b1e50769a68c4a777a0d155de2d95fad7 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/dto/filter/DtoFilter.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/filter/DtoFilter.java
@@ -71,4 +71,15 @@ public interface DtoFilter<T> {
default String getPath(String root, String fieldName) {
return StringUtils.isBlank(root) ? fieldName : root + '.' + fieldName;
}
+
+ /**
+ * Whether this type of data should be restrained to a limited set, or return
+ * all data that is found.
+ *
+ * @return true if limit should be used, false otherwise.
+ */
+ default boolean useLimit() {
+ return true;
+ }
+
}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/filter/InstallMetricsFilter.java b/src/main/java/org/eclipsefoundation/marketplace/dto/filter/InstallMetricsFilter.java
new file mode 100644
index 0000000000000000000000000000000000000000..38e75c081cbd92a0b6feed2be1c50b6bff033e56
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/filter/InstallMetricsFilter.java
@@ -0,0 +1,57 @@
+/* Copyright (c) 2019 Eclipse Foundation and others.
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Public License 2.0
+ * which is available at http://www.eclipse.org/legal/epl-v20.html,
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.marketplace.dto.filter;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+import javax.enterprise.context.ApplicationScoped;
+
+import org.bson.conversions.Bson;
+import org.eclipsefoundation.marketplace.dto.InstallMetrics;
+import org.eclipsefoundation.marketplace.model.RequestWrapper;
+import org.eclipsefoundation.marketplace.namespace.DatabaseFieldNames;
+import org.eclipsefoundation.marketplace.namespace.UrlParameterNames;
+
+import com.mongodb.client.model.Filters;
+
+/**
+ * Filter implementation for the {@linkplain InstallMetrics} class.
+ *
+ * @author Martin Lowe
+ *
+ */
+@ApplicationScoped
+public class InstallMetricsFilter implements DtoFilter<InstallMetrics> {
+
+ @Override
+ public List<Bson> getFilters(RequestWrapper wrap, String root) {
+ List<Bson> filters = new ArrayList<>();
+ // perform following checks only if there is no doc root
+ if (root == null) {
+ // ID check
+ Optional<String> id = wrap.getFirstParam(UrlParameterNames.ID);
+ if (id.isPresent()) {
+ filters.add(Filters.eq(DatabaseFieldNames.DOCID, id.get()));
+ }
+ }
+ return filters;
+ }
+
+ @Override
+ public List<Bson> getAggregates(RequestWrapper wrap) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public Class<InstallMetrics> getType() {
+ return InstallMetrics.class;
+ }
+
+}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/filter/ListingFilter.java b/src/main/java/org/eclipsefoundation/marketplace/dto/filter/ListingFilter.java
index ceac34a0e0a45267c44246f530ea1257b9c8ea9f..d773e722dcd49ff73772357fd0865474d2425329 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/dto/filter/ListingFilter.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/filter/ListingFilter.java
@@ -22,7 +22,9 @@ import org.eclipsefoundation.marketplace.namespace.DtoTableNames;
import org.eclipsefoundation.marketplace.namespace.UrlParameterNames;
import com.mongodb.client.model.Aggregates;
+import com.mongodb.client.model.Field;
import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.Projections;
/**
* Filter implementation for the {@linkplain Listing} class.
@@ -80,13 +82,21 @@ public class ListingFilter implements DtoFilter<Listing> {
aggs.add(Aggregates.lookup(DtoTableNames.LISTING_VERSION.getTableName(), DatabaseFieldNames.DOCID, DatabaseFieldNames.LISTING_ID,
DatabaseFieldNames.LISTING_VERSIONS));
aggs.addAll(listingVersionFilter.wrapFiltersToAggregate(wrap, DatabaseFieldNames.LISTING_VERSIONS));
- aggs.add(Aggregates.lookup(DtoTableNames.CATEGORY.getTableName(), DatabaseFieldNames.CATEGORY_IDS, DatabaseFieldNames.DOCID,
- DatabaseFieldNames.LISTING_CATEGORIES));
+ aggs.add(Aggregates.lookup(DtoTableNames.CATEGORY.getTableName(), DatabaseFieldNames.CATEGORY_IDS,
+ DatabaseFieldNames.DOCID, DatabaseFieldNames.LISTING_CATEGORIES));
List<String> marketIds = wrap.getParams(UrlParameterNames.MARKET_IDS);
if (!marketIds.isEmpty()) {
aggs.add(Aggregates.match(Filters.in("categories.market_ids", marketIds)));
}
-
+ // adds a $lookup aggregate, joining install metrics on ids as "installs"
+ aggs.add(Aggregates.lookup(DtoTableNames.INSTALL_METRIC.getTableName(), DatabaseFieldNames.DOCID,
+ DatabaseFieldNames.DOCID, "installs"));
+ // unwinds the installs out of arrays
+ aggs.add(Aggregates.unwind("$installs"));
+ // push the installs counts to the listing, and remove the installs merged in
+ aggs.add(Aggregates.addFields(new Field<String>(DatabaseFieldNames.RECENT_INSTALLS, "$installs.offset_0.count"),
+ new Field<String>(DatabaseFieldNames.TOTAL_INSTALLS, "$installs.count")));
+ aggs.add(Aggregates.project(Projections.exclude("installs")));
return aggs;
}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/filter/MetricPeriodFilter.java b/src/main/java/org/eclipsefoundation/marketplace/dto/filter/MetricPeriodFilter.java
new file mode 100644
index 0000000000000000000000000000000000000000..1f6b9bbc8e7f1f3911eb7d58318a5eecf3b1547a
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/filter/MetricPeriodFilter.java
@@ -0,0 +1,99 @@
+/* Copyright (c) 2019 Eclipse Foundation and others.
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Public License 2.0
+ * which is available at http://www.eclipse.org/legal/epl-v20.html,
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.marketplace.dto.filter;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+import javax.enterprise.context.ApplicationScoped;
+
+import org.bson.BsonArray;
+import org.bson.BsonDocument;
+import org.bson.BsonString;
+import org.bson.conversions.Bson;
+import org.eclipsefoundation.marketplace.dto.MetricPeriod;
+import org.eclipsefoundation.marketplace.model.RequestWrapper;
+import org.eclipsefoundation.marketplace.namespace.DatabaseFieldNames;
+import org.eclipsefoundation.marketplace.namespace.UrlParameterNames;
+
+import com.mongodb.client.model.Aggregates;
+import com.mongodb.client.model.BsonField;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.Projections;
+
+/**
+ * Filter implementation for the {@linkplain MetricPeriod} class.
+ *
+ * @author Martin Lowe
+ *
+ */
+@ApplicationScoped
+public class MetricPeriodFilter implements DtoFilter<MetricPeriod> {
+
+ @Override
+ public List<Bson> getFilters(RequestWrapper wrap, String root) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<Bson> getAggregates(RequestWrapper wrap) {
+ // check that we have required fields first
+ Optional<String> startDate = wrap.getFirstParam(UrlParameterNames.START);
+ Optional<String> endDate = wrap.getFirstParam(UrlParameterNames.END);
+ List<Bson> aggregates = new ArrayList<>();
+ if (startDate.isPresent() && endDate.isPresent()) {
+ // check for all listings that are after the start date
+ BsonArray startDateComparison = new BsonArray();
+ startDateComparison.add(new BsonString("$" + DatabaseFieldNames.INSTALL_DATE));
+ BsonDocument startDoc = new BsonDocument();
+ startDoc.append("dateString", new BsonString(startDate.get()));
+ startDoc.append("format", new BsonString("%Y-%m-%dT%H:%M:%SZ"));
+ // build doc to convert string to date to be used in query
+ BsonDocument startDateConversion = new BsonDocument("$dateFromString", startDoc);
+ startDateComparison.add(startDateConversion);
+
+ // check for all listings that are before the end date
+ BsonArray endDateComparison = new BsonArray();
+ endDateComparison.add(new BsonString("$" + DatabaseFieldNames.INSTALL_DATE));
+ BsonDocument endDoc = new BsonDocument();
+ endDoc.append("dateString", new BsonString(endDate.get()));
+ endDoc.append("format", new BsonString("%Y-%m-%dT%H:%M:%SZ"));
+ // build doc to convert string to date to be used in query
+ BsonDocument endDateConversion = new BsonDocument("$dateFromString", endDoc);
+ endDateComparison.add(endDateConversion);
+
+ // add the 2 date comparisons to the pipeline
+ aggregates.add(Aggregates.match(Filters
+ .expr(Filters.eq("$and", new BsonArray(Arrays.asList(new BsonDocument("$gte", startDateComparison),
+ new BsonDocument("$lte", endDateComparison)))))));
+ // group the results by listing ID
+ aggregates.add(Aggregates.group("$listing_id", new BsonField(DatabaseFieldNames.PERIOD_COUNT, Filters.eq("$sum", 1))));
+ // project the start + end date into the end result
+ aggregates.add(Aggregates.project(Projections.fields(Projections.include(DatabaseFieldNames.PERIOD_COUNT),
+ Projections.computed(DatabaseFieldNames.PERIOD_START, startDateConversion),
+ Projections.computed(DatabaseFieldNames.PERIOD_END, endDateConversion))));
+ } else {
+ // count all existing installs and group them by listing ID
+ aggregates.add(Aggregates.group("$listing_id", new BsonField(DatabaseFieldNames.PERIOD_COUNT, Filters.eq("$sum", 1))));
+ }
+
+ return aggregates;
+ }
+
+ @Override
+ public Class<MetricPeriod> getType() {
+ return MetricPeriod.class;
+ }
+
+ @Override
+ public boolean useLimit() {
+ return false;
+ }
+}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/providers/InstallMetricsCodecProvider.java b/src/main/java/org/eclipsefoundation/marketplace/dto/providers/InstallMetricsCodecProvider.java
new file mode 100644
index 0000000000000000000000000000000000000000..b83a0c795b37eb7772a05e26284ee962e98ac8b1
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/providers/InstallMetricsCodecProvider.java
@@ -0,0 +1,35 @@
+/* Copyright (c) 2019 Eclipse Foundation and others.
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Public License 2.0
+ * which is available at http://www.eclipse.org/legal/epl-v20.html,
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.marketplace.dto.providers;
+
+import org.bson.codecs.Codec;
+import org.bson.codecs.configuration.CodecProvider;
+import org.bson.codecs.configuration.CodecRegistry;
+import org.eclipsefoundation.marketplace.dto.InstallMetrics;
+import org.eclipsefoundation.marketplace.dto.codecs.InstallMetricsCodec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Provides the {@link InstallMetricsCodec} to MongoDB for conversions of
+ * {@link InstallMetrics} objects.
+ *
+ * @author Martin Lowe
+ */
+public class InstallMetricsCodecProvider implements CodecProvider {
+ private static final Logger LOGGER = LoggerFactory.getLogger(InstallMetricsCodecProvider.class);
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> Codec<T> get(Class<T> clazz, CodecRegistry registry) {
+ if (clazz == InstallMetrics.class) {
+ LOGGER.debug("Registering custom InstallMetrics class MongoDB codec");
+ return (Codec<T>) new InstallMetricsCodec();
+ }
+ return null;
+ }
+}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/dto/providers/MetricPeriodCodecProvider.java b/src/main/java/org/eclipsefoundation/marketplace/dto/providers/MetricPeriodCodecProvider.java
new file mode 100644
index 0000000000000000000000000000000000000000..445a48a6e614f942f2e2375b4b3cca7f72273fb9
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/marketplace/dto/providers/MetricPeriodCodecProvider.java
@@ -0,0 +1,35 @@
+/* Copyright (c) 2019 Eclipse Foundation and others.
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Public License 2.0
+ * which is available at http://www.eclipse.org/legal/epl-v20.html,
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.marketplace.dto.providers;
+
+import org.bson.codecs.Codec;
+import org.bson.codecs.configuration.CodecProvider;
+import org.bson.codecs.configuration.CodecRegistry;
+import org.eclipsefoundation.marketplace.dto.MetricPeriod;
+import org.eclipsefoundation.marketplace.dto.codecs.MetricPeriodCodec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Provides the {@link MetricPeriodCodec} to MongoDB for conversions of
+ * {@link MetricPeriod} objects.
+ *
+ * @author Martin Lowe
+ */
+public class MetricPeriodCodecProvider implements CodecProvider {
+ private static final Logger LOGGER = LoggerFactory.getLogger(MetricPeriodCodecProvider.class);
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> Codec<T> get(Class<T> clazz, CodecRegistry registry) {
+ if (clazz == MetricPeriod.class) {
+ LOGGER.debug("Registering custom MetricPeriod class MongoDB codec");
+ return (Codec<T>) new MetricPeriodCodec();
+ }
+ return null;
+ }
+}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/helper/DateTimeHelper.java b/src/main/java/org/eclipsefoundation/marketplace/helper/DateTimeHelper.java
index d6b4157d6c445e202a21fb8f5f9e6221417d3ac1..16df77d4628f107444fd5ad9db6403a75107d253 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/helper/DateTimeHelper.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/helper/DateTimeHelper.java
@@ -43,7 +43,7 @@ public class DateTimeHelper {
return null;
}
}
-
+
/**
* Converts passed date to RFC 3339 compliant date string. Time is adjusted to
* be in UTC time.
diff --git a/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java b/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java
index d792d51a0d4a9d27e785fc8f1b01fac02894296f..1b487325947c97c789dc3f6fb21fa5b1d793956e 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/model/MongoQuery.java
@@ -61,13 +61,13 @@ public class MongoQuery<T> {
// clear old values if set to default
this.filter = null;
this.sort = null;
- this.order= SortOrder.NONE;
+ this.order = SortOrder.NONE;
this.aggregates = new ArrayList<>();
// get the filters for the current DTO
List<Bson> filters = new ArrayList<>();
filters.addAll(dtoFilter.getFilters(wrapper, null));
-
+
// get fields that make up the required fields to enable pagination and check
Optional<String> sortOpt = wrapper.getFirstParam(UrlParameterNames.SORT);
if (sortOpt.isPresent()) {
@@ -85,7 +85,7 @@ public class MongoQuery<T> {
this.filter = Filters.and(filters);
}
this.aggregates = dtoFilter.getAggregates(wrapper);
-
+
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("MongoDB query initialized with filter: {}", this.filter);
}
@@ -110,7 +110,7 @@ public class MongoQuery<T> {
// add base aggregates (joins)
out.addAll(aggregates);
// add sample if we aren't sorting
- if (sort == null || SortOrder.RANDOM.equals(order)) {
+ if ((sort == null || SortOrder.RANDOM.equals(order)) && dtoFilter.useLimit()) {
out.add(Aggregates.sample(limit));
}
if (sort != null) {
@@ -136,7 +136,7 @@ public class MongoQuery<T> {
private void setSort(String sortField, String sortOrder, List<Bson> filters) {
Optional<String> lastOpt = wrapper.getFirstParam(UrlParameterNames.LAST_SEEN);
-
+
List<Sortable<?>> fields = SortableHelper.getSortableFields(getDocType());
Optional<Sortable<?>> fieldContainer = SortableHelper.getSortableFieldByName(fields, sortField);
if (fieldContainer.isPresent()) {
@@ -180,6 +180,13 @@ public class MongoQuery<T> {
return this.filter;
}
+ /**
+ * @return the DTO filter
+ */
+ public DtoFilter<T> getDTOFilter() {
+ return this.dtoFilter;
+ }
+
/**
* @return the docType
*/
diff --git a/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java b/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java
index 0b473baad4f8fa5b416af7fae7c6a7f396be12e0..aec134a4a2ebf5f92680293a873c5e6c9f66b2eb 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/model/RequestWrapper.java
@@ -100,7 +100,6 @@ public class RequestWrapper {
* Adds the given value for the given key, preserving previous values if they
* exist.
*
- * @param wrapper map containing parameters to update
* @param key string key to add the value to, must not be null
* @param value the value to add to the key
*/
@@ -112,6 +111,23 @@ public class RequestWrapper {
getParams().computeIfAbsent(key, k -> new ArrayList<>()).add(value);
}
+ /**
+ * Sets the value as the value for the given key, removing previous values if
+ * they exist.
+ *
+ * @param key string key to add the value to, must not be null
+ * @param value the value to add to the key
+ */
+ public void setParam(String key, String value) {
+ if (StringUtils.isBlank(key)) {
+ throw new IllegalArgumentException(EMPTY_KEY_MESSAGE);
+ }
+ Objects.requireNonNull(value);
+ // remove current value, and add new value in its place
+ getParams().remove(key);
+ addParam(key, value);
+ }
+
/**
* Returns this QueryParams object as a Map of param values indexed by the param
* name.
diff --git a/src/main/java/org/eclipsefoundation/marketplace/namespace/DatabaseFieldNames.java b/src/main/java/org/eclipsefoundation/marketplace/namespace/DatabaseFieldNames.java
index a205252b94b30705869b6599f07bc7e65ae6c370..c27f12756193b8ab8a37a0fc72fb7f199e3298ec 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/namespace/DatabaseFieldNames.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/namespace/DatabaseFieldNames.java
@@ -30,8 +30,8 @@ public final class DatabaseFieldNames {
public static final String LISTING_AUTHORS = "authors";
public static final String LISTING_STATUS = "status";
public static final String LISTING_ORGANIZATIONS = "organization";
- public static final String RECENT_NSTALLS = "installs_recent";
- public static final String TOTAL_NSTALLS = "installs_total";
+ public static final String RECENT_INSTALLS = "installs_recent";
+ public static final String TOTAL_INSTALLS = "installs_total";
public static final String MARKETPLACE_FAVORITES = "favorite_count";
public static final String LICENSE_TYPE = "license_type";
public static final String PLATFORMS = "platforms";
@@ -78,6 +78,15 @@ public final class DatabaseFieldNames {
public static final String ECLIPSE_VERSION = "eclipse_version";
public static final String LOCALE = "locale";
+ // metric period fields
+ public static final String PERIOD_END = "period_end";
+ public static final String PERIOD_START = "period_start";
+ public static final String PERIOD_COUNT = "count";
+
+ // install metric fields
+ public static final String METRIC_PERIODS = "periods";
+ public static final String MONTH_OFFSET_PREFIX = "offset_";
+
private DatabaseFieldNames() {
}
}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/namespace/DtoTableNames.java b/src/main/java/org/eclipsefoundation/marketplace/namespace/DtoTableNames.java
index 6c3042818388b1e63c5e86c950e4d9c26be70b99..a3d92ef242999a641a750c1f35fd8526f72119d7 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/namespace/DtoTableNames.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/namespace/DtoTableNames.java
@@ -10,9 +10,11 @@ import org.eclipsefoundation.marketplace.dto.Catalog;
import org.eclipsefoundation.marketplace.dto.Category;
import org.eclipsefoundation.marketplace.dto.ErrorReport;
import org.eclipsefoundation.marketplace.dto.Install;
+import org.eclipsefoundation.marketplace.dto.InstallMetrics;
import org.eclipsefoundation.marketplace.dto.Listing;
import org.eclipsefoundation.marketplace.dto.ListingVersion;
import org.eclipsefoundation.marketplace.dto.Market;
+import org.eclipsefoundation.marketplace.dto.MetricPeriod;
/**
* Mapping of DTO classes to their respective tables in the DB.
@@ -23,6 +25,7 @@ import org.eclipsefoundation.marketplace.dto.Market;
public enum DtoTableNames {
LISTING(Listing.class, "listings"), CATEGORY(Category.class, "categories"), CATALOG(Catalog.class, "catalogs"),
MARKET(Market.class, "markets"), ERRORREPORT(ErrorReport.class, "errorreports"), INSTALL(Install.class, "installs"),
+ INSTALL_METRIC(InstallMetrics.class, "install_metrics"), METRIC_PERIOD(MetricPeriod.class, "installs"),
LISTING_VERSION(ListingVersion.class, "listing_versions");
private Class<?> baseClass;
diff --git a/src/main/java/org/eclipsefoundation/marketplace/namespace/UrlParameterNames.java b/src/main/java/org/eclipsefoundation/marketplace/namespace/UrlParameterNames.java
index 5148395fc5118a9b08312ea8a8e7f8895d056a64..a9352a63a1fb046f7a309752f9c1f58839b75a55 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/namespace/UrlParameterNames.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/namespace/UrlParameterNames.java
@@ -32,6 +32,8 @@ public final class UrlParameterNames {
public static final String FEATURE_ID = "feature_id";
public static final String VERSION = "version";
public static final String DATE_FROM = "from";
+ public static final String END = "end";
+ public static final String START = "start";
private UrlParameterNames() {
}
diff --git a/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java b/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java
index 01559ea81fc180a355167271b3c0898efb790248..8c1562d81d84b1df9593b4dbe56a3652bef9bdff 100644
--- a/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java
+++ b/src/main/java/org/eclipsefoundation/marketplace/resource/InstallResource.java
@@ -6,10 +6,17 @@
*/
package org.eclipsefoundation.marketplace.resource;
-import java.sql.Date;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Calendar;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
import javax.annotation.security.PermitAll;
import javax.annotation.security.RolesAllowed;
@@ -26,7 +33,10 @@ import javax.ws.rs.core.Response.Status;
import org.eclipsefoundation.marketplace.dao.MongoDao;
import org.eclipsefoundation.marketplace.dto.Install;
+import org.eclipsefoundation.marketplace.dto.InstallMetrics;
+import org.eclipsefoundation.marketplace.dto.MetricPeriod;
import org.eclipsefoundation.marketplace.dto.filter.DtoFilter;
+import org.eclipsefoundation.marketplace.helper.DateTimeHelper;
import org.eclipsefoundation.marketplace.helper.StreamHelper;
import org.eclipsefoundation.marketplace.model.Error;
import org.eclipsefoundation.marketplace.model.MongoQuery;
@@ -53,32 +63,38 @@ public class InstallResource {
MongoDao dao;
@Inject
RequestWrapper wrapper;
+
+ // insert required filters for different objects + states
@Inject
DtoFilter<Install> dtoFilter;
+ @Inject
+ DtoFilter<MetricPeriod> periodFilter;
+ @Inject
+ DtoFilter<InstallMetrics> metricFilter;
// Inject 2 caching service references, as we want to cache count results.
@Inject
CachingService<Long> countCache;
@Inject
- CachingService<List<Install>> installCache;
+ CachingService<List<InstallMetrics>> installCache;
/**
- * Endpoint for /listing/\<listingId\>/installs to retrieve install metrics for
- * a specific listing from the database.
+ * Endpoint for /installs/${listingId} to retrieve install counts for a specific
+ * listing from the database with given filters.
*
- * @param listingId int version of the listing ID
+ * @param listingId the listing ID
* @return response for the browser
*/
@GET
@PermitAll
@Path("/{listingId}")
- public Response selectInstallMetrics(@PathParam("listingId") String listingId) {
+ public Response selectInstallCount(@PathParam("listingId") String listingId) {
wrapper.addParam(UrlParameterNames.ID, listingId);
- MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, installCache);
+ MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, null);
Optional<Long> cachedResults = countCache.get(listingId, wrapper,
() -> StreamHelper.awaitCompletionStage(dao.count(q)));
if (!cachedResults.isPresent()) {
- LOGGER.error("Error while retrieving cached listing for ID {}", listingId);
+ LOGGER.error("Error while retrieving cached install metrics for ID {}", listingId);
return Response.serverError().build();
}
@@ -88,21 +104,20 @@ public class InstallResource {
/**
*
- * Endpoint for /listing/\<listingId\>/installs/\<version\> to retrieve install
- * metrics for a specific listing version from the database.
+ * Endpoint for /installs/${listingId}/${version} to retrieve install counts for
+ * a specific listing version from the database.
*
- * @param listingId int version of the listing ID
- * @param version int version of the listing version number
+ * @param listingId the listing ID
+ * @param version the listing version number
* @return response for the browser
*/
@GET
@PermitAll
@Path("/{listingId}/{version}")
- public Response selectInstallMetrics(@PathParam("listingId") String listingId,
- @PathParam("version") String version) {
+ public Response selectInstallCount(@PathParam("listingId") String listingId, @PathParam("version") String version) {
wrapper.addParam(UrlParameterNames.ID, listingId);
wrapper.addParam(UrlParameterNames.VERSION, version);
- MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, installCache);
+ MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, null);
Optional<Long> cachedResults = countCache.get(getCompositeKey(listingId, version), wrapper,
() -> StreamHelper.awaitCompletionStage(dao.count(q)));
if (!cachedResults.isPresent()) {
@@ -115,11 +130,35 @@ public class InstallResource {
}
/**
- * Endpoint for /listing/\<listingId\>/installs/\<version\> to post install
- * metrics for a specific listing version to a database.
+ * Endpoint for /installs/${listingId}/metrics to retrieve install metrics for a
+ * specific listing from the database.
*
- * @param listingId int version of the listing ID
- * @param version int version of the listing version number
+ * @param listingId the listing ID
+ * @return response for the browser
+ */
+ @GET
+ @PermitAll
+ @Path("/{listingId}/metrics")
+ public Response selectInstallMetrics(@PathParam("listingId") String listingId) {
+ wrapper.addParam(UrlParameterNames.ID, listingId);
+ MongoQuery<InstallMetrics> q = new MongoQuery<>(wrapper, metricFilter, null);
+ Optional<List<InstallMetrics>> cachedResults = installCache.get(listingId, wrapper,
+ () -> StreamHelper.awaitCompletionStage(dao.get(q)));
+ if (!cachedResults.isPresent()) {
+ LOGGER.error("Error while retrieving cached install metrics for ID {}", listingId);
+ return Response.serverError().build();
+ }
+
+ // return the results as a response
+ return Response.ok(cachedResults.get()).build();
+ }
+
+ /**
+ * Endpoint for /installs/${listingId}/${version} to post install metrics for a
+ * specific listing version to a database.
+ *
+ * @param listingId the listing ID
+ * @param version the listing version number
* @return response for the browser
*/
@POST
@@ -145,12 +184,11 @@ public class InstallResource {
}
// update the install details to reflect the current request
- record.setInstallDate(new Date(System.currentTimeMillis()));
record.setListingId(listingId);
record.setVersion(version);
// create the query wrapper to pass to DB dao
- MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, installCache);
+ MongoQuery<Install> q = new MongoQuery<>(wrapper, dtoFilter, null);
// add the object, and await the result
StreamHelper.awaitCompletionStage(dao.add(q, Arrays.asList(record)));
@@ -159,6 +197,80 @@ public class InstallResource {
return Response.ok().build();
}
+ /**
+ * Regenerates the install_metrics table, using the install table as the base.
+ * Creates 12 columns for metric periods, to provide users with a count of
+ * installs over the past year. For months with no installs, an empty metric
+ * period is generated to avoid gaps in the stats.
+ *
+ * TODO: This should be moved to a separate job resource and be callable through
+ * a service that tracks last run time. https://github.com/EclipseFdn/marketplace-rest-api/issues/54
+ *
+ * @return an OK response when finished
+ */
+ @GET
+ @RolesAllowed("marketplace_admin_access")
+ @Path("/generate_metrics")
+ public Response generateInstallStats() {
+ List<CompletionStage<List<MetricPeriod>>> stages = new ArrayList<>();
+ // get total install count for all listings available
+ Map<String, Integer> overallCounts = new HashMap<>();
+ CompletionStage<List<MetricPeriod>> stage = dao.get(new MongoQuery<>(wrapper, periodFilter, null));
+ stage.whenComplete((metrics, e) -> {
+ // if theres an error, immediately stop processing
+ if (e != null) {
+ throw new RuntimeException(e);
+ }
+ // for each metric, insert total count into the map
+ for (MetricPeriod metric : metrics) {
+ overallCounts.put(metric.getListingId(), metric.getCount());
+ }
+ });
+ stages.add(stage);
+
+ // use thread safe map impl for storing metrics
+ Map<String, List<MetricPeriod>> r = new ConcurrentHashMap<>();
+ // get the last 12 months of stats for installs asynchronously
+ Calendar c = Calendar.getInstance();
+ for (int m = 0; m < 12; m++) {
+ // set up the date ranges for the current call
+ String end = DateTimeHelper.toRFC3339(c.getTime());
+ c.add(Calendar.MONTH, -1);
+ String start = DateTimeHelper.toRFC3339(c.getTime());
+ wrapper.setParam(UrlParameterNames.END, end);
+ wrapper.setParam(UrlParameterNames.START, start);
+
+ // create the query wrapper to pass to DB dao. No cache needed as this info
+ // won't be cached
+ MongoQuery<MetricPeriod> q = new MongoQuery<>(wrapper, periodFilter, null);
+ // run query, and set up a completion activity to record data
+ CompletionStage<List<MetricPeriod>> statStage = dao.get(q);
+ statStage.whenComplete((metrics, e) -> {
+ // if theres an error, immediately stop processing
+ if (e != null) {
+ throw new RuntimeException(e);
+ }
+ // for each metric, insert into the map
+ for (MetricPeriod metric : metrics) {
+ r.computeIfAbsent(metric.getListingId(), k -> new ArrayList<MetricPeriod>()).add(metric);
+ }
+ });
+ // keep stage reference to check when complete
+ stages.add(statStage);
+ }
+ // wrap futures and await all calls to finish
+ StreamHelper.awaitCompletionStage(CompletableFuture.allOf(stages.toArray(new CompletableFuture[] {})));
+
+ // convert the map to a list of install metric objects, adding in total count
+ List<InstallMetrics> installMetrics = r.entrySet().stream().map(entry -> new InstallMetrics(entry.getKey(),
+ entry.getValue(), overallCounts.getOrDefault(entry.getKey(), 0))).collect(Collectors.toList());
+
+ // push the content to the database, and await for it to finish
+ StreamHelper.awaitCompletionStage(dao.add(new MongoQuery<>(wrapper, metricFilter, null), installMetrics));
+ // return the results as a response
+ return Response.ok().build();
+ }
+
private String getCompositeKey(String listingId, String version) {
return listingId + ':' + version;
}
diff --git a/src/main/node/index.js b/src/main/node/index.js
index bb88462e57c5451747ee3d7d7516ea5f758eceaa..58b46c0b420a47dadefca838a274e54dd0eff7e2 100644
--- a/src/main/node/index.js
+++ b/src/main/node/index.js
@@ -1,6 +1,7 @@
const axios = require('axios');
+const time = require('moment');
const instance = axios.create({
- timeout: 2500,
+ timeout: 5000,
headers: {'User-Agent': 'mpc/0.0.0'}
});
const randomWords = require('random-words');
@@ -190,7 +191,10 @@ function generateInstallJSON(listing,versions) {
var javaVersions = Array.from(javaVs).splice(javaVs.indexOf(version["min_java_version"]));
var eclipseVersions = Array.from(eclipseVs).splice(eclipseVs.indexOf(version["eclipse_version"]));
+ var daysAgo = Math.floor(Math.random() * (365*2));
+
return {
+ "install_date": time().subtract(daysAgo, 'days').format(),
"listing_id": listing.id,
"version": version.version,
"java_version": shuff(javaVersions)[0],