Fix category relationship with markets #37

Updated relationship that Markets have with Categories. Added logic to
market filters to inject categories based on the listings available to a
market. If the listing contains a category, it is then returned as an
"active" category.

Change-Id: Icfb15058ffabd39ff65c8f575e72087fc67c1313
Signed-off-by: Martin Lowe <martin.lowe@eclipse-foundation.org>

src/main/java/org/eclipsefoundation/marketplace/dto/Listing.java

@@ -57,6 +57,7 @@ public class Listing extends NodeBase {
 	private long updateDate;
 	@JsonbProperty(DatabaseFieldNames.LICENSE_TYPE)
 	private String license;
+	private List<String> marketIds;
 	private List<String> categoryIds;
 	private List<Category> categories;
 	private Organization organization;
@@ -71,6 +72,7 @@ public class Listing extends NodeBase {
 		this.authors = new ArrayList<>();
 		this.tags = new ArrayList<>();
 		this.versions = new ArrayList<>();
+		this.marketIds = new ArrayList<>();
 		this.categoryIds = new ArrayList<>();
 		this.categories = new ArrayList<>();
 	}
@@ -273,6 +275,21 @@ public class Listing extends NodeBase {
 		this.categoryIds = new ArrayList<>(categoryIds);
 	}
 
+
+	/**
+	 * @return the categoryIds
+	 */
+	public List<String> getMarketIds() {
+		return new ArrayList<>(marketIds);
+	}
+
+	/**
+	 * @param marketIds the categoryIds to set
+	 */
+	public void setMarketIds(List<String> marketIds) {
+		this.marketIds = new ArrayList<>(marketIds);
+	}
+
 	/**
 	 * @return the categories
 	 */

src/main/java/org/eclipsefoundation/marketplace/dto/Market.java

@@ -26,7 +26,6 @@ import io.quarkus.runtime.annotations.RegisterForReflection;
  */
 @RegisterForReflection
 public class Market extends NodeBase {
-	private List<String> categoryIds;
 	private List<Category> categories;
 
 
@@ -36,7 +35,6 @@ public class Market extends NodeBase {
 	 */
 	public Market() {
 		this.categories = new LinkedList<>();
-		this.categoryIds = new LinkedList<>();
 	}
 
 	/**
@@ -54,26 +52,11 @@ public class Market extends NodeBase {
 		this.categories = new ArrayList<>(categories);
 	}
 
-	/**
-	 * @return the categoryIds
-	 */
-	@JsonbTransient
-	public List<String> getCategoryIds() {
-		return new ArrayList<>(categoryIds);
-	}
-
-	/**
-	 * @param categoryIds the categoryIds to set
-	 */
-	public void setCategoryIds(List<String> categoryIds) {
-		this.categoryIds = new ArrayList<>(categoryIds);
-	}
-
 	@Override
 	public int hashCode() {
 		final int prime = 31;
 		int result = super.hashCode();
-		result = prime * result + Objects.hash(categories, categoryIds);
+		result = prime * result + Objects.hash(categories);
 		return result;
 	}
 
@@ -89,6 +72,6 @@ public class Market extends NodeBase {
 			return false;
 		}
 		Market other = (Market) obj;
-		return Objects.equals(categories, other.categories) && Objects.equals(categoryIds, other.categoryIds);
+		return Objects.equals(categories, other.categories);
 	}
 }

src/main/java/org/eclipsefoundation/marketplace/dto/codecs/ListingCodec.java

@@ -80,6 +80,7 @@ public class ListingCodec implements CollectibleCodec<Listing> {
 		doc.put(DatabaseFieldNames.CREATION_DATE, new Date(value.getCreationDate()));
 		doc.put(DatabaseFieldNames.FOUNDATION_MEMBER_FLAG, value.isFoundationMember());
 		doc.put(DatabaseFieldNames.CATEGORY_IDS, value.getCategoryIds());
+		doc.put(DatabaseFieldNames.MARKET_IDS, value.getMarketIds());
 
 		// for nested document types, use the converters to safely transform into BSON
 		// documents
@@ -118,6 +119,7 @@ public class ListingCodec implements CollectibleCodec<Listing> {
 		out.setFavoriteCount(document.getLong(DatabaseFieldNames.MARKETPLACE_FAVORITES));
 		out.setFoundationMember(document.getBoolean(DatabaseFieldNames.FOUNDATION_MEMBER_FLAG));
 		out.setCategoryIds(document.getList(DatabaseFieldNames.CATEGORY_IDS, String.class));
+		out.setMarketIds(document.getList(DatabaseFieldNames.MARKET_IDS, String.class));
 
 		// for nested document types, use the converters to safely transform into POJO
 		out.setAuthors(document.getList(DatabaseFieldNames.LISTING_AUTHORS, Document.class).stream()

src/main/java/org/eclipsefoundation/marketplace/dto/codecs/MarketCodec.java

@@ -7,6 +7,7 @@
 package org.eclipsefoundation.marketplace.dto.codecs;
 
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 import org.bson.BsonReader;
@@ -19,6 +20,7 @@ import org.bson.codecs.CollectibleCodec;
 import org.bson.codecs.DecoderContext;
 import org.bson.codecs.EncoderContext;
 import org.eclipsefoundation.marketplace.dto.Market;
+import org.eclipsefoundation.marketplace.dto.converters.CategoryConverter;
 import org.eclipsefoundation.marketplace.namespace.DatabaseFieldNames;
 
 import com.mongodb.MongoClient;
@@ -32,6 +34,7 @@ import com.mongodb.MongoClient;
  */
 public class MarketCodec implements CollectibleCodec<Market> {
 	private final Codec<Document> documentCodec;
+	private final CategoryConverter categoryConverter;
 
 	/**
 	 * Creates the codec and initializes the codecs and converters needed to create
@@ -39,6 +42,7 @@ public class MarketCodec implements CollectibleCodec<Market> {
 	 */
 	public MarketCodec() {
 		this.documentCodec = MongoClient.getDefaultCodecRegistry().get(Document.class);
+		this.categoryConverter = new CategoryConverter();
 	}
 	
 	@Override
@@ -48,7 +52,6 @@ public class MarketCodec implements CollectibleCodec<Market> {
 		doc.put(DatabaseFieldNames.DOCID, value.getId());
 		doc.put(DatabaseFieldNames.URL, value.getUrl());
 		doc.put(DatabaseFieldNames.TITLE, value.getTitle());
-		doc.put(DatabaseFieldNames.CATEGORY_IDS, value.getCategoryIds());
 
 		documentCodec.encode(writer, doc, encoderContext);
 	}
@@ -61,10 +64,13 @@ public class MarketCodec implements CollectibleCodec<Market> {
 	@Override
 	public Market decode(BsonReader reader, DecoderContext decoderContext) {
 		Document document = documentCodec.decode(reader, decoderContext);
+
 		Market out = new Market();
 		out.setId(document.getString(DatabaseFieldNames.DOCID));
 		out.setUrl(document.getString(DatabaseFieldNames.URL));
 		out.setTitle(document.getString(DatabaseFieldNames.TITLE));
+		out.setCategories(document.getList(DatabaseFieldNames.LISTING_CATEGORIES, Document.class).stream()
+				.map(categoryConverter::convert).collect(Collectors.toList()));
 		
 		return out;
 	}

src/main/java/org/eclipsefoundation/marketplace/dto/filter/MarketFilter.java

@@ -6,19 +6,28 @@
  */
 package org.eclipsefoundation.marketplace.dto.filter;
 
+import static com.mongodb.client.model.Filters.and;
+import static com.mongodb.client.model.Filters.eq;
+import static com.mongodb.client.model.Filters.expr;
+
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
 import javax.enterprise.context.ApplicationScoped;
 
+import org.bson.BsonArray;
 import org.bson.conversions.Bson;
 import org.eclipsefoundation.marketplace.dto.Market;
 import org.eclipsefoundation.marketplace.model.RequestWrapper;
 import org.eclipsefoundation.marketplace.namespace.DatabaseFieldNames;
 import org.eclipsefoundation.marketplace.namespace.DtoTableNames;
 
+import com.mongodb.client.model.Accumulators;
 import com.mongodb.client.model.Aggregates;
+import com.mongodb.client.model.Projections;
+import com.mongodb.client.model.Variable;
 
 /**
  * Filter implementation for the {@linkplain Market} class.
@@ -36,10 +45,48 @@ public class MarketFilter implements DtoFilter<Market> {
 	@Override
 	public List<Bson> getAggregates(RequestWrapper wrap) {
 		List<Bson> aggs = new ArrayList<>();
+
+		String tempFieldName = "tmp";
+		List<Bson> pipeline = new ArrayList<>();
+		// match the listings on the given market_id
+		pipeline.add(
+				Aggregates.match(expr(eq("$in", Arrays.asList("$$market_id", "$" + DatabaseFieldNames.MARKET_IDS)))));
+		// suppress all fields except category_ids
+		pipeline.add(Aggregates.project(
+				Projections.fields(Projections.excludeId(), Projections.include(DatabaseFieldNames.CATEGORY_IDS))));
+
+		// set up a var reference for the _id
+		Variable<String> id = new Variable<String>("market_id", "$" + DatabaseFieldNames.DOCID);
+		// lookup all category IDS from listings with the given market ID
+		aggs.add(Aggregates.lookup(DtoTableNames.LISTING.getTableName(), Arrays.asList(id), pipeline, tempFieldName));
+		// explode all category IDS for collection
+		aggs.add(Aggregates.unwind("$" + tempFieldName));
+
+		// flatten categories using projection, and retain original data through data
+		// field
+		aggs.add(Aggregates.group("$_id", Accumulators.first("data", "$$ROOT"),
+				Accumulators.push(tempFieldName, "$" + tempFieldName + "." + DatabaseFieldNames.CATEGORY_IDS)));
+
+		// no reduction shortcuts in driver, build documents from scratch
+		// in operation merges multiple lists using sets to deduplicate
+		Bson inOperation = eq("$setUnion", Arrays.asList("$$value", "$$this"));
+		Bson reductionOptions = eq(tempFieldName, eq("$reduce",
+				and(eq("input", "$" + tempFieldName), eq("initialValue", new BsonArray()), eq("in", inOperation))));
+
+		// using projections, retain data-root + tmp category IDS and reduce them
+		aggs.add(Aggregates.project(Projections.fields(Projections.include("data"), reductionOptions)));
+
+		// create custom array as mergeObjects uses non-standard syntax
+		BsonArray ba = BsonArray.parse("[ '$data', {'" + tempFieldName + "': '$" + tempFieldName + "'}]");
+		// replaceRoot to restore original root data + set data for category IDs
+		aggs.add(Aggregates.replaceRoot(eq("$mergeObjects", ba)));
+
 		// adds a $lookup aggregate, joining categories on categoryIDS as "categories"
-		aggs.add(Aggregates.lookup(DtoTableNames.CATEGORY.getTableName(), DatabaseFieldNames.CATEGORY_IDS, DatabaseFieldNames.DOCID,
+		aggs.add(Aggregates.lookup(DtoTableNames.CATEGORY.getTableName(), tempFieldName, DatabaseFieldNames.DOCID,
 				"categories"));
-		
+
+		// remove the unneeded temporary field
+		aggs.add(Aggregates.project(Projections.exclude(tempFieldName)));
 		return aggs;
 	}
 

src/main/node/index.js

@@ -27,11 +27,11 @@ const platforms = ["windows","macos","linux"];
 const eclipseVs = ["4.6","4.7","4.8","4.9","4.10","4.11","4.12"];
 const javaVs = ["1.5", "1.6", "1.7", "1.8", "1.9", "1.10"];
 const categoryIds = [];
-for (var i=0;i<20;i++) {
+for (var i=0;i<200;i++) {
   categoryIds.push(uuid.v4());
 }
 const marketIds = [];
-for (i=0;i<5;i++) {
+for (var i=0;i<5;i++) {
   marketIds.push(uuid.v4());
 }
 
@@ -147,7 +147,8 @@ function generateJSON(id) {
   		}
   	],
   	"versions": solutions,
-  	"category_ids": splice(categoryIds).splice(0,Math.ceil(Math.random()*5)+1)
+	"market_ids": splice(marketIds).splice(0,Math.ceil(Math.random()*2)),
+  	"category_ids": splice(categoryIds).splice(0,Math.ceil(Math.random()*5))
   };
 }
 
@@ -163,8 +164,7 @@ function generateMarketJSON(id) {
   return {
     "id": id,
     "name": randomWords({exactly:1, wordsPerString:Math.ceil(Math.random()*4)})[0],
-    "url": "https://www.eclipse.org",
-    "category_ids": splice(categoryIds).splice(0,Math.ceil(Math.random()*5)+1)
+    "url": "https://www.eclipse.org"
   };
 }