Merge branch 'zs/zacharysabourin/main/13' into 'main'

feat: Add loading cache for GH CVE data

Closes #13

See merge request !14

Makefile

@@ -8,7 +8,7 @@ setup:;
 	@source .env && rm -f ./config/application/secret.properties && envsubst < config/application/secret.properties.sample > config/application/secret.properties
 
 dev-start:;
-	mvn compile -e quarkus:dev
+	mvn compile -e quarkus:dev -Dconfig.secret.path=$$PWD/config/application/secret.properties
 
 clean:;
 	mvn clean

pom.xml

@@ -17,7 +17,7 @@
     <auto-value.version>1.8.2</auto-value.version>
     <hibernate.version>5.5.6.Final</hibernate.version>
     <org.mapstruct.version>1.4.1.Final</org.mapstruct.version>
-    <eclipse-api-version>0.6.6-SNAPSHOT</eclipse-api-version>
+    <eclipse-api-version>0.6.6</eclipse-api-version>
   </properties>
   <repositories>
     <repository>

src/main/java/org/eclipsefoundation/cve/api/GithubCveAPI.java

@@ -20,7 +20,6 @@ import javax.ws.rs.core.Response;
 
 import org.eclipse.microprofile.rest.client.inject.RegisterRestClient;
 
-@ApplicationScoped
 @RegisterRestClient
 @Consumes("application/json")
 public interface GithubCveAPI {

src/main/java/org/eclipsefoundation/cve/config/CveSourceServiceProvider.java

-/*******************************************************************************
- * Copyright (C) 2022 Eclipse Foundation
- * 
- * This program and the accompanying materials are made
- * available under the terms of the Eclipse Public License 2.0
- * which is available at https://www.eclipse.org/legal/epl-2.0/
- * 
- * SPDX-License-Identifier: EPL-2.0
- ******************************************************************************/
-package org.eclipsefoundation.cve.config;
-
-import javax.enterprise.context.Dependent;
-import javax.enterprise.inject.Produces;
-import javax.inject.Inject;
-
-import org.eclipse.microprofile.rest.client.inject.RestClient;
-import org.eclipsefoundation.core.service.CachingService;
-import org.eclipsefoundation.cve.api.GithubCveAPI;
-import org.eclipsefoundation.cve.model.mapper.CveDataMapper;
-import org.eclipsefoundation.cve.service.CveSourceService;
-import org.eclipsefoundation.cve.service.GoogleAPIService;
-import org.eclipsefoundation.cve.service.impl.DefaultCveSourceService;
-import org.eclipsefoundation.cve.service.impl.StubbedCveSourceService;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import io.quarkus.arc.DefaultBean;
-import io.quarkus.arc.properties.IfBuildProperty;
-
-/**
- * Provider for CVE source service, allows for swapping back to stubbed service for tests/development to avoid having to
- * use real data.
- * 
- * @author Martin Lowe, Zachary Sabourin
- *
- */
-@Dependent
-public class CveSourceServiceProvider {
-    public static final String PROVIDER_TYPE_PROPERTY_NAME = "eclipse.cve.provider";
-
-    @Inject
-    @RestClient
-    GithubCveAPI cveDetailsApi;
-
-    @Produces
-    @DefaultBean
-    public CveSourceService defaultService(GoogleAPIService driveApi, CachingService cache, CveDataMapper mapper, ObjectMapper objectMapper) {
-        return new DefaultCveSourceService(driveApi, cveDetailsApi, cache, mapper, objectMapper);
-    }
-
-    @Produces
-    @IfBuildProperty(name = PROVIDER_TYPE_PROPERTY_NAME, stringValue = "stubbed")
-    public CveSourceService stubbedService() {
-        return new StubbedCveSourceService();
-    }
-}

src/main/java/org/eclipsefoundation/cve/service/CveSourceService.java

@@ -24,29 +24,32 @@ import org.slf4j.LoggerFactory;
 /**
  * Service for retrieving and augmenting CVE data for return in the API.
  * 
- * @author Martin Lowe
+ * @author Martin Lowe, Zachary Sabourin
  *
  */
 public interface CveSourceService {
     public static final Logger LOGGER = LoggerFactory.getLogger(CveSourceService.class);
 
     /**
-     * Gets all CVE data objects, including ones that are not considered completed/public knowledge.
+     * Gets all CVE data objects, including ones that are not considered
+     * completed/public knowledge.
      * 
      * @return all known CVE data objects.
      */
     List<CveData> getAllCves();
 
     /**
-     * Checks the cache for a CirclCve with a matching id. Hits the Circl API if no CVE found in cache.
+     * Checks the cache for a GH Cve with a matching id. Hits the GH API if no
+     * CVE found in cache.
      * 
      * @param id the id to match against CVEs.
      * @return the Circl CVE data or empty optional if it can't be found.
      */
-    Optional<CveProjectData> getCveDetails(String id);
+    CveProjectData getCveDetails(String id);
 
     /**
-     * Retrieves a list of all public CVEs. Any non-public CVEs will be filtered out for security.
+     * Retrieves a list of all public CVEs. Any non-public CVEs will be filtered out
+     * for security.
      * 
      * @return all public CVE entries.
      */
@@ -57,7 +60,8 @@ public interface CveSourceService {
     /**
      * Gets list of CVEs for given project name using case-insensitive matching.
      * 
-     * @param projectName name of the project to use as CVE filter (in form of 'technology.dash', as an example)
+     * @param projectName     name of the project to use as CVE filter (in form of
+     *                        'technology.dash', as an example)
      * @param includeInternal whether non-completed CVEs should be included
      * @return list of CVE entries that match the given filters.
      */
@@ -69,7 +73,7 @@ public interface CveSourceService {
     }
 
     /**
-     * Retreives a CVE with matching id. Appends additional Circl CVE summary and score.
+     * Retreives a CVE with matching id. Appends additional CVE summary and score.
      * 
      * @param id the id to match against CVEs.
      * @return CveData object or null.
@@ -79,8 +83,8 @@ public interface CveSourceService {
     }
 
     /**
-     * Does basic shared processing of CVEs for other methods. Returns in process stream for further processing or
-     * collection.
+     * Does basic shared processing of CVEs for other methods. Returns in process
+     * stream for further processing or collection.
      * 
      * @param includeInternal whether to include non-completed/internal CVE entries
      * @return stream of CVEs filtered on the status.
@@ -90,27 +94,27 @@ public interface CveSourceService {
     }
 
     /**
-     * Checks the cache for a CirclCve with a matching id. Hits the Circl API if no CVE found in cache.
+     * Checks the cache for a GH Cve with a matching id. Hits the GH API if no
+     * CVE found in cache.
      * 
      * @param id the id to match against CVEs.
-     * @return CirclCveData object or null.
+     * @return Modified CveData object or original.
      */
     default CveData augmentCveData(CveData orig) {
         try {
-            Optional<CveProjectData> cveDetails = getCveDetails(orig.getId());
-            if (cveDetails.isPresent()) {
+            CveProjectData cveDetails = getCveDetails(orig.getId());
+            if (cveDetails != null) {
                 return CveData
                         .copy(orig)
                         .setSummary(cveDetails
-                                .get()
                                 .getDescription()
                                 .getDescriptionData()
                                 .stream()
-                                .filter(dd -> dd.getLang().equalsIgnoreCase("en"))
+                                .filter(dd -> dd.getLang().equalsIgnoreCase("eng"))
                                 .findFirst()
                                 .orElse(LocalizedValue.builder().setLang("en").setValue("").build())
                                 .getValue())
-                        .setCvss(cveDetails.get().getImpact().isPresent() ? cveDetails.get().getImpact().get().getCvss().getBaseScore()
+                        .setCvss(cveDetails.getImpact().isPresent() ? cveDetails.getImpact().get().getCvss().getBaseScore()
                                 : null)
                         .build();
             }

src/main/java/org/eclipsefoundation/cve/service/impl/DefaultCveSourceService.java

@@ -12,17 +12,23 @@
 
 package org.eclipsefoundation.cve.service.impl;
 
+import java.io.IOException;
 import java.io.InputStream;
+import java.time.Duration;
 import java.util.Collections;
 import java.util.List;
-import java.util.Optional;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.zip.GZIPInputStream;
 
+import javax.annotation.PostConstruct;
+import javax.enterprise.context.ApplicationScoped;
+import javax.inject.Inject;
 import javax.ws.rs.core.Response;
 
+import org.eclipse.microprofile.context.ManagedExecutor;
+import org.eclipse.microprofile.rest.client.inject.RestClient;
 import org.eclipsefoundation.core.service.CachingService;
 import org.eclipsefoundation.cve.api.GithubCveAPI;
 import org.eclipsefoundation.cve.model.CveData;
@@ -33,54 +39,98 @@ import org.eclipsefoundation.cve.service.GoogleAPIService;
 import org.jboss.resteasy.specimpl.MultivaluedMapImpl;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+
+import io.quarkus.runtime.Startup;
 
 /**
  * Default implementation of the CVE Service
  * 
- * @author Martin Lowe
+ * @author Martin Lowe, Zachary Sabourin
  *
  */
+@Startup
+@ApplicationScoped
 public class DefaultCveSourceService implements CveSourceService {
     public static final Pattern CVE_ID_PARTS = Pattern.compile("^CVE-(\\d{4})-(\\d+?)\\d{3}$");
 
-    private GoogleAPIService api;
-    private GithubCveAPI cveDetailsApi;
-    private CachingService cache;
-    private CveDataMapper mapper;
-    private ObjectMapper om;
+    @Inject
+    GoogleAPIService api;
+    @Inject
+    CachingService cacheService;
+    @Inject
+    CveDataMapper mapper;
+    @Inject
+    ObjectMapper om;
+    @Inject
+    ManagedExecutor executor;
+
+    @Inject
+    @RestClient
+    GithubCveAPI cveDetailsApi;
+
+    private AsyncLoadingCache<String, CveProjectData> loadingCache;
 
     /**
-     * Constructor that loads in all of the external services required to retrieve CVE data.
+     * Builds async cache, sets the expiry and refresh timers, and loads data.
      */
-    public DefaultCveSourceService(GoogleAPIService api, GithubCveAPI cveDetailsApi, CachingService cache, CveDataMapper mapper,
-            ObjectMapper om) {
-        this.api = api;
-        this.mapper = mapper;
-        this.cveDetailsApi = cveDetailsApi;
-        this.cache = cache;
-        this.om = om;
+    @PostConstruct
+    public void init() {
+        this.loadingCache = Caffeine
+                .newBuilder()
+                .executor(executor)
+                .maximumSize(50)
+                .expireAfterWrite(Duration.ofMinutes(60))
+                .refreshAfterWrite(Duration.ofMinutes(60))
+                .buildAsync(this::loadData);
     }
 
     @Override
     public List<CveData> getAllCves() {
-        return cache
+        return cacheService
                 .get("all", new MultivaluedMapImpl<>(), CveData.class,
                         () -> api.getCveSource().stream().map(mapper::toJsonModel).collect(Collectors.toList()))
                 .orElse(Collections.emptyList());
     }
 
     @Override
-    public Optional<CveProjectData> getCveDetails(String id) {
-        Matcher m = CVE_ID_PARTS.matcher(id);
-        if (!m.matches()) {
-            LOGGER.warn("CVE passed with '{}' could not be parsed as a valid CVE ID, returning empty data", id);
-            return Optional.empty();
+    public CveProjectData getCveDetails(String id) {
+        try {
+            return getLoadedData(id);
+        } catch (Exception e) {
+            throw new RuntimeException(String.format("Could not fetch CVE %s from GitHub", id), e);
         }
-        // get the cached data
-        return cache.get(id, new MultivaluedMapImpl<>(), CveProjectData.class, () -> {
-            Response r = cveDetailsApi.getCveDetails(m.group(1), m.group(2), id);
-            Object responseBody = r.getEntity();
-            return om.readerFor(CveProjectData.class).readValue(new GZIPInputStream((InputStream) responseBody));
-        });
+    }
+
+    /**
+     * Retreives data from cache using desired key
+     * 
+     * @param key the cache key
+     * @return the cached/fetched CVE data
+     * @throws Exception
+     */
+    private CveProjectData getLoadedData(String key) throws Exception {
+        return loadingCache.get(key).get();
+    }
+
+    /**
+     * Loads cache by attempting to fetch CVE data from GH API.
+     * 
+     * @param key the cache key
+     * @return The fetched CVE data
+     * @throws IOException
+     * 
+     */
+    private CveProjectData loadData(String key) throws IOException {
+        Matcher matcher = CVE_ID_PARTS.matcher(key);
+        if (!matcher.matches()) {
+            LOGGER.warn("CVE passed with '{}' could not be parsed as a valid CVE ID, returning empty data", key);
+            return null;
+        }
+
+        Response r = cveDetailsApi.getCveDetails(matcher.group(1), matcher.group(2), key);
+        Object responseBody = r.getEntity();
+        return om.readerFor(CveProjectData.class).readValue(new GZIPInputStream((InputStream) responseBody));
     }
 }

src/test/java/org/eclipsefoundation/cve/resources/CveResourceTest.java

@@ -3,29 +3,11 @@ package org.eclipsefoundation.cve.resources;
 import static io.restassured.RestAssured.given;
 import static io.restassured.module.jsv.JsonSchemaValidator.matchesJsonSchemaInClasspath;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import javax.enterprise.context.Dependent;
-import javax.enterprise.inject.Produces;
-
-import org.eclipsefoundation.cve.model.CveCSVData;
-import org.eclipsefoundation.cve.model.mapper.CveDataMapper;
-import org.eclipsefoundation.cve.service.GoogleAPIService;
-import org.eclipsefoundation.cve.service.impl.StubbedCveSourceService;
 import org.eclipsefoundation.cve.test.helpers.SchemaNamespaceHelper;
-import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.TestInstance.Lifecycle;
-import org.mapstruct.factory.Mappers;
-import org.mockito.Mockito;
-
-import com.google.api.services.drive.Drive;
-import com.google.inject.Inject;
 
-import io.quarkus.test.junit.QuarkusMock;
 import io.quarkus.test.junit.QuarkusTest;
 import io.restassured.http.ContentType;
 

src/test/java/org/eclipsefoundation/cve/service/impl/StubbedCveSourceServiceTest.java

-package org.eclipsefoundation.cve.service.impl;
-
-import java.util.List;
-
-import javax.inject.Inject;
-
-import org.eclipsefoundation.cve.model.CveData;
-import org.eclipsefoundation.cve.service.CveSourceService;
-import org.eclipsefoundation.cve.test.StubbedCVETestProfile;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import io.quarkus.test.junit.QuarkusTest;
-import io.quarkus.test.junit.TestProfile;
-
-@QuarkusTest
-@TestProfile(StubbedCVETestProfile.class)
-class StubbedCveSourceServiceTest {
-    public static final String PROJECT_NAME_PRESENT = "technology.dash";
-    public static final String PROJECT_NAME_MISSING = "radical.cowabunga";
-
-    @Inject
-    CveSourceService cveService;
-
-    @Test
-    void getClass_stubbedType() {
-        // ensure that stubbed service activates when config value set (see profile)
-        Assertions.assertEquals(cveService.getClass(), StubbedCveSourceService.class);
-    }
-
-    @Test
-    void getAllCves_success_notNull() {
-        // assert that the list is never null
-        Assertions.assertNotNull(cveService.getAllCves());
-    }
-
-    @Test
-    void getAllCves_success_containsActiveCve() {
-        // assert that the stubbed service will return at least one hidden cve for all
-        Assertions
-                .assertTrue(cveService
-                        .getAllCves()
-                        .stream()
-                        .anyMatch(cve -> !cve.getStatus().equalsIgnoreCase("complete")));
-    }
-
-    @Test
-    void getPublicCves_success_notNull() {
-        // assert that the list is never null
-        Assertions.assertNotNull(cveService.getPublicCves());
-    }
-
-    @Test
-    void getPublicCves_success_containsNoncompleteCve() {
-        // assert that the stubbed service will return at least one hidden cve for all
-        Assertions
-                .assertTrue(cveService
-                        .getPublicCves()
-                        .stream()
-                        .noneMatch(cve -> !cve.getStatus().equalsIgnoreCase("complete")));
-    }
-
-    @Test
-    void getCvesForProject_success() {
-        List<CveData> cves = cveService.getForProject(PROJECT_NAME_PRESENT, true);
-        Assertions.assertTrue(cves != null && !cves.isEmpty());
-        Assertions.assertTrue(cves.stream().allMatch(cve -> cve.getProject().equalsIgnoreCase(PROJECT_NAME_PRESENT)));
-    }
-
-    @Test
-    void getCvesForProject_success_noResults() {
-        List<CveData> cves = cveService.getForProject(PROJECT_NAME_MISSING, true);
-        Assertions.assertTrue(cves != null && cves.isEmpty());
-    }
-
-    @Test
-    void getCvesForProject_success_filtersNoncomplete() {
-        List<CveData> cves = cveService.getForProject(PROJECT_NAME_PRESENT, false);
-        Assertions.assertTrue(cves != null && !cves.isEmpty());
-        Assertions.assertTrue(cves.stream().allMatch(cve -> cve.getStatus().equalsIgnoreCase("complete")));
-    }
-
-    @Test
-    void getCvesForProject_success_allowsNoncomplete() {
-        List<CveData> cves = cveService.getForProject(PROJECT_NAME_PRESENT, true);
-        Assertions.assertTrue(cves != null && !cves.isEmpty());
-        Assertions.assertTrue(cves.stream().anyMatch(cve -> !cve.getStatus().equalsIgnoreCase("complete")));
-    }
-}

src/test/java/org/eclipsefoundation/cve/test/StubbedCVETestProfile.java

-package org.eclipsefoundation.cve.test;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.eclipsefoundation.cve.config.CveSourceServiceProvider;
-
-import io.quarkus.test.junit.QuarkusTestProfile;
-
-/**
- * Used to enable stubbed CVE service tests.
- * 
- * @author Martin Lowe
- */
-public class StubbedCVETestProfile implements QuarkusTestProfile {
-
-    // private immutable copy of the configs for auth state
-    private static final Map<String, String> CONFIG_OVERRIDES;
-    static {
-        Map<String, String> tmp = new HashMap<>();
-        tmp.put(CveSourceServiceProvider.PROVIDER_TYPE_PROPERTY_NAME, "stubbed");
-        CONFIG_OVERRIDES = Collections.unmodifiableMap(tmp);
-    }
-
-    @Override
-    public Map<String, String> getConfigOverrides() {
-        return CONFIG_OVERRIDES;
-    }
-}

src/main/java/org/eclipsefoundation/cve/service/impl/StubbedCveSourceService.java → src/test/java/org/eclipsefoundation/cve/test/service/impl/StubbedCveSourceService.java

@@ -10,7 +10,7 @@
  *  
  * SPDX-License-Identifier: EPL-2.0
  */
-package org.eclipsefoundation.cve.service.impl;
+package org.eclipsefoundation.cve.test.service.impl;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -22,6 +22,8 @@ import org.eclipsefoundation.cve.model.CveData;
 import org.eclipsefoundation.cve.model.CveProjectData;
 import org.eclipsefoundation.cve.service.CveSourceService;
 
+import io.quarkus.test.Mock;
+
 /**
  * Stubbed implementation of CVE data source service. This will provide faked and consistent CVE data for demo/testing
  * purposes.
@@ -29,6 +31,7 @@ import org.eclipsefoundation.cve.service.CveSourceService;
  * @author Martin Lowe, Zachary Sabourin
  *
  */
+@Mock
 public class StubbedCveSourceService implements CveSourceService {
     private List<CveData> internal;
 
@@ -83,7 +86,7 @@ public class StubbedCveSourceService implements CveSourceService {
     }
 
     @Override
-    public Optional<CveProjectData> getCveDetails(String id) {
-        return Optional.empty();
+    public CveProjectData getCveDetails(String id) {
+        return null;
     }
 }

src/test/java/org/eclipsefoundation/cve/test/service/impl/StubbedGoogleAPIService.java

@@ -7,7 +7,6 @@ import java.util.stream.Collectors;
 import org.eclipsefoundation.cve.model.CveCSVData;
 import org.eclipsefoundation.cve.model.mapper.CveDataMapper;
 import org.eclipsefoundation.cve.service.GoogleAPIService;
-import org.eclipsefoundation.cve.service.impl.StubbedCveSourceService;
 import org.mapstruct.factory.Mappers;
 
 import io.quarkus.test.Mock;

src/test/resources/application.properties

@@ -3,4 +3,6 @@ quarkus.oidc.enabled=false
 quarkus.keycloak.devservices.enabled=false
 
 eclipse.google.cve-file-id=sample
-eclipse.google.jwt-location=/tmp
\ No newline at end of file
+eclipse.google.jwt-location=/tmp
+
+quarkus.rest-client."org.eclipsefoundation.cve.api.GithubCveAPI".url=https://raw.githubusercontent.com/CVEProject/cvelist/master
\ No newline at end of file