Skip to content
Snippets Groups Projects
Commit 3ce8b1bf authored by Martin Lowe's avatar Martin Lowe :flag_ca:
Browse files

Merge branch 'malowe/main/test-fix' into 'main' 

Refactor CVE service to remove unneeded extra service

See merge request !17
parents f74a5654 7791cd74
No related branches found
No related tags found
1 merge request!17Refactor CVE service to remove unneeded extra service
Pipeline #11796 passed
Hide whitespace changes
Inline Side-by-side
Showing
with 144 additions and 214 deletions
src/main/java/org/eclipsefoundation/cve/api/GitlabCveAPI.java
+ 2
− 2
View file @ 3ce8b1bf
......@@ -45,7 +45,7 @@ public interface GitlabCveAPI {
public static abstract class GitlabRequestParams {
@PathParam("id")
public abstract String getId();
public abstract Integer getId();
@PathParam("filePath")
public abstract String getFilePath();
......@@ -64,7 +64,7 @@ public interface GitlabCveAPI {
@JsonPOJOBuilder(withPrefix = "set")
public static abstract class Builder {
public abstract Builder setId(String id);
public abstract Builder setId(Integer id);
public abstract Builder setFilePath(String filePath);
......
src/main/java/org/eclipsefoundation/cve/config/GitlabCveLoaderConfig.java 0 → 100644
+ 17
− 0
View file @ 3ce8b1bf
package org.eclipsefoundation.cve.config;
import io.smallrye.config.ConfigMapping;
import io.smallrye.config.WithDefault;
@ConfigMapping(prefix = "eclipse.gitlab.cve-project")
public interface GitlabCveLoaderConfig {
@WithDefault("advisories")
String filePath();
Integer projectId();
@WithDefault("main")
String ref();
String token();
}
src/main/java/org/eclipsefoundation/cve/service/CveCachingService.java deleted 100644 → 0
+ 0
− 39
View file @ f74a5654
/*********************************************************************
* Copyright (c) 2022 Eclipse Foundation.
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* Author: Zachary Sabourin <zachary.sabourin@eclipse-foundation.org>
*
* SPDX-License-Identifier: EPL-2.0
**********************************************************************/
package org.eclipsefoundation.cve.service;
import java.util.List;
import org.eclipsefoundation.cve.model.CveData;
import org.eclipsefoundation.cve.model.CveProjectData;
/**
* Service for caching and fetching GitLab and Github CVE data
*/
public interface CveCachingService {
/**
* Retreives data from GL cache using desired key
*
* @param key the cache key
* @return the cached/fetched CVE data
*/
List<CveData> getLoadedGitlabData(String key);
/**
* Retreives data from GH cache using desired key
*
* @param key the cache key
* @return the cached/fetched CVE data
*/
CveProjectData getLoadedGithubData(String key);
}
src/main/java/org/eclipsefoundation/cve/service/impl/DefaultCveCachingService.java deleted 100644 → 0
+ 0
− 167
View file @ f74a5654
/*********************************************************************
* Copyright (c) 2022 Eclipse Foundation.
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* Author: Zachary Sabourin <zachary.sabourin@eclipse-foundation.org>
*
* SPDX-License-Identifier: EPL-2.0
**********************************************************************/
package org.eclipsefoundation.cve.service.impl;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.PostConstruct;
import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.eclipse.microprofile.context.ManagedExecutor;
import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.cve.api.GithubCveAPI;
import org.eclipsefoundation.cve.api.GitlabCveAPI;
import org.eclipsefoundation.cve.api.GitlabCveAPI.GitlabRequestParams;
import org.eclipsefoundation.cve.model.CveData;
import org.eclipsefoundation.cve.model.CveProjectData;
import org.eclipsefoundation.cve.service.CveCachingService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import com.github.benmanes.caffeine.cache.Caffeine;
import io.quarkus.runtime.Startup;
@Startup
@ApplicationScoped
public class DefaultCveCachingService implements CveCachingService {
public static final Logger LOGGER = LoggerFactory.getLogger(DefaultCveCachingService.class);
public static final Pattern CVE_ID_PARTS = Pattern.compile("^CVE-(\\d{4})-(\\d+?)\\d{3}$");
@ConfigProperty(name = "eclipse.gitlab.cve-project")
Map<String, String> configProperties;
@Inject
@RestClient
GitlabCveAPI gitlabApi;
@Inject
@RestClient
GithubCveAPI githubApi;
@Inject
ObjectMapper om;
@Inject
ManagedExecutor executor;
private AsyncLoadingCache<String, CveProjectData> githubLoadingCache;
private AsyncLoadingCache<String, List<CveData>> gitlabLoadingCache;
/**
* Builds async caches, sets the refresh timers, and loads data.
*/
@PostConstruct
public void init() {
this.gitlabLoadingCache = Caffeine
.newBuilder()
.executor(executor)
.maximumSize(10)
.refreshAfterWrite(Duration.ofMinutes(60))
.buildAsync(this::loadGitlabData);
this.githubLoadingCache = Caffeine
.newBuilder()
.executor(executor)
.maximumSize(100)
.refreshAfterWrite(Duration.ofMinutes(60))
.buildAsync(this::loadGithubData);
preLoadCaches();
}
@Override
public List<CveData> getLoadedGitlabData(String key) {
try {
return gitlabLoadingCache.get(key).get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException("Could not get cached GL CVE data", e);
}
}
@Override
public CveProjectData getLoadedGithubData(String key) {
try {
return githubLoadingCache.get(key).get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(String.format("Could not get cached GH CVE data for id: %s", key), e);
}
}
/**
* Pre-loads both the GH and GL loading caches.
*/
private void preLoadCaches() {
getLoadedGitlabData("all").stream().forEach(cve -> {
if (!cve.getId().isEmpty()) {
getLoadedGithubData(cve.getId());
}
});
}
/**
* Loads GL cache by attempting to fetch CVE data from GL API.
*
* @return the fetched CveData entities
*/
private List<CveData> loadGitlabData(String key) {
try {
LOGGER.info("Loading GL data for key: {}", key);
GitlabRequestParams params = GitlabRequestParams.builder()
.setId(configProperties.get("project-id"))
.setFilePath(configProperties.get("file-path"))
.setRef(configProperties.get("ref"))
.setPrivateToken(configProperties.get("token"))
.build();
return om.readerForListOf(CveData.class).readValue(gitlabApi.getCveFile(params));
} catch (Exception e) {
throw new RuntimeException("Could not fetch CVE data from GitLab", e);
}
}
/**
* Loads GH cache by attempting to fetch CVE data from GH API.
*
* @param key the cache key
* @return The fetched CVE data
*
*/
private CveProjectData loadGithubData(String key) {
try {
Matcher matcher = CVE_ID_PARTS.matcher(key);
if (!matcher.matches()) {
LOGGER.warn("CVE passed with '{}' could not be parsed as a valid CVE ID, returning empty data", key);
return null;
}
LOGGER.info("Loading GH data for key: {}", key);
return om.readerFor(CveProjectData.class)
.readValue(githubApi.getCveDetails(matcher.group(1), matcher.group(2), key));
} catch (Exception e) {
throw new RuntimeException(String.format("Could not fetch CVE %s from GitHub", key), e);
}
}
}
src/main/java/org/eclipsefoundation/cve/service/impl/DefaultCveService.java
+ 122
− 6
View file @ 3ce8b1bf
......@@ -13,40 +13,156 @@
package org.eclipsefoundation.cve.service.impl;
import java.time.Duration;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.PostConstruct;
import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.microprofile.context.ManagedExecutor;
import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.cve.api.GithubCveAPI;
import org.eclipsefoundation.cve.api.GitlabCveAPI;
import org.eclipsefoundation.cve.api.GitlabCveAPI.GitlabRequestParams;
import org.eclipsefoundation.cve.config.GitlabCveLoaderConfig;
import org.eclipsefoundation.cve.model.CveData;
import org.eclipsefoundation.cve.model.CveProjectData;
import org.eclipsefoundation.cve.service.CveCachingService;
import org.eclipsefoundation.cve.service.CveService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import com.github.benmanes.caffeine.cache.Caffeine;
import io.quarkus.runtime.Startup;
/**
* Default implementation of the CVE Service
* Default implementation of the CVE Service. Uses loading caches to reduce delays from slower fetching and processing
* of raw data from public APIs.
*
* @author Martin Lowe, Zachary Sabourin
*
*/
@Startup
@ApplicationScoped
public class DefaultCveService implements CveService {
public static final Logger LOGGER = LoggerFactory.getLogger(DefaultCveService.class);
public static final Pattern CVE_ID_PARTS = Pattern.compile("^CVE-(\\d{4})-(\\d+?)\\d{3}$");
@Inject
GitlabCveLoaderConfig glCveLoaderConfig;
@Inject
@RestClient
GitlabCveAPI gitlabApi;
@Inject
@RestClient
GithubCveAPI githubApi;
@Inject
ObjectMapper om;
@Inject
CveCachingService cachingService;
ManagedExecutor executor;
private AsyncLoadingCache<String, CveProjectData> cveProjectCache;
private AsyncLoadingCache<String, List<CveData>> advisoriesCache;
/**
* Builds async caches, sets the refresh timers, and loads data.
*/
@PostConstruct
public void init() {
this.advisoriesCache = Caffeine
.newBuilder()
.executor(executor)
.maximumSize(10)
.refreshAfterWrite(Duration.ofMinutes(60))
.buildAsync(k -> loadAdvisoriesData());
this.cveProjectCache = Caffeine
.newBuilder()
.executor(executor)
.maximumSize(1000)
.refreshAfterWrite(Duration.ofMinutes(60))
.buildAsync(this::loadCveProjectData);
getAllCves().stream().forEach(cve -> {
if (StringUtils.isNotBlank(cve.getId())) {
getCveDetails(cve.getId());
}
});
}
@Override
public List<CveData> getAllCves() {
LOGGER.debug("Fetching CVE data from GitLab");
return cachingService.getLoadedGitlabData("all");
try {
return advisoriesCache.get("all").get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException("Could not get cached GL CVE data", e);
}
}
@Override
public CveProjectData getCveDetails(String id) {
LOGGER.debug("Fetching CVE {} from GitHub", id);
return cachingService.getLoadedGithubData(id);
}
try {
return cveProjectCache.get(id).get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(String.format("Could not get cached GH CVE data for id: %s", id), e);
}
}
/**
* Loads GL cache by attempting to fetch CVE data from GL API.
*
* @return the fetched CveData entities
*/
private List<CveData> loadAdvisoriesData() {
try {
LOGGER.info("Loading CVE advisories data from Gitlab");
GitlabRequestParams params = GitlabRequestParams
.builder()
.setId(glCveLoaderConfig.projectId())
.setFilePath(glCveLoaderConfig.filePath())
.setRef(glCveLoaderConfig.ref())
.setPrivateToken(glCveLoaderConfig.token())
.build();
return om.readerForListOf(CveData.class).readValue(gitlabApi.getCveFile(params));
} catch (Exception e) {
throw new RuntimeException("Could not fetch CVE data from GitLab", e);
}
}
/**
* Loads GH cache by attempting to fetch CVE data from GH API.
*
* @param key the cache key
* @return The fetched CVE data
*
*/
private CveProjectData loadCveProjectData(String key) {
try {
Matcher matcher = CVE_ID_PARTS.matcher(key);
if (!matcher.matches()) {
LOGGER.warn("CVE passed with '{}' could not be parsed as a valid CVE ID, returning empty data", key);
return null;
}
LOGGER.info("Loading GH data for key: {}", key);
return om.readerFor(CveProjectData.class).readValue(githubApi.getCveDetails(matcher.group(1), matcher.group(2), key));
} catch (Exception e) {
throw new RuntimeException(String.format("Could not fetch CVE %s from GitHub", key), e);
}
}
}
src/main/resources/application.properties
+ 3
− 0
View file @ 3ce8b1bf
......@@ -4,3 +4,6 @@ quarkus.rest-client."org.eclipsefoundation.cve.api.GitlabCveAPI".url=https://git
#eclipse.cve.provider=stubbed
quarkus.oidc.enabled=false
quarkus.keycloak.devservices.enabled=false
eclipse.gitlab.cve-project.file-path=advisories
eclipse.gitlab.cve-project.ref=main
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Copyright © Eclipse Foundation, Inc. All Rights Reserved.     Privacy Policy | Terms of Use | Copyright Agent