Commit a303acca authored by Martin Lowe's avatar Martin Lowe 🇨🇦 Committed by Christopher Guindon
Browse files

Updated projects to properly fetch paginated results+precache on start



Added layer between validation and projects API layer. New service for
pagination of projects. This uses manually incrementing pages rather
than using link headers. This is not easily done within the RESTeasy API
builder paradigm, and will be taken as technical debt to fix. Added
separate cache layer to solve issue with long load times and always
available data with Loading rather than on demand cache.
Signed-off-by: Martin Lowe's avatarMartin Lowe <martin.lowe@eclipse-foundation.org>
parent 15e1cb24
......@@ -14,10 +14,10 @@
<maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<quarkus-plugin.version>1.1.1.Final</quarkus-plugin.version>
<quarkus-plugin.version>1.3.0.Final</quarkus-plugin.version>
<quarkus.platform.artifact-id>quarkus-universe-bom</quarkus.platform.artifact-id>
<quarkus.platform.group-id>io.quarkus</quarkus.platform.group-id>
<quarkus.platform.version>1.1.1.Final</quarkus.platform.version>
<quarkus.platform.version>1.3.0.Final</quarkus.platform.version>
<surefire-plugin.version>2.22.1</surefire-plugin.version>
<sonar.sources>src/main</sonar.sources>
<sonar.tests>src/test</sonar.tests>
......
......@@ -14,6 +14,7 @@ import java.util.List;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import org.eclipse.microprofile.rest.client.inject.RegisterRestClient;
import org.eclipsefoundation.git.eca.model.Project;
......@@ -37,5 +38,5 @@ public interface ProjectsAPI {
*/
@GET
@Produces("application/json")
List<Project> getProject();
List<Project> getProject(@QueryParam("page") int page, @QueryParam("pagesize") int pageSize);
}
......@@ -40,6 +40,7 @@ public class SecretConfigSource implements ConfigSource {
private Map<String, String> secrets;
@Override
@SuppressWarnings({"rawtypes", "unchecked"})
public Map<String, String> getProperties() {
if (secrets == null) {
this.secrets = new HashMap<>();
......
......@@ -26,7 +26,6 @@ import javax.ws.rs.core.Response;
import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.git.eca.api.AccountsAPI;
import org.eclipsefoundation.git.eca.api.BotsAPI;
import org.eclipsefoundation.git.eca.api.ProjectsAPI;
import org.eclipsefoundation.git.eca.helper.CommitHelper;
import org.eclipsefoundation.git.eca.model.BotUser;
import org.eclipsefoundation.git.eca.model.Commit;
......@@ -39,6 +38,7 @@ import org.eclipsefoundation.git.eca.namespace.APIStatusCode;
import org.eclipsefoundation.git.eca.namespace.ProviderType;
import org.eclipsefoundation.git.eca.service.CachingService;
import org.eclipsefoundation.git.eca.service.OAuthService;
import org.eclipsefoundation.git.eca.service.ProjectsService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -63,9 +63,6 @@ public class ValidationResource {
AccountsAPI accounts;
@Inject
@RestClient
ProjectsAPI projects;
@Inject
@RestClient
BotsAPI bots;
// external API/service harnesses
......@@ -73,7 +70,9 @@ public class ValidationResource {
OAuthService oauth;
@Inject
CachingService cache;
@Inject
ProjectsService projects;
/**
* Consuming a JSON request, this method will validate all passed commits, using
* the repo URL and the repository provider. These commits will be validated to
......@@ -304,25 +303,24 @@ public class ValidationResource {
private List<Project> retrieveProjectsForRequest(ValidationRequest req) {
String repoUrl = req.getRepoUrl();
// check for all projects that make use of the given repo
@SuppressWarnings("unchecked")
Optional<List<Project>> cachedProjects = cache.get("projects", () -> projects.getProject(),
(Class<List<Project>>) (Object) List.class);
if (!cachedProjects.isPresent() || cachedProjects.get().isEmpty()) {
List<Project> availableProjects = projects.getProjects();
if (availableProjects == null || availableProjects.isEmpty()) {
return Collections.emptyList();
}
LOGGER.debug("Number of projects found: {}", availableProjects.size());
// filter the projects based on the repo URL. At least one repo in project must
// match the repo URL to be valid
if (ProviderType.GITLAB.equals(req.getProvider())) {
return cachedProjects.get().stream()
return availableProjects.stream()
.filter(p -> p.getGitlabRepos().stream().anyMatch(re -> re.getUrl().equals(repoUrl)))
.collect(Collectors.toList());
} else if (ProviderType.GITHUB.equals(req.getProvider())) {
return cachedProjects.get().stream()
return availableProjects.stream()
.filter(p -> p.getGithubRepos().stream().anyMatch(re -> re.getUrl().equals(repoUrl)))
.collect(Collectors.toList());
} else {
return cachedProjects.get().stream()
return availableProjects.stream()
.filter(p -> p.getRepos().stream().anyMatch(re -> re.getUrl().equals(repoUrl)))
.collect(Collectors.toList());
}
......
/*******************************************************************************
* Copyright (C) 2020 Eclipse Foundation
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
******************************************************************************/
package org.eclipsefoundation.git.eca.service;
import java.util.List;
import org.eclipsefoundation.git.eca.model.Project;
/**
* Intermediate layer between resource and API layers that handles retrieval of
* all projects and caching of that data for availability purposes.
*
* @author Martin Lowe
*
*/
public interface ProjectsService {
/**
* Retrieves all currently available projects from cache if available, otherwise
* going to API to retrieve a fresh copy of the data.
*
* @return list of projects available from API.
*/
List<Project> getProjects();
}
/*******************************************************************************
* Copyright (C) 2020 Eclipse Foundation
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
******************************************************************************/
package org.eclipsefoundation.git.eca.service.impl;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.git.eca.api.ProjectsAPI;
import org.eclipsefoundation.git.eca.model.Project;
import org.eclipsefoundation.git.eca.service.ProjectsService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import io.quarkus.runtime.Startup;
/**
* Projects service implementation that handles pagination of data manually, as
* well as makes use of a loading cache to have data be always available with as
* little latency to the user as possible.
*
* @author Martin Lowe
*/
@Startup
@ApplicationScoped
public class PagintationProjectsService implements ProjectsService {
private static final Logger LOGGER = LoggerFactory.getLogger(PagintationProjectsService.class);
@Inject
@RestClient
ProjectsAPI projects;
// this class has a separate cache as this data is long to load and should be
// always available.
LoadingCache<String, List<Project>> internalCache;
/**
* Initializes the internal loader cache and pre-populates the data with the one
* available key. If more than one key is used, eviction of previous results
* will happen and create degraded performance.
*/
@PostConstruct
public void init() {
// set up the internal cache
this.internalCache = CacheBuilder.newBuilder().maximumSize(1).refreshAfterWrite(3600, TimeUnit.SECONDS)
.build(new CacheLoader<String, List<Project>>() {
@Override
public List<Project> load(String key) throws Exception {
return getProjectsInternal();
}
});
// pre-cache the projects to reduce load time for other users
LOGGER.debug("Starting pre-cache of projects");
if (getProjects() == null) {
LOGGER.warn(
"Unable to populate pre-cache for Eclipse projects. Calls may experience degraded performance.");
}
LOGGER.debug("Completed pre-cache of projects assets");
}
@Override
public List<Project> getProjects() {
try {
return internalCache.get("projects");
} catch (ExecutionException e) {
throw new RuntimeException("Could not load Eclipse projects", e);
}
}
/**
* Logic for retrieving projects from API. Will loop until there are no more
* projects to be found
*
* @return list of projects for the
*/
private List<Project> getProjectsInternal() {
int page = 0;
int pageSize = 100;
List<Project> out = new LinkedList<>();
List<Project> in;
do {
page++;
in = projects.getProject(page, pageSize);
out.addAll(in);
} while (in != null && !in.isEmpty());
return out;
}
}
......@@ -11,6 +11,7 @@ package org.eclipsefoundation.git.eca.api;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import javax.annotation.PostConstruct;
......@@ -80,8 +81,8 @@ public class MockProjectsAPI implements ProjectsAPI {
}
@Override
public List<Project> getProject() {
return new ArrayList<>(src);
public List<Project> getProject(int page, int pageSize) {
return page == 1 ? new ArrayList<>(src) : Collections.emptyList();
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment