Skip to content
Snippets Groups Projects
Commit 12034c03 authored by Martin Lowe's avatar Martin Lowe :flag_ca:
Browse files

Merge branch 'malowe/master/87' into 'master'

Iss #87 - Move validation logic to validation service in prep for webhook updates

See merge request !107
parents ceb411b2 2e2283f2
No related branches found
No related tags found
1 merge request!107Iss #87 - Move validation logic to validation service in prep for webhook updates
Pipeline #13069 failed
Showing
with 545 additions and 550 deletions
......@@ -11,10 +11,8 @@
**********************************************************************/
package org.eclipsefoundation.git.eca.api.models;
import javax.annotation.Nullable;
import org.eclipsefoundation.git.eca.model.GitUser;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
......@@ -38,7 +36,6 @@ public abstract class EclipseUser {
public abstract boolean getIsCommitter();
@Nullable
@JsonIgnore
public abstract Boolean getIsBot();
......@@ -49,8 +46,14 @@ public abstract class EclipseUser {
* @return a stubbed Eclipse user bot object.
*/
public static EclipseUser createBotStub(GitUser user) {
return EclipseUser.builder().setUid(0).setName(user.getName()).setMail(user.getMail())
.setECA(ECA.builder().build()).setIsBot(true).build();
return EclipseUser
.builder()
.setUid(0)
.setName(user.getName())
.setMail(user.getMail())
.setECA(ECA.builder().build())
.setIsBot(true)
.build();
}
public static Builder builder() {
......@@ -71,7 +74,7 @@ public abstract class EclipseUser {
public abstract Builder setIsCommitter(boolean isCommitter);
@JsonIgnore
public abstract Builder setIsBot(@Nullable Boolean isBot);
public abstract Builder setIsBot(boolean isBot);
public abstract EclipseUser build();
}
......@@ -79,11 +82,9 @@ public abstract class EclipseUser {
@AutoValue
@JsonDeserialize(builder = AutoValue_EclipseUser_ECA.Builder.class)
public abstract static class ECA {
@Nullable
public abstract Boolean getSigned();
public abstract boolean getSigned();
@Nullable
public abstract Boolean getCanContributeSpecProject();
public abstract boolean getCanContributeSpecProject();
public static Builder builder() {
return new AutoValue_EclipseUser_ECA.Builder().setCanContributeSpecProject(false).setSigned(false);
......@@ -92,9 +93,9 @@ public abstract class EclipseUser {
@AutoValue.Builder
@JsonPOJOBuilder(withPrefix = "set")
public abstract static class Builder {
public abstract Builder setSigned(@Nullable Boolean signed);
public abstract Builder setSigned(boolean signed);
public abstract Builder setCanContributeSpecProject(@Nullable Boolean canContributeSpecProject);
public abstract Builder setCanContributeSpecProject(boolean canContributeSpecProject);
public abstract ECA build();
}
......
......@@ -35,7 +35,6 @@ import com.google.auto.value.AutoValue;
@JsonNaming(LowerCamelCaseStrategy.class)
@JsonDeserialize(builder = AutoValue_ValidationRequest.Builder.class)
public abstract class ValidationRequest {
@Nullable
@JsonProperty("repoUrl")
public abstract URI getRepoUrl();
......@@ -55,7 +54,7 @@ public abstract class ValidationRequest {
@JsonPOJOBuilder(withPrefix = "set")
public abstract static class Builder {
@JsonProperty("repoUrl")
public abstract Builder setRepoUrl(@Nullable URI repoUrl);
public abstract Builder setRepoUrl(URI repoUrl);
public abstract Builder setCommits(List<Commit> commits);
......
......@@ -50,14 +50,30 @@ public abstract class ValidationResponse {
public boolean getPassed() {
return getErrorCount() <= 0;
};
}
@Memoized
public int getErrorCount() {
return getCommits().values().stream().mapToInt(s -> s.getErrors().size()).sum();
}
/** @param message message to add to the API response */
/**
* Add a message with a success status to the response array.
*
* @param hash the hash of the commit that the message applies to
* @param message the actual message contents
*/
public void addMessage(String hash, String message) {
addMessage(hash, message, APIStatusCode.SUCCESS_DEFAULT);
}
/**
* Add a message with the given status to the response message map.
*
* @param hash the hash of the commit that the message applies to
* @param message the actual message contents
* @param code the code to associate with the message
*/
public void addMessage(String hash, String message, APIStatusCode code) {
getCommits().computeIfAbsent(getHashKey(hash), k -> CommitStatus.builder().build()).addMessage(message, code);
}
......@@ -69,7 +85,11 @@ public abstract class ValidationResponse {
/** @param error message to add to the API response */
public void addError(String hash, String error, APIStatusCode code) {
getCommits().computeIfAbsent(getHashKey(hash), k -> CommitStatus.builder().build()).addError(error, code);
if (getTrackedProject()) {
getCommits().computeIfAbsent(getHashKey(hash), k -> CommitStatus.builder().build()).addError(error, code);
} else {
addWarning(hash, error, code);
}
}
public static String getHashKey(String hash) {
......@@ -79,8 +99,8 @@ public abstract class ValidationResponse {
/**
* Converts the APIResponse to a web response with appropriate status.
*
* @return a web response with status {@link Status.OK} if the commits pass
* validation, {@link Status.FORBIDDEN} otherwise.
* @return a web response with status {@link Status.OK} if the commits pass validation, {@link Status.FORBIDDEN}
* otherwise.
*/
public Response toResponse() {
// update error count before returning
......@@ -92,8 +112,11 @@ public abstract class ValidationResponse {
}
public static Builder builder() {
return new AutoValue_ValidationResponse.Builder().setStrictMode(false).setTrackedProject(false)
.setTime(ZonedDateTime.now()).setCommits(new HashMap<>());
return new AutoValue_ValidationResponse.Builder()
.setStrictMode(false)
.setTrackedProject(false)
.setTime(ZonedDateTime.now())
.setCommits(new HashMap<>());
}
@AutoValue.Builder
......
......@@ -14,21 +14,41 @@ package org.eclipsefoundation.git.eca.service;
import java.util.List;
import org.eclipsefoundation.git.eca.model.Project;
import org.eclipsefoundation.git.eca.model.ValidationRequest;
import org.eclipsefoundation.git.eca.namespace.ProviderType;
/**
* Intermediate layer between resource and API layers that handles retrieval of
* all projects and caching of that data for availability purposes.
* Intermediate layer between resource and API layers that handles retrieval of all projects and caching of that data
* for availability purposes.
*
* @author Martin Lowe
*
*/
public interface ProjectsService {
/**
* Retrieves all currently available projects from cache if available, otherwise
* going to API to retrieve a fresh copy of the data.
*
* @return list of projects available from API.
*/
List<Project> getProjects();
/**
* Retrieves all currently available projects from cache if available, otherwise going to API to retrieve a fresh copy
* of the data.
*
* @return list of projects available from API.
*/
List<Project> getProjects();
/**
* Retrieves projects valid for the current request, or an empty list if no data or matching project repos could be
* found.
*
* @param req the current request
* @return list of matching projects for the current request, or an empty list if none found.
*/
List<Project> retrieveProjectsForRequest(ValidationRequest req);
/**
* Retrieves projects for given provider, using the repo URL to match to a stored repository.
*
* @param repoUrl the repo URL to match
* @param provider the provider that is being served for the request.
* @return a list of matching projects, or an empty list if none are found.
*/
List<Project> retrieveProjectsForRepoURL(String repoUrl, ProviderType provider);
}
......@@ -24,55 +24,60 @@ import org.eclipsefoundation.git.eca.model.ValidationResponse;
import io.undertow.util.HexConverter;
/**
* Service containing logic for validating commits, and retrieving/updating previous validation statuses.
*
* @author martin
* @author Martin Lowe
*
*/
public interface ValidationService {
/**
* Retrieves a set of validation status objects given the validation request
* fingerprint.
* Validate an incoming request, checking to make sure that ECA is present for users interacting with the API, as well
* as maintain access rights to spec projects, blocking non-elevated access to the repositories.
*
* @param wrapper current request wrapper object
* @param req the request to validate
* @param wrapper the current request wrapper
* @return the validation results to return to the user.
*/
public ValidationResponse validateIncomingRequest(ValidationRequest req, RequestWrapper wrapper);
/**
* Retrieves a set of validation status objects given the validation request fingerprint.
*
* @param wrapper current request wrapper object
* @param fingerprint the validation request fingerprint
* @return the list of historic validation status objects, or an empty list.
*/
public List<CommitValidationStatus> getHistoricValidationStatus(RequestWrapper wrapper, String fingerprint);
/**
* Retrieves a set of commit validation status objects given a validation
* request and target project.
* Retrieves a set of commit validation status objects given a validation request and target project.
*
* @param wrapper current request wrapper object
* @param req the current validation request
* @param wrapper current request wrapper object
* @param req the current validation request
* @param projectId the project targeted by the validation request
* @return the list of existing validation status objects for the validation
* request, or an empty list.
* @return the list of existing validation status objects for the validation request, or an empty list.
*/
public List<CommitValidationStatus> getRequestCommitValidationStatus(RequestWrapper wrapper, ValidationRequest req,
String projectId);
public List<CommitValidationStatus> getRequestCommitValidationStatus(RequestWrapper wrapper, ValidationRequest req, String projectId);
/**
* Updates or creates validation status objects for the commits validated as
* part of the current validation request. Uses information from both the
* original request and the final response to generate details to be preserved
* in commit status objects.
* Updates or creates validation status objects for the commits validated as part of the current validation request.
* Uses information from both the original request and the final response to generate details to be preserved in commit
* status objects.
*
* @param wrapper current request wrapper object
* @param r the final validation response
* @param req the current validation request
* @param wrapper current request wrapper object
* @param r the final validation response
* @param req the current validation request
* @param statuses list of existing commit validation objects to update
* @param p the project targeted by the validation request.
* @param p the project targeted by the validation request.
*/
public void updateCommitValidationStatus(RequestWrapper wrapper, ValidationResponse r, ValidationRequest req,
List<CommitValidationStatus> statuses, Project p);
/**
* Generates a request fingerprint for looking up requests that have already
* been processed in the past. Collision here is extremely unlikely, and low
* risk on the change it does. For that reason, a more secure but heavier
* hashing alg. wasn't chosen.
* Generates a request fingerprint for looking up requests that have already been processed in the past. Collision here
* is extremely unlikely, and low risk on the change it does. For that reason, a more secure but heavier hashing alg.
* wasn't chosen.
*
* @param req the request to generate a fingerprint for
* @return the fingerprint for the request
......
......@@ -24,6 +24,7 @@ import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import javax.ws.rs.WebApplicationException;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.core.service.CachingService;
......@@ -77,6 +78,9 @@ public class CachedUserService implements UserService {
@Override
public EclipseUser getUser(String mail) {
if (StringUtils.isBlank(mail)) {
return null;
}
Optional<EclipseUser> u = cache.get(mail, new MultivaluedMapImpl<>(), EclipseUser.class,
() -> retrieveUser(mail));
if (u.isPresent()) {
......@@ -89,6 +93,9 @@ public class CachedUserService implements UserService {
@Override
public EclipseUser getUserByGithubUsername(String username) {
if (StringUtils.isBlank(username)) {
return null;
}
Optional<EclipseUser> u = cache.get("gh:" + username, new MultivaluedMapImpl<>(), EclipseUser.class,
() -> accounts.getUserByGithubUname(getBearerToken(), username));
if (u.isPresent()) {
......@@ -101,7 +108,7 @@ public class CachedUserService implements UserService {
@Override
public boolean userIsABot(String mail, List<Project> filteredProjects) {
if (mail == null || "".equals(mail.trim())) {
if (StringUtils.isBlank(mail)) {
return false;
}
List<JsonNode> botObjs = getBots();
......@@ -133,6 +140,10 @@ public class CachedUserService implements UserService {
* @return the user account if found by mail, or null if none found.
*/
private EclipseUser retrieveUser(String mail) {
if (StringUtils.isBlank(mail)) {
LOGGER.debug("Blank mail passed, cannot fetch user");
return null;
}
LOGGER.debug("Getting fresh user for {}", mail);
// check for noreply (no reply will never have user account, and fails fast)
EclipseUser eclipseUser = checkForNoReplyUser(mail);
......@@ -162,6 +173,10 @@ public class CachedUserService implements UserService {
* mapped, otherwise null
*/
private EclipseUser checkForNoReplyUser(String mail) {
if (StringUtils.isBlank(mail)) {
LOGGER.debug("Blank mail passed, cannot fetch user");
return null;
}
LOGGER.debug("Checking user with mail {} for no-reply", mail);
boolean isNoReply = patterns.stream().anyMatch(pattern -> pattern.matcher(mail.trim()).find());
if (isNoReply) {
......
......@@ -84,28 +84,25 @@ public class DefaultSystemHookService implements SystemHookService {
}
/**
* Gathers all relevant data related to the created project and persists the
* information into a database
* Gathers all relevant data related to the created project and persists the information into a database
*
* @param wrapper the request wrapper containing all uri, ip, and query params
* @param hook the incoming system hook
* @param hook the incoming system hook
*/
@Transactional
void trackPrivateProjectCreation(RequestWrapper wrapper, SystemHook hook) {
try {
LOGGER.debug("Tracking creation of project: [id: {}, path: {}]", hook.getProjectId(),
hook.getPathWithNamespace());
LOGGER.debug("Tracking creation of project: [id: {}, path: {}]", hook.getProjectId(), hook.getPathWithNamespace());
Optional<GitlabProjectResponse> response = cache.get(Integer.toString(hook.getProjectId()),
new MultivaluedMapImpl<>(), GitlabProjectResponse.class,
() -> api.getProjectInfo(apiToken, hook.getProjectId()));
Optional<GitlabProjectResponse> response = cache
.get(Integer.toString(hook.getProjectId()), new MultivaluedMapImpl<>(), GitlabProjectResponse.class,
() -> api.getProjectInfo(apiToken, hook.getProjectId()));
if (response.isPresent()) {
PrivateProjectEvent dto = mapToDto(hook, response.get());
dao.add(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class)), Arrays.asList(dto));
} else {
LOGGER.error("No info for project: [id: {}, path: {}]", hook.getProjectId(),
hook.getPathWithNamespace());
LOGGER.error("No info for project: [id: {}, path: {}]", hook.getProjectId(), hook.getPathWithNamespace());
}
} catch (Exception e) {
......@@ -114,61 +111,54 @@ public class DefaultSystemHookService implements SystemHookService {
}
/**
* Retrieves the event record for the project that has been deleted. Adds the
* deletionDate field and updates the DB record.
* Retrieves the event record for the project that has been deleted. Adds the deletionDate field and updates the DB
* record.
*
* @param wrapper the request wrapper containing all uri, ip, and query params
* @param hook the incoming system hook
* @param hook the incoming system hook
*/
@Transactional
void trackPrivateProjectDeletion(RequestWrapper wrapper, SystemHook hook) {
try {
LOGGER.debug("Tracking deletion of project: [id: {}, path: {}]", hook.getProjectId(),
hook.getPathWithNamespace());
LOGGER.debug("Tracking deletion of project: [id: {}, path: {}]", hook.getProjectId(), hook.getPathWithNamespace());
MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
params.add(GitEcaParameterNames.PROJECT_ID.getName(), Integer.toString(hook.getProjectId()));
params.add(GitEcaParameterNames.PROJECT_PATH.getName(), hook.getPathWithNamespace());
List<PrivateProjectEvent> results = dao
.get(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class), params));
List<PrivateProjectEvent> results = dao.get(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class), params));
if (results.isEmpty()) {
LOGGER.error("No results for project event: [id: {}, path: {}]", hook.getProjectId(),
hook.getPathWithNamespace());
LOGGER.error("No results for project event: [id: {}, path: {}]", hook.getProjectId(), hook.getPathWithNamespace());
} else {
results.get(0).setDeletionDate(hook.getUpdatedAt().toLocalDateTime());
dao.add(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class)), results);
}
} catch (Exception e) {
LOGGER.error(String.format("Error updating project: [id: %s, path: %s]", hook.getProjectId(),
hook.getPathWithNamespace()), e);
LOGGER.error(String.format("Error updating project: [id: %s, path: %s]", hook.getProjectId(), hook.getPathWithNamespace()), e);
}
}
/**
* Retrieves the event record for the project that has been renamed. Updates the
* projectPath field and creates a new DB record.
* Retrieves the event record for the project that has been renamed. Updates the projectPath field and creates a new DB
* record.
*
* @param wrapper the request wrapper containing all uri, ip, and query params
* @param hook the incoming system hook
* @param hook the incoming system hook
*/
@Transactional
void trackPrivateProjectRenaming(RequestWrapper wrapper, SystemHook hook) {
try {
LOGGER.debug("Tracking renaming of project: [id: {}, path: {}]", hook.getProjectId(),
hook.getOldPathWithNamespace());
LOGGER.debug("Tracking renaming of project: [id: {}, path: {}]", hook.getProjectId(), hook.getOldPathWithNamespace());
MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
params.add(GitEcaParameterNames.PROJECT_ID.getName(), Integer.toString(hook.getProjectId()));
params.add(GitEcaParameterNames.PROJECT_PATH.getName(), hook.getOldPathWithNamespace());
List<PrivateProjectEvent> results = dao
.get(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class), params));
List<PrivateProjectEvent> results = dao.get(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class), params));
if (results.isEmpty()) {
LOGGER.error("No results for project event: [id: {}, path: {}]", hook.getProjectId(),
hook.getOldPathWithNamespace());
LOGGER.error("No results for project event: [id: {}, path: {}]", hook.getProjectId(), hook.getOldPathWithNamespace());
} else {
// Create a new event and track in the DB
PrivateProjectEvent event = new PrivateProjectEvent(results.get(0).getCompositeId().getUserId(),
......@@ -180,23 +170,22 @@ public class DefaultSystemHookService implements SystemHookService {
dao.add(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class)), Arrays.asList(event));
}
} catch (Exception e) {
LOGGER.error(String.format("Error updating project: [id: %s, path: %s]", hook.getProjectId(),
hook.getOldPathWithNamespace()), e);
LOGGER
.error(String.format("Error updating project: [id: %s, path: %s]", hook.getProjectId(), hook.getOldPathWithNamespace()),
e);
}
}
/**
* Takes the received hook and Gitlab api response body and maps the values to a
* PrivateProjectEvent dto.
* Takes the received hook and Gitlab api response body and maps the values to a PrivateProjectEvent dto.
*
* @param hookthe received system hook
* @param hookthe received system hook
* @param gitlabInfo The gitlab project data
* @return A PrivateProjectEvent object
*/
private PrivateProjectEvent mapToDto(SystemHook hook, GitlabProjectResponse gitlabInfo) {
PrivateProjectEvent eventDto = new PrivateProjectEvent(gitlabInfo.getCreatorId(), hook.getProjectId(),
hook.getPathWithNamespace());
PrivateProjectEvent eventDto = new PrivateProjectEvent(gitlabInfo.getCreatorId(), hook.getProjectId(), hook.getPathWithNamespace());
eventDto.setCreationDate(hook.getCreatedAt().toLocalDateTime());
......
......@@ -14,27 +14,35 @@ package org.eclipsefoundation.git.eca.service.impl;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.eclipsefoundation.core.helper.DateTimeHelper;
import org.eclipsefoundation.core.model.RequestWrapper;
import org.eclipsefoundation.core.service.CachingService;
import org.eclipsefoundation.git.eca.api.models.EclipseUser;
import org.eclipsefoundation.git.eca.dto.CommitValidationMessage;
import org.eclipsefoundation.git.eca.dto.CommitValidationStatus;
import org.eclipsefoundation.git.eca.dto.CommitValidationStatusGrouping;
import org.eclipsefoundation.git.eca.helper.CommitHelper;
import org.eclipsefoundation.git.eca.model.Commit;
import org.eclipsefoundation.git.eca.model.CommitStatus;
import org.eclipsefoundation.git.eca.model.GitUser;
import org.eclipsefoundation.git.eca.model.Project;
import org.eclipsefoundation.git.eca.model.ValidationRequest;
import org.eclipsefoundation.git.eca.model.ValidationResponse;
import org.eclipsefoundation.git.eca.namespace.APIStatusCode;
import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
import org.eclipsefoundation.git.eca.service.InterestGroupService;
import org.eclipsefoundation.git.eca.service.ProjectsService;
import org.eclipsefoundation.git.eca.service.UserService;
import org.eclipsefoundation.git.eca.service.ValidationService;
import org.eclipsefoundation.persistence.dao.PersistenceDao;
import org.eclipsefoundation.persistence.model.RDBMSQuery;
......@@ -47,11 +55,68 @@ import org.slf4j.LoggerFactory;
public class DefaultValidationService implements ValidationService {
private static final Logger LOGGER = LoggerFactory.getLogger(DefaultValidationService.class);
@ConfigProperty(name = "eclipse.mail.allowlist")
List<String> allowListUsers;
@Inject
ProjectsService projects;
@Inject
InterestGroupService ig;
@Inject
UserService users;
@Inject
CachingService cache;
@Inject
PersistenceDao dao;
@Inject
FilterService filters;
@Override
public ValidationResponse validateIncomingRequest(ValidationRequest req, RequestWrapper wrapper) {
// get the projects associated with the current request, if any
List<Project> filteredProjects = projects.retrieveProjectsForRequest(req);
// build the response object that we can populate with messages (the message/error arrays are mutable unlike the rest of
// the class)
ValidationResponse r = ValidationResponse
.builder()
.setStrictMode(Boolean.TRUE.equals(req.getStrictMode()))
.setTrackedProject(!filteredProjects.isEmpty())
.setFingerprint(generateRequestHash(req))
.build();
// check to make sure commits are valid
if (req.getCommits().stream().anyMatch(c -> !CommitHelper.validateCommit(c))) {
// for each invalid commit, add errors to output and return
req
.getCommits()
.stream()
.filter(c -> !CommitHelper.validateCommit(c))
.forEach(c -> r
.addError(c.getHash(), "One or more commits were invalid. Please check the payload and try again",
APIStatusCode.ERROR_DEFAULT));
return r;
}
// get previous validation status messages
List<CommitValidationStatus> statuses = getRequestCommitValidationStatus(wrapper, req,
filteredProjects.isEmpty() ? null : filteredProjects.get(0).getProjectId());
req.getCommits().stream().forEach(c -> {
// get the current status if present
Optional<CommitValidationStatus> status = statuses.stream().filter(s -> s.getCommitHash().equals(c.getHash())).findFirst();
// skip the commit validation if already passed
if (status.isPresent() && status.get().getErrors().isEmpty()) {
r.addMessage(c.getHash(), "Commit was previously validated, skipping processing", APIStatusCode.SUCCESS_SKIPPED);
return;
}
// process the current commit
processCommit(c, r, filteredProjects);
});
updateCommitValidationStatus(wrapper, r, req, statuses, filteredProjects.isEmpty() ? null : filteredProjects.get(0));
return r;
}
@Override
public List<CommitValidationStatus> getHistoricValidationStatus(RequestWrapper wrapper, String fingerprint) {
if (StringUtils.isAllBlank(fingerprint)) {
......@@ -79,11 +144,7 @@ public class DefaultValidationService implements ValidationService {
List<CommitValidationStatus> statuses, Project p) {
// iterate over commit responses, and update statuses in DB
List<CommitValidationStatus> updatedStatuses = new ArrayList<>();
for (Entry<String, CommitStatus> e : r.getCommits().entrySet()) {
if (ValidationResponse.NIL_HASH_PLACEHOLDER.equalsIgnoreCase(e.getKey())) {
LOGGER.warn("Not logging errors/validation associated with unknown commit");
continue;
}
r.getCommits().entrySet().stream().filter(e -> !ValidationResponse.NIL_HASH_PLACEHOLDER.equalsIgnoreCase(e.getKey())).forEach(e -> {
// update the status if present, otherwise make new one.
Optional<CommitValidationStatus> status = statuses.stream().filter(s -> e.getKey().equals(s.getCommitHash())).findFirst();
CommitValidationStatus base;
......@@ -100,14 +161,14 @@ public class DefaultValidationService implements ValidationService {
base.setLastModified(DateTimeHelper.now());
updatedStatuses.add(base);
// get the commit for current status
Optional<Commit> commit = req.getCommits().stream().filter(c -> c.getHash().equals(e.getKey())).findFirst();
Optional<Commit> commit = req.getCommits().stream().filter(c -> e.getKey().equals(c.getHash())).findFirst();
if (commit.isEmpty()) {
LOGGER.error("Could not find request commit associated with commit messages for commit hash '{}'", e.getKey());
continue;
return;
}
Commit c = commit.get();
// if there are errors, update validation messages
if (e.getValue().getErrors().size() > 0 || (base.getErrors() != null && base.getErrors().size() > 0)) {
if (!e.getValue().getErrors().isEmpty() || (base.getErrors() != null && !base.getErrors().isEmpty())) {
// generate new errors, looking for errors not found in current list
List<CommitValidationMessage> currentErrors = base.getErrors() != null ? base.getErrors() : new ArrayList<>();
List<CommitValidationMessage> newErrors = e
......@@ -136,7 +197,7 @@ public class DefaultValidationService implements ValidationService {
LOGGER.trace("Encountered {} errors: {}", currentErrors.size(), currentErrors);
base.setErrors(currentErrors);
}
}
});
String fingerprint = generateRequestHash(req);
// update the base commit status and messages
dao.add(new RDBMSQuery<>(wrapper, filters.get(CommitValidationStatus.class)), updatedStatuses);
......@@ -144,4 +205,220 @@ public class DefaultValidationService implements ValidationService {
.add(new RDBMSQuery<>(wrapper, filters.get(CommitValidationStatusGrouping.class)),
updatedStatuses.stream().map(s -> new CommitValidationStatusGrouping(fingerprint, s)).collect(Collectors.toList()));
}
/**
* REQUEST VALIDATION LOGIC
*/
/**
* Process the current request, validating that the passed commit is valid. The author and committers Eclipse Account is
* retrieved, which are then used to check if the current commit is valid for the current project.
*
* @param c the commit to process
* @param response the response container
* @param filteredProjects tracked projects for the current request
*/
private void processCommit(Commit c, ValidationResponse response, List<Project> filteredProjects) {
// retrieve the author + committer for the current request
GitUser author = c.getAuthor();
GitUser committer = c.getCommitter();
response.addMessage(c.getHash(), String.format("Reviewing commit: %1$s", c.getHash()));
response.addMessage(c.getHash(), String.format("Authored by: %1$s <%2$s>", author.getName(), author.getMail()));
// skip processing if a merge commit
if (c.getParents().size() > 1) {
response
.addMessage(c.getHash(),
String.format("Commit '%1$s' has multiple parents, merge commit detected, passing", c.getHash()));
return;
}
// retrieve the eclipse account for the author
EclipseUser eclipseAuthor = getIdentifiedUser(author);
// if the user is a bot, generate a stubbed user
if (isAllowedUser(author.getMail()) || users.userIsABot(author.getMail(), filteredProjects)) {
response
.addMessage(c.getHash(),
String.format("Automated user '%1$s' detected for author of commit %2$s", author.getMail(), c.getHash()));
eclipseAuthor = EclipseUser.createBotStub(author);
} else if (eclipseAuthor == null) {
response
.addMessage(c.getHash(),
String
.format("Could not find an Eclipse user with mail '%1$s' for author of commit %2$s", author.getMail(),
c.getHash()));
response.addError(c.getHash(), "Author must have an Eclipse Account", APIStatusCode.ERROR_AUTHOR);
return;
}
// retrieve the eclipse account for the committer
EclipseUser eclipseCommitter = getIdentifiedUser(committer);
// check if whitelisted or bot
if (isAllowedUser(committer.getMail()) || users.userIsABot(committer.getMail(), filteredProjects)) {
response
.addMessage(c.getHash(),
String.format("Automated user '%1$s' detected for committer of commit %2$s", committer.getMail(), c.getHash()));
eclipseCommitter = EclipseUser.createBotStub(committer);
} else if (eclipseCommitter == null) {
response
.addMessage(c.getHash(),
String
.format("Could not find an Eclipse user with mail '%1$s' for committer of commit %2$s",
committer.getMail(), c.getHash()));
response.addError(c.getHash(), "Committing user must have an Eclipse Account", APIStatusCode.ERROR_COMMITTER);
return;
}
// validate author access to the current repo
validateUserAccess(response, c, eclipseAuthor, filteredProjects, APIStatusCode.ERROR_AUTHOR);
// check committer general access
boolean isCommittingUserCommitter = isCommitter(response, eclipseCommitter, c.getHash(), filteredProjects);
validateUserAccessPartial(response, c, eclipseCommitter, isCommittingUserCommitter, APIStatusCode.ERROR_COMMITTER);
}
/**
* Validates author access for the current commit. If there are errors, they are recorded in the response for the
* current request to be returned once all validation checks are completed.
*
* @param r the current response object for the request
* @param c the commit that is being validated
* @param eclipseUser the user to validate on a branch
* @param filteredProjects tracked projects for the current request
* @param errorCode the error code to display if the user does not have access
*/
private void validateUserAccess(ValidationResponse r, Commit c, EclipseUser eclipseUser, List<Project> filteredProjects,
APIStatusCode errorCode) {
// call isCommitter inline and pass to partial call
validateUserAccessPartial(r, c, eclipseUser, isCommitter(r, eclipseUser, c.getHash(), filteredProjects), errorCode);
}
/**
* Allows for isCommitter to be called external to this method. This was extracted to ensure that isCommitter isn't
* called twice for the same user when checking committer proxy push rules and committer general access.
*
* @param r the current response object for the request
* @param c the commit that is being validated
* @param eclipseUser the user to validate on a branch
* @param isCommitter the results of the isCommitter call from this class.
* @param errorCode the error code to display if the user does not have access
*/
private void validateUserAccessPartial(ValidationResponse r, Commit c, EclipseUser eclipseUser, boolean isCommitter,
APIStatusCode errorCode) {
String userType = "author";
if (APIStatusCode.ERROR_COMMITTER.equals(errorCode)) {
userType = "committer";
}
if (isCommitter) {
r
.addMessage(c.getHash(),
String.format("Eclipse user '%s'(%s) is a committer on the project.", eclipseUser.getName(), userType));
} else {
r
.addMessage(c.getHash(),
String.format("Eclipse user '%s'(%s) is not a committer on the project.", eclipseUser.getName(), userType));
// check if the author is signed off if not a committer
if (eclipseUser.getECA().getSigned()) {
r
.addMessage(c.getHash(),
String
.format("Eclipse user '%s'(%s) has a current Eclipse Contributor Agreement (ECA) on file.",
eclipseUser.getName(), userType));
} else {
r
.addMessage(c.getHash(), String
.format("Eclipse user '%s'(%s) does not have a current Eclipse Contributor Agreement (ECA) on file.\n"
+ "If there are multiple commits, please ensure that each author has a ECA.", eclipseUser.getName(),
userType));
r
.addError(c.getHash(),
String
.format("An Eclipse Contributor Agreement is required for Eclipse user '%s'(%s).",
eclipseUser.getName(), userType),
errorCode);
}
}
}
/**
* Checks whether the given user is a committer on the project. If they are and the project is also a specification for
* a working group, an additional access check is made against the user.
*
* <p>
* Additionally, a check is made to see if the user is a registered bot user for the given project. If they match for
* the given project, they are granted committer-like access to the repository.
*
* @param r the current response object for the request
* @param user the user to validate on a branch
* @param hash the hash of the commit that is being validated
* @param filteredProjects tracked projects for the current request
* @return true if user is considered a committer, false otherwise.
*/
private boolean isCommitter(ValidationResponse r, EclipseUser user, String hash, List<Project> filteredProjects) {
// iterate over filtered projects
for (Project p : filteredProjects) {
LOGGER.debug("Checking project '{}' for user '{}'", p.getName(), user.getName());
// check if any of the committers usernames match the current user
if (p.getCommitters().stream().anyMatch(u -> u.getUsername().equals(user.getName()))) {
// check if the current project is a committer project, and if the user can
// commit to specs
if (p.getSpecWorkingGroup() != null && !user.getECA().getCanContributeSpecProject()) {
// set error + update response status
r
.addError(hash, String
.format("Project is a specification for the working group '%1$s', but user does not have permission to modify a specification project",
p.getSpecWorkingGroup()),
APIStatusCode.ERROR_SPEC_PROJECT);
return false;
} else {
LOGGER.debug("User '{}' was found to be a committer on current project repo '{}'", user.getMail(), p.getName());
return true;
}
}
}
// check if user is a bot, either through early detection or through on-demand
// check
if ((user.getIsBot() != null && user.getIsBot()) || users.userIsABot(user.getMail(), filteredProjects)) {
LOGGER.debug("User '{} <{}>' was found to be a bot", user.getName(), user.getMail());
return true;
}
return false;
}
private boolean isAllowedUser(String mail) {
return StringUtils.isNotBlank(mail) && allowListUsers.indexOf(mail) != -1;
}
/**
* Retrieves an Eclipse Account user object given the Git users email address (at minimum). This is facilitated using
* the Eclipse Foundation accounts API, along short lived in-memory caching for performance and some protection against
* duplicate requests.
*
* @param user the user to retrieve Eclipse Account information for
* @return the Eclipse Account user information if found, or null if there was an error or no user exists.
*/
private EclipseUser getIdentifiedUser(GitUser user) {
// don't process an empty email as it will never have a match
if (StringUtils.isBlank(user.getMail())) {
LOGGER.debug("Cannot get identified user if user is empty, returning null");
return null;
}
// get the Eclipse account for the user
try {
// use cache to avoid asking for the same user repeatedly on repeated requests
EclipseUser foundUser = users.getUser(user.getMail());
if (foundUser == null) {
LOGGER.warn("No users found for mail '{}'", user.getMail());
}
return foundUser;
} catch (WebApplicationException e) {
Response r = e.getResponse();
if (r != null && r.getStatus() == 404) {
LOGGER.error("No users found for mail '{}'", user.getMail());
} else {
LOGGER.error("Error while checking for user", e);
}
}
return null;
}
}
......@@ -12,6 +12,8 @@
**********************************************************************/
package org.eclipsefoundation.git.eca.service.impl;
import java.net.URI;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
......@@ -25,9 +27,14 @@ import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.eclipse.microprofile.context.ManagedExecutor;
import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.core.service.APIMiddleware;
import org.eclipsefoundation.core.service.CachingService;
import org.eclipsefoundation.git.eca.api.ProjectsAPI;
import org.eclipsefoundation.git.eca.model.Project;
import org.eclipsefoundation.git.eca.model.ValidationRequest;
import org.eclipsefoundation.git.eca.namespace.ProviderType;
import org.eclipsefoundation.git.eca.service.InterestGroupService;
import org.eclipsefoundation.git.eca.service.ProjectsService;
import org.jboss.resteasy.specimpl.MultivaluedMapImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -40,10 +47,8 @@ import com.google.common.util.concurrent.ListenableFutureTask;
import io.quarkus.runtime.Startup;
/**
* Projects service implementation that handles pagination of data manually, as
* well as makes use of
* a loading cache to have data be always available with as little latency to
* the user as possible.
* Projects service implementation that handles pagination of data manually, as well as makes use of a loading cache to
* have data be always available with as little latency to the user as possible.
*
* @author Martin Lowe
* @author Zachary Sabourin
......@@ -53,74 +58,69 @@ import io.quarkus.runtime.Startup;
public class PaginationProjectsService implements ProjectsService {
private static final Logger LOGGER = LoggerFactory.getLogger(PaginationProjectsService.class);
@Inject
ManagedExecutor exec;
@ConfigProperty(name = "cache.pagination.refresh-frequency-seconds", defaultValue = "3600")
long refreshAfterWrite;
@Inject
@RestClient
ProjectsAPI projects;
@Inject
InterestGroupService ig;
@Inject
CachingService cache;
@Inject
APIMiddleware middleware;
@Inject
@RestClient
ProjectsAPI projects;
ManagedExecutor exec;
// this class has a separate cache as this data is long to load and should be
// always available.
LoadingCache<String, List<Project>> internalCache;
/**
* Initializes the internal loader cache and pre-populates the data with the one
* available key. If
* more than one key is used, eviction of previous results will happen and
* create degraded
* performance.
* Initializes the internal loader cache and pre-populates the data with the one available key. If more than one key is
* used, eviction of previous results will happen and create degraded performance.
*/
@PostConstruct
public void init() {
// set up the internal cache
this.internalCache = CacheBuilder.newBuilder()
this.internalCache = CacheBuilder
.newBuilder()
.maximumSize(1)
.refreshAfterWrite(refreshAfterWrite, TimeUnit.SECONDS)
.build(
new CacheLoader<String, List<Project>>() {
@Override
public List<Project> load(String key) throws Exception {
return getProjectsInternal();
}
/**
* Implementation required for refreshAfterRewrite to be async rather than sync
* and blocking while awaiting for expensive reload to complete.
*/
@Override
public ListenableFuture<List<Project>> reload(String key, List<Project> oldValue)
throws Exception {
ListenableFutureTask<List<Project>> task = ListenableFutureTask.create(
() -> {
LOGGER.debug("Retrieving new project data async");
List<Project> newProjects = oldValue;
try {
newProjects = getProjectsInternal();
} catch (Exception e) {
LOGGER.error(
"Error while reloading internal projects data, data will be stale for current cycle.",
e);
}
LOGGER.debug("Done refreshing project values");
return newProjects;
});
// run the task using the Quarkus managed executor
exec.execute(task);
return task;
.build(new CacheLoader<String, List<Project>>() {
@Override
public List<Project> load(String key) throws Exception {
return getProjectsInternal();
}
/**
* Implementation required for refreshAfterRewrite to be async rather than sync and blocking while awaiting for
* expensive reload to complete.
*/
@Override
public ListenableFuture<List<Project>> reload(String key, List<Project> oldValue) throws Exception {
ListenableFutureTask<List<Project>> task = ListenableFutureTask.create(() -> {
LOGGER.debug("Retrieving new project data async");
List<Project> newProjects = oldValue;
try {
newProjects = getProjectsInternal();
} catch (Exception e) {
LOGGER.error("Error while reloading internal projects data, data will be stale for current cycle.", e);
}
LOGGER.debug("Done refreshing project values");
return newProjects;
});
// run the task using the Quarkus managed executor
exec.execute(task);
return task;
}
});
// pre-cache the projects to reduce load time for other users
LOGGER.debug("Starting pre-cache of projects");
if (getProjects() == null) {
LOGGER.warn(
"Unable to populate pre-cache for Eclipse projects. Calls may experience degraded performance.");
LOGGER.warn("Unable to populate pre-cache for Eclipse projects. Calls may experience degraded performance.");
}
LOGGER.debug("Completed pre-cache of projects assets");
}
......@@ -134,6 +134,59 @@ public class PaginationProjectsService implements ProjectsService {
}
}
@Override
public List<Project> retrieveProjectsForRequest(ValidationRequest req) {
String repoUrl = req.getRepoUrl().getPath();
if (repoUrl == null) {
LOGGER.warn("Can not match null repo URL to projects");
return Collections.emptyList();
}
return retrieveProjectsForRepoURL(repoUrl, req.getProvider());
}
@Override
public List<Project> retrieveProjectsForRepoURL(String repoUrl, ProviderType provider) {
if (repoUrl == null) {
LOGGER.warn("Can not match null repo URL to projects");
return Collections.emptyList();
}
// check for all projects that make use of the given repo
List<Project> availableProjects = getProjects();
availableProjects
.addAll(cache
.get("all", new MultivaluedMapImpl<>(), Project.class, () -> ig.adaptInterestGroups(ig.getInterestGroups()))
.orElse(Collections.emptyList()));
if (availableProjects.isEmpty()) {
LOGGER.warn("Could not find any projects to match against");
return Collections.emptyList();
}
LOGGER.debug("Checking projects for repos that end with: {}", repoUrl);
// filter the projects based on the repo URL. At least one repo in project must
// match the repo URL to be valid
switch (provider) {
case GITLAB:
String projectNamespace = URI.create(repoUrl).getPath().substring(1).toLowerCase();
return availableProjects
.stream()
.filter(p -> projectNamespace.startsWith(p.getGitlab().getProjectGroup() + "/")
&& p.getGitlab().getIgnoredSubGroups().stream().noneMatch(sg -> projectNamespace.startsWith(sg + "/")))
.collect(Collectors.toList());
case GITHUB:
return availableProjects
.stream()
.filter(p -> p.getGithubRepos().stream().anyMatch(re -> re.getUrl() != null && re.getUrl().endsWith(repoUrl)))
.collect(Collectors.toList());
case GERRIT:
return availableProjects
.stream()
.filter(p -> p.getGerritRepos().stream().anyMatch(re -> re.getUrl() != null && re.getUrl().endsWith(repoUrl)))
.collect(Collectors.toList());
default:
return Collections.emptyList();
}
}
/**
* Logic for retrieving projects from API.
*
......@@ -141,17 +194,13 @@ public class PaginationProjectsService implements ProjectsService {
*/
private List<Project> getProjectsInternal() {
return middleware.getAll(params -> projects.getProjects(params), Project.class)
.stream()
.map(proj -> {
proj.getGerritRepos()
.forEach(repo -> {
if (repo.getUrl().endsWith(".git")) {
repo.setUrl(repo.getUrl().substring(0, repo.getUrl().length() - 4));
}
});
return proj;
})
.collect(Collectors.toList());
return middleware.getAll(params -> projects.getProjects(params), Project.class).stream().map(proj -> {
proj.getGerritRepos().forEach(repo -> {
if (repo.getUrl().endsWith(".git")) {
repo.setUrl(repo.getUrl().substring(0, repo.getUrl().length() - 4));
}
});
return proj;
}).collect(Collectors.toList());
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment