diff --git a/config/mariadb/initdb.d/init.sql b/config/mariadb/initdb.d/init.sql
index 45938283d5c8f57751c83219404ca36206f9670e..002b10e748dc44ccb6c5c36e278dd15354095f4c 100644
--- a/config/mariadb/initdb.d/init.sql
+++ b/config/mariadb/initdb.d/init.sql
@@ -49,5 +49,7 @@ CREATE TABLE GithubWebhookTracking (
repositoryFullName varchar(127) NOT NULL,
pullRequestNumber int NOT NULL,
lastUpdated datetime DEFAULT NULL,
+ needsRevalidation tinyint(1) DEFAULT 0,
+ manualRevalidationCount int DEFAULT 0,
PRIMARY KEY (id)
);
\ No newline at end of file
diff --git a/src/main/java/org/eclipsefoundation/git/eca/dto/GithubWebhookTracking.java b/src/main/java/org/eclipsefoundation/git/eca/dto/GithubWebhookTracking.java
index 72cb98ceac7b3941ca8df976b2ad6262fc067f7d..e40f3f3395da07b0cbc3186c4abf207a035c26c5 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/dto/GithubWebhookTracking.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/dto/GithubWebhookTracking.java
@@ -30,6 +30,7 @@ import org.eclipsefoundation.persistence.dto.filter.DtoFilter;
import org.eclipsefoundation.persistence.model.DtoTable;
import org.eclipsefoundation.persistence.model.ParameterizedSQLStatement;
import org.eclipsefoundation.persistence.model.ParameterizedSQLStatementBuilder;
+import org.eclipsefoundation.persistence.model.SortableField;
/**
* Github Webhook request tracking info. Tracking is required to trigger revalidation through the Github API from this
@@ -51,7 +52,10 @@ public class GithubWebhookTracking extends BareNode {
private String headSha;
private Integer pullRequestNumber;
private String lastKnownState;
+ @SortableField
private ZonedDateTime lastUpdated;
+ private boolean needsRevalidation;
+ private Integer manualRevalidationCount;
@Override
public Long getId() {
@@ -149,12 +153,41 @@ public class GithubWebhookTracking extends BareNode {
this.lastUpdated = lastUpdated;
}
+ /**
+ * @return the needsRevalidation
+ */
+ public boolean isNeedsRevalidation() {
+ return needsRevalidation;
+ }
+
+ /**
+ * @param needsRevalidation the needsRevalidation to set
+ */
+ public void setNeedsRevalidation(boolean needsRevalidation) {
+ this.needsRevalidation = needsRevalidation;
+ }
+
+ /**
+ * @return the manualRevalidationCount
+ */
+ public Integer getManualRevalidationCount() {
+ return manualRevalidationCount;
+ }
+
+ /**
+ * @param manualRevalidationCount the manualRevalidationCount to set
+ */
+ public void setManualRevalidationCount(Integer manualRevalidationCount) {
+ this.manualRevalidationCount = manualRevalidationCount;
+ }
+
@Override
public int hashCode() {
final int prime = 31;
int result = super.hashCode();
- result = prime * result
- + Objects.hash(headSha, id, installationId, lastKnownState, lastUpdated, pullRequestNumber, repositoryFullName);
+ result = prime * result + Objects
+ .hash(headSha, id, installationId, lastKnownState, lastUpdated, manualRevalidationCount, needsRevalidation,
+ pullRequestNumber, repositoryFullName);
return result;
}
@@ -172,7 +205,8 @@ public class GithubWebhookTracking extends BareNode {
GithubWebhookTracking other = (GithubWebhookTracking) obj;
return Objects.equals(headSha, other.headSha) && Objects.equals(id, other.id)
&& Objects.equals(installationId, other.installationId) && Objects.equals(lastKnownState, other.lastKnownState)
- && Objects.equals(lastUpdated, other.lastUpdated) && Objects.equals(pullRequestNumber, other.pullRequestNumber)
+ && Objects.equals(lastUpdated, other.lastUpdated) && Objects.equals(manualRevalidationCount, other.manualRevalidationCount)
+ && needsRevalidation == other.needsRevalidation && Objects.equals(pullRequestNumber, other.pullRequestNumber)
&& Objects.equals(repositoryFullName, other.repositoryFullName);
}
@@ -204,6 +238,10 @@ public class GithubWebhookTracking extends BareNode {
.addClause(new ParameterizedSQLStatement.Clause(TABLE.getAlias() + ".repositoryFullName = ?",
new Object[] { repositoryFullName }));
}
+ String needsRevalidation = params.getFirst(GitEcaParameterNames.NEEDS_REVALIDATION_RAW);
+ if (Boolean.TRUE.equals(Boolean.valueOf(needsRevalidation))) {
+ statement.addClause(new ParameterizedSQLStatement.Clause(TABLE.getAlias() + ".needsRevalidation = TRUE", new Object[] {}));
+ }
return statement;
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java b/src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java
new file mode 100644
index 0000000000000000000000000000000000000000..9f4c34a02151c1f9186befefcf35812791247838
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java
@@ -0,0 +1,391 @@
+/**
+ * Copyright (c) 2023 Eclipse Foundation
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * Author: Martin Lowe <martin.lowe@eclipse-foundation.org>
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.git.eca.helper;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import javax.enterprise.context.ApplicationScoped;
+import javax.inject.Inject;
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.ServerErrorException;
+import javax.ws.rs.core.MultivaluedMap;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpStatus;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+import org.eclipse.microprofile.rest.client.inject.RestClient;
+import org.eclipsefoundation.core.helper.DateTimeHelper;
+import org.eclipsefoundation.core.model.RequestWrapper;
+import org.eclipsefoundation.core.service.APIMiddleware;
+import org.eclipsefoundation.git.eca.api.GithubAPI;
+import org.eclipsefoundation.git.eca.api.models.GithubCommit;
+import org.eclipsefoundation.git.eca.api.models.GithubCommit.ParentCommit;
+import org.eclipsefoundation.git.eca.api.models.GithubCommitStatusRequest;
+import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
+import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
+import org.eclipsefoundation.git.eca.config.WebhooksConfig;
+import org.eclipsefoundation.git.eca.dto.CommitValidationStatus;
+import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
+import org.eclipsefoundation.git.eca.model.Commit;
+import org.eclipsefoundation.git.eca.model.GitUser;
+import org.eclipsefoundation.git.eca.model.ValidationRequest;
+import org.eclipsefoundation.git.eca.model.ValidationResponse;
+import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
+import org.eclipsefoundation.git.eca.namespace.GithubCommitStatuses;
+import org.eclipsefoundation.git.eca.namespace.ProviderType;
+import org.eclipsefoundation.git.eca.service.GithubApplicationService;
+import org.eclipsefoundation.git.eca.service.ValidationService;
+import org.eclipsefoundation.persistence.dao.PersistenceDao;
+import org.eclipsefoundation.persistence.model.RDBMSQuery;
+import org.eclipsefoundation.persistence.service.FilterService;
+import org.jboss.resteasy.specimpl.MultivaluedMapImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class is used to adapt Github requests to the standard validation workflow in a way that could be reused by both
+ * resource calls and scheduled tasks for revalidation.
+ */
+@ApplicationScoped
+public class GithubValidationHelper {
+ private static final Logger LOGGER = LoggerFactory.getLogger(GithubValidationHelper.class);
+
+ private static final String VALIDATION_LOGGING_MESSAGE = "Setting validation state for {}/#{} to {}";
+
+ @ConfigProperty(name = "eclipse.github.default-api-version", defaultValue = "2022-11-28")
+ String apiVersion;
+
+ @Inject
+ WebhooksConfig webhooksConfig;
+
+ @Inject
+ JwtHelper jwtHelper;
+ @Inject
+ APIMiddleware middleware;
+ @Inject
+ PersistenceDao dao;
+ @Inject
+ FilterService filters;
+
+ @Inject
+ ValidationService validation;
+ @Inject
+ GithubApplicationService ghAppService;
+
+ @RestClient
+ GithubAPI ghApi;
+
+ /**
+ * Using the wrapper and the passed unique information about a Github validation instance, lookup or create the tracking
+ * request and validate the data.
+ *
+ * @param wrapper the wrapper for the current request
+ * @param org the name of the GH organization
+ * @param repoName the slug of the repo that has the PR to be validated
+ * @param prNo the PR number for the current request.
+ * @return the validated response if it is a valid request, or throws a web exception if there is a problem validating
+ * the request.
+ */
+ public ValidationResponse validateIncomingRequest(RequestWrapper wrapper, String org, String repoName, Integer prNo) {
+ String fullRepoName = org + '/' + repoName;
+ // get the installation ID for the given repo if it exists, and if the PR noted exists
+ String installationId = ghAppService.getInstallationForRepo(fullRepoName);
+ Optional<PullRequest> prResponse = ghAppService.getPullRequest(installationId, fullRepoName, prNo);
+ if (StringUtils.isBlank(installationId)) {
+ throw new BadRequestException("Could not find an ECA app installation for repo name: " + fullRepoName);
+ } else if (prResponse.isEmpty()) {
+ throw new NotFoundException(String.format("Could not find PR '%d' in repo name '%s'", prNo, fullRepoName));
+ }
+
+ // prepare the request for consumption
+ String repoUrl = getRepoUrl(org, repoName);
+ ValidationRequest vr = generateRequest(installationId, fullRepoName, prNo, repoUrl);
+
+ // build the commit sha list based on the prepared request
+ List<String> commitShas = vr.getCommits().stream().map(Commit::getHash).collect(Collectors.toList());
+ // there should always be commits for a PR, but in case, lets check
+ if (commitShas.isEmpty()) {
+ throw new BadRequestException(String.format("Could not find any commits for %s#%d", fullRepoName, prNo));
+ }
+ LOGGER.debug("Found {} commits for '{}#{}'", commitShas.size(), fullRepoName, prNo);
+
+ // retrieve the webhook tracking info, or generate an entry to track this PR if it's missing.
+ GithubWebhookTracking updatedTracking = retrieveAndUpdateTrackingInformation(wrapper, installationId, fullRepoName,
+ prResponse.get());
+ if (updatedTracking == null) {
+ throw new ServerErrorException("Error while attempting to revalidate request, try again later.",
+ HttpStatus.SC_INTERNAL_SERVER_ERROR);
+ }
+
+ // get the commit status of commits to use for checking historic validation
+ List<CommitValidationStatus> statuses = validation.getHistoricValidationStatusByShas(wrapper, commitShas);
+ if (!"open".equalsIgnoreCase(prResponse.get().getState()) && statuses.isEmpty()) {
+ throw new BadRequestException("Cannot find validation history for current non-open PR, cannot provide validation status");
+ }
+
+ // we only want to update/revalidate for open PRs, so don't do this check if the PR is merged/closed
+ if ("open".equalsIgnoreCase(prResponse.get().getState()) && commitShas.size() != statuses.size()) {
+ LOGGER.debug("Validation for {}#{} does not seem to be current, revalidating commits", fullRepoName, prNo);
+ // using the updated tracking, perform the validation
+ return handleGithubWebhookValidation(GithubWebhookRequest.buildFromTracking(updatedTracking), vr, wrapper);
+ }
+ return null;
+ }
+
+ /**
+ * Generate a ValidationRequest object based on data pulled from Github, grabbing commits from the noted pull request
+ * using the installation ID for access/authorization.
+ *
+ * @param installationId the ECA app installation ID for the organization
+ * @param repositoryFullName the full name of the repository where the PR resides
+ * @param pullRequestNumber the pull request number that is being validated
+ * @param repositoryUrl the URL of the repository that contains the commits to validate
+ * @return the populated validation request for the Github request information
+ */
+ @SuppressWarnings("null")
+ public ValidationRequest generateRequest(String installationId, String repositoryFullName, int pullRequestNumber,
+ String repositoryUrl) {
+ checkRequestParameters(installationId, repositoryFullName, pullRequestNumber);
+ // get the commits that will be validated, don't cache as changes can come in too fast for it to be useful
+ List<GithubCommit> commits = middleware
+ .getAll(i -> ghApi
+ .getCommits(jwtHelper.getGhBearerString(installationId), apiVersion, repositoryFullName, pullRequestNumber),
+ GithubCommit.class);
+ LOGGER.trace("Retrieved {} commits for PR {} in repo {}", commits.size(), pullRequestNumber, repositoryUrl);
+ // set up the validation request from current data
+ return ValidationRequest
+ .builder()
+ .setProvider(ProviderType.GITHUB)
+ .setRepoUrl(URI.create(repositoryUrl))
+ .setStrictMode(true)
+ .setCommits(commits
+ .stream()
+ .map(c -> Commit
+ .builder()
+ .setHash(c.getSha())
+ .setAuthor(GitUser
+ .builder()
+ .setMail(getNullableString(() -> c.getCommit().getAuthor().getEmail()))
+ .setName(getNullableString(() -> c.getCommit().getAuthor().getName()))
+ .setExternalId(getNullableString(() -> c.getAuthor().getLogin()))
+ .build())
+ .setCommitter(GitUser
+ .builder()
+ .setMail(getNullableString(() -> c.getCommit().getCommitter().getEmail()))
+ .setName(getNullableString(() -> c.getCommit().getCommitter().getName()))
+ .setExternalId(getNullableString(() -> c.getCommitter().getLogin()))
+ .build())
+ .setParents(c.getParents().stream().map(ParentCommit::getSha).collect(Collectors.toList()))
+ .build())
+ .collect(Collectors.toList()))
+ .build();
+ }
+
+ /**
+ * Process the current request and update the checks state to pending then success or failure. Contains verbose TRACE
+ * logging for more info on the states of the validation for more information
+ *
+ * @param request information about the request from the GH webhook on what resource requested revalidation. Used to
+ * target the commit status of the resources
+ * @param vr the pseudo request generated from the contextual webhook data. Used to make use of existing validation
+ * logic.
+ * @return true if the validation passed, false otherwise.
+ */
+ public ValidationResponse handleGithubWebhookValidation(GithubWebhookRequest request, ValidationRequest vr, RequestWrapper wrapper) {
+ // update the status before processing
+ LOGGER
+ .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
+ GithubCommitStatuses.PENDING);
+ updateCommitStatus(request, GithubCommitStatuses.PENDING);
+
+ // validate the response
+ LOGGER
+ .trace("Begining validation of request for {}/#{}", request.getRepository().getFullName(),
+ request.getPullRequest().getNumber());
+ ValidationResponse r = validation.validateIncomingRequest(vr, wrapper);
+ if (r.getPassed()) {
+ LOGGER
+ .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
+ GithubCommitStatuses.SUCCESS);
+ updateCommitStatus(request, GithubCommitStatuses.SUCCESS);
+ } else {
+ LOGGER
+ .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
+ GithubCommitStatuses.FAILURE);
+ updateCommitStatus(request, GithubCommitStatuses.FAILURE);
+ }
+ return r;
+ }
+
+ /**
+ * Shortcut method that will retrieve existing GH tracking info, create new entries if missing, and will update the
+ * state of existing requests as well.
+ *
+ * @param installationId the installation ID for the ECA app in the given repository
+ * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
+ * @param pr the pull request targeted by the validation request.
+ * @return a new or updated tracking object, or null if there was an error in saving the information
+ */
+ public GithubWebhookTracking retrieveAndUpdateTrackingInformation(RequestWrapper wrapper, String installationId,
+ String repositoryFullName, PullRequest pr) {
+ return updateGithubTrackingIfMissing(wrapper,
+ getExistingRequestInformation(wrapper, installationId, repositoryFullName, pr.getNumber()), pr, installationId,
+ repositoryFullName);
+ }
+
+ /**
+ * Checks if the Github tracking is present for the current request, and if missing will generate a new record and save
+ * it.
+ *
+ * @param tracking the optional tracking entry for the current request
+ * @param request the pull request that is being validated
+ * @param installationId the ECA app installation ID for the current request
+ * @param fullRepoName the full repo name for the validation request
+ */
+ public GithubWebhookTracking updateGithubTrackingIfMissing(RequestWrapper wrapper, Optional<GithubWebhookTracking> tracking,
+ PullRequest request, String installationId, String fullRepoName) {
+ // if there is no tracking present, create the missing tracking and persist it
+ GithubWebhookTracking updatedTracking;
+ if (tracking.isEmpty()) {
+ updatedTracking = new GithubWebhookTracking();
+ updatedTracking.setHeadSha(request.getHead().getSha());
+ updatedTracking.setInstallationId(installationId);
+ updatedTracking.setLastUpdated(DateTimeHelper.now());
+ updatedTracking.setPullRequestNumber(request.getNumber());
+ updatedTracking.setRepositoryFullName(fullRepoName);
+ updatedTracking.setLastKnownState(request.getState());
+ updatedTracking.setNeedsRevalidation(false);
+ updatedTracking.setManualRevalidationCount(0);
+ if (!"open".equalsIgnoreCase(request.getState())) {
+ LOGGER
+ .warn("The PR {} in {} is not in an open state, and will not be validated to follow our validation practice",
+ updatedTracking.getPullRequestNumber(), fullRepoName);
+ }
+ } else {
+ // at least update the state on every run
+ updatedTracking = tracking.get();
+ updatedTracking.setLastKnownState(request.getState());
+ }
+
+ // save the data, and log on its success or failure
+ List<GithubWebhookTracking> savedTracking = dao
+ .add(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class)), Arrays.asList(updatedTracking));
+ if (savedTracking.isEmpty()) {
+ LOGGER.warn("Unable to create new GH tracking record for request to validate {}#{}", fullRepoName, request.getNumber());
+ return null;
+ }
+ // return the updated tracking when successful
+ LOGGER.debug("Created/updated GH tracking record for request to validate {}#{}", fullRepoName, request.getNumber());
+ return savedTracking.get(0);
+ }
+
+ /**
+ * Attempts to retrieve a webhook tracking record given the installation, repository, and pull request number.
+ *
+ * @param installationId the installation ID for the ECA app in the given repository
+ * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
+ * @param pullRequestNumber the pull request number that is being processed currently
+ * @return the webhook tracking record if it can be found, or an empty optional.
+ */
+ public Optional<GithubWebhookTracking> getExistingRequestInformation(RequestWrapper wrapper, String installationId,
+ String repositoryFullName, int pullRequestNumber) {
+ checkRequestParameters(installationId, repositoryFullName, pullRequestNumber);
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
+ params.add(GitEcaParameterNames.INSTALLATION_ID_RAW, installationId);
+ params.add(GitEcaParameterNames.REPOSITORY_FULL_NAME_RAW, repositoryFullName);
+ params.add(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW, Integer.toString(pullRequestNumber));
+ return dao.get(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class), params)).stream().findFirst();
+ }
+
+ /**
+ * Simple helper method so we don't have to repeat static strings in multiple places.
+ *
+ * @param fullRepoName the full repo name including org for the target PR.
+ * @return the full repo URL on Github for the request
+ */
+ public static String getRepoUrl(String org, String repoName) {
+ return "https://github.com/" + org + '/' + repoName;
+ }
+
+ /**
+ * Sends off a POST to update the commit status given a context for the current PR.
+ *
+ * @param request the current webhook update request
+ * @param state the state to set the status to
+ * @param fingerprint the internal unique string for the set of commits being processed
+ */
+ private void updateCommitStatus(GithubWebhookRequest request, GithubCommitStatuses state) {
+ LOGGER
+ .trace("Generated access token for installation {}: {}", request.getInstallation().getId(),
+ jwtHelper.getGithubAccessToken(request.getInstallation().getId()).getToken());
+ ghApi
+ .updateStatus(jwtHelper.getGhBearerString(request.getInstallation().getId()), apiVersion,
+ request.getRepository().getFullName(), request.getPullRequest().getHead().getSha(),
+ GithubCommitStatusRequest
+ .builder()
+ .setDescription(state.getMessage())
+ .setState(state.toString())
+ .setTargetUrl(webhooksConfig.github().serverTarget() + "/git/eca/status/gh/"
+ + request.getRepository().getFullName() + '/' + request.getPullRequest().getNumber())
+ .setContext(webhooksConfig.github().context())
+ .build());
+ }
+
+ /**
+ * Wraps a nullable value fetch to handle errors and will return null if the value can't be retrieved.
+ *
+ * @param supplier the method with potentially nullable values
+ * @return the value if it can be found, or null
+ */
+ private String getNullableString(Supplier<String> supplier) {
+ try {
+ return supplier.get();
+ } catch (NullPointerException e) {
+ // suppress, as we don't care at this point if its null
+ }
+ return null;
+ }
+
+ /**
+ * Validates required fields for processing requests.
+ *
+ * @param installationId the installation ID for the ECA app in the given repository
+ * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
+ * @param pullRequestNumber the pull request number that is being processed currently
+ * @throws BadRequestException if at least one of the parameters is in an invalid state.
+ */
+ private void checkRequestParameters(String installationId, String repositoryFullName, int pullRequestNumber) {
+ List<String> missingFields = new ArrayList<>();
+ if (StringUtils.isBlank(installationId)) {
+ missingFields.add(GitEcaParameterNames.INSTALLATION_ID_RAW);
+ }
+ if (StringUtils.isBlank(repositoryFullName)) {
+ missingFields.add(GitEcaParameterNames.REPOSITORY_FULL_NAME_RAW);
+ }
+ if (pullRequestNumber < 1) {
+ missingFields.add(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW);
+ }
+
+ // throw exception if some fields are missing as we can't continue to process the request
+ if (!missingFields.isEmpty()) {
+ throw new BadRequestException("Missing fields in order to prepare request: " + StringUtils.join(missingFields, ' '));
+ }
+ }
+
+}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java b/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java
index a16499237e51d4313a1e7505d0e73f187f702327..cbfaa0ee2926ae4a6d13ae3b6cc4dd40d7cb0ad9 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java
@@ -39,6 +39,7 @@ public final class GitEcaParameterNames implements UrlParameterNamespace {
public static final String INSTALLATION_ID_RAW = "installation_id";
public static final String PULL_REQUEST_NUMBER_RAW = "pull_request_number";
public static final String USER_MAIL_RAW = "user_mail";
+ public static final String NEEDS_REVALIDATION_RAW = "needs_revalidation";
public static final UrlParameter COMMIT_ID = new UrlParameter(COMMIT_ID_RAW);
public static final UrlParameter SHA = new UrlParameter(SHA_RAW);
public static final UrlParameter SHAS = new UrlParameter(SHAS_RAW);
@@ -58,13 +59,14 @@ public final class GitEcaParameterNames implements UrlParameterNamespace {
public static final UrlParameter INSTALLATION_ID = new UrlParameter(INSTALLATION_ID_RAW);
public static final UrlParameter PULL_REQUEST_NUMBER = new UrlParameter(PULL_REQUEST_NUMBER_RAW);
public static final UrlParameter USER_MAIL = new UrlParameter(USER_MAIL_RAW);
+ public static final UrlParameter NEEDS_REVALIDATION = new UrlParameter(NEEDS_REVALIDATION_RAW);
@Override
public List<UrlParameter> getParameters() {
return Arrays
.asList(COMMIT_ID, SHA, SHAS, PROJECT_ID, PROJECT_IDS, NOT_IN_PROJECT_IDS, REPO_URL, FINGERPRINT, USER_ID, PROJECT_PATH,
PARENT_PROJECT, STATUS_ACTIVE, STATUS_DELETED, SINCE, UNTIL, REPOSITORY_FULL_NAME, INSTALLATION_ID,
- PULL_REQUEST_NUMBER, USER_MAIL);
+ PULL_REQUEST_NUMBER, USER_MAIL, NEEDS_REVALIDATION);
}
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/resource/GithubAdjacentResource.java b/src/main/java/org/eclipsefoundation/git/eca/resource/GithubAdjacentResource.java
index a7fc852eb799a75c9f58672b3a3648bad92c5c0c..6fd5d3a14addfc4d7b94c9c730750a6017830eaa 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/resource/GithubAdjacentResource.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/resource/GithubAdjacentResource.java
@@ -11,48 +11,16 @@
*/
package org.eclipsefoundation.git.eca.resource;
-import java.net.URI;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
import javax.inject.Inject;
-import javax.ws.rs.BadRequestException;
-import javax.ws.rs.core.MultivaluedMap;
-import org.apache.commons.lang3.StringUtils;
-import org.eclipse.microprofile.config.inject.ConfigProperty;
-import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.core.helper.DateTimeHelper;
import org.eclipsefoundation.core.model.RequestWrapper;
-import org.eclipsefoundation.core.service.APIMiddleware;
-import org.eclipsefoundation.git.eca.api.GithubAPI;
-import org.eclipsefoundation.git.eca.api.models.GithubCommit;
-import org.eclipsefoundation.git.eca.api.models.GithubCommit.ParentCommit;
-import org.eclipsefoundation.git.eca.api.models.GithubCommitStatusRequest;
-import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
-import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
-import org.eclipsefoundation.git.eca.config.WebhooksConfig;
import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
-import org.eclipsefoundation.git.eca.helper.JwtHelper;
-import org.eclipsefoundation.git.eca.model.Commit;
-import org.eclipsefoundation.git.eca.model.GitUser;
-import org.eclipsefoundation.git.eca.model.ValidationRequest;
-import org.eclipsefoundation.git.eca.model.ValidationResponse;
-import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
-import org.eclipsefoundation.git.eca.namespace.GithubCommitStatuses;
-import org.eclipsefoundation.git.eca.namespace.ProviderType;
-import org.eclipsefoundation.git.eca.service.GithubApplicationService;
-import org.eclipsefoundation.git.eca.service.ValidationService;
import org.eclipsefoundation.persistence.dao.PersistenceDao;
import org.eclipsefoundation.persistence.model.RDBMSQuery;
import org.eclipsefoundation.persistence.service.FilterService;
-import org.jboss.resteasy.specimpl.MultivaluedMapImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Contains operations and properties that are common to resources that interact with Github validation.
@@ -61,258 +29,18 @@ import org.slf4j.LoggerFactory;
*
*/
public abstract class GithubAdjacentResource {
- private static final Logger LOGGER = LoggerFactory.getLogger(GithubAdjacentResource.class);
-
- private static final String VALIDATION_LOGGING_MESSAGE = "Setting validation state for {}/#{} to {}";
-
- @ConfigProperty(name = "eclipse.github.default-api-version", defaultValue = "2022-11-28")
- String apiVersion;
-
- @Inject
- WebhooksConfig webhooksConfig;
- @Inject
- JwtHelper jwtHelper;
- @Inject
- APIMiddleware middleware;
@Inject
PersistenceDao dao;
@Inject
FilterService filters;
- @Inject
- ValidationService validation;
- @Inject
- GithubApplicationService ghAppService;
-
@Inject
RequestWrapper wrapper;
- @RestClient
- GithubAPI ghApi;
-
- /**
- * Generate a ValidationRequest object based on data pulled from Github, grabbing commits from the noted pull request
- * using the installation ID for access/authorization.
- *
- * @param installationId the ECA app installation ID for the organization
- * @param repositoryFullName the full name of the repository where the PR resides
- * @param pullRequestNumber the pull request number that is being validated
- * @param repositoryUrl the URL of the repository that contains the commits to validate
- * @return the populated validation request for the Github request information
- */
- @SuppressWarnings("null")
- ValidationRequest generateRequest(String installationId, String repositoryFullName, int pullRequestNumber, String repositoryUrl) {
- checkRequestParameters(installationId, repositoryFullName, pullRequestNumber);
- // get the commits that will be validated, don't cache as changes can come in too fast for it to be useful
- List<GithubCommit> commits = middleware
- .getAll(i -> ghApi
- .getCommits(jwtHelper.getGhBearerString(installationId), apiVersion, repositoryFullName, pullRequestNumber),
- GithubCommit.class);
- LOGGER.trace("Retrieved {} commits for PR {} in repo {}", commits.size(), pullRequestNumber, repositoryUrl);
- // set up the validation request from current data
- return ValidationRequest
- .builder()
- .setProvider(ProviderType.GITHUB)
- .setRepoUrl(URI.create(repositoryUrl))
- .setStrictMode(true)
- .setCommits(commits
- .stream()
- .map(c -> Commit
- .builder()
- .setHash(c.getSha())
- .setAuthor(GitUser
- .builder()
- .setMail(getNullableString(() -> c.getCommit().getAuthor().getEmail()))
- .setName(getNullableString(() -> c.getCommit().getAuthor().getName()))
- .setExternalId(getNullableString(() -> c.getAuthor().getLogin()))
- .build())
- .setCommitter(GitUser
- .builder()
- .setMail(getNullableString(() -> c.getCommit().getCommitter().getEmail()))
- .setName(getNullableString(() -> c.getCommit().getCommitter().getName()))
- .setExternalId(getNullableString(() -> c.getCommitter().getLogin()))
- .build())
- .setParents(c.getParents().stream().map(ParentCommit::getSha).collect(Collectors.toList()))
- .build())
- .collect(Collectors.toList()))
- .build();
- }
-
- /**
- * Shortcut method that will retrieve existing GH tracking info, create new entries if missing, and will update the
- * state of existing requests as well.
- *
- * @param installationId the installation ID for the ECA app in the given repository
- * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
- * @param pr the pull request targeted by the validation request.
- * @return a new or updated tracking object, or null if there was an error in saving the information
- */
- GithubWebhookTracking retrieveAndUpdateTrackingInformation(String installationId, String repositoryFullName, PullRequest pr) {
- return updateGithubTrackingIfMissing(getExistingRequestInformation(installationId, repositoryFullName, pr.getNumber()), pr,
- installationId, repositoryFullName);
- }
-
- /**
- * Attempts to retrieve a webhook tracking record given the installation, repository, and pull request number.
- *
- * @param installationId the installation ID for the ECA app in the given repository
- * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
- * @param pullRequestNumber the pull request number that is being processed currently
- * @return the webhook tracking record if it can be found, or an empty optional.
- */
- Optional<GithubWebhookTracking> getExistingRequestInformation(String installationId, String repositoryFullName, int pullRequestNumber) {
- checkRequestParameters(installationId, repositoryFullName, pullRequestNumber);
- MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
- params.add(GitEcaParameterNames.INSTALLATION_ID_RAW, installationId);
- params.add(GitEcaParameterNames.REPOSITORY_FULL_NAME_RAW, repositoryFullName);
- params.add(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW, Integer.toString(pullRequestNumber));
- return dao.get(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class), params)).stream().findFirst();
+ void setRevalidationFlagForTracking(GithubWebhookTracking tracking) {
+ tracking.setNeedsRevalidation(true);
+ tracking.setLastUpdated(DateTimeHelper.now());
+ dao.add(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class)), Arrays.asList(tracking));
}
-
- /**
- * Checks if the Github tracking is present for the current request, and if missing will generate a new record and save
- * it.
- *
- * @param tracking the optional tracking entry for the current request
- * @param request the pull request that is being validated
- * @param installationId the ECA app installation ID for the current request
- * @param fullRepoName the full repo name for the validation request
- */
- GithubWebhookTracking updateGithubTrackingIfMissing(Optional<GithubWebhookTracking> tracking, PullRequest request,
- String installationId, String fullRepoName) {
- // if there is no tracking present, create the missing tracking and persist it
- GithubWebhookTracking updatedTracking;
- if (tracking.isEmpty()) {
- updatedTracking = new GithubWebhookTracking();
- updatedTracking.setHeadSha(request.getHead().getSha());
- updatedTracking.setInstallationId(installationId);
- updatedTracking.setLastUpdated(DateTimeHelper.now());
- updatedTracking.setPullRequestNumber(request.getNumber());
- updatedTracking.setRepositoryFullName(fullRepoName);
- updatedTracking.setLastKnownState(request.getState());
- if (!"open".equalsIgnoreCase(request.getState())) {
- LOGGER
- .warn("The PR {} in {} is not in an open state, and will not be validated to follow our validation practice",
- updatedTracking.getPullRequestNumber(), fullRepoName);
- }
- } else {
- // at least update the state on every run
- updatedTracking = tracking.get();
- updatedTracking.setLastKnownState(request.getState());
- }
-
- // save the data, and log on its success or failure
- List<GithubWebhookTracking> savedTracking = dao
- .add(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class)), Arrays.asList(updatedTracking));
- if (savedTracking.isEmpty()) {
- LOGGER.warn("Unable to create new GH tracking record for request to validate {}#{}", fullRepoName, request.getNumber());
- return null;
- }
- // return the updated tracking when successful
- LOGGER.debug("Created new GH tracking record for request to validate {}#{}", fullRepoName, request.getNumber());
- return savedTracking.get(0);
- }
-
- /**
- * Process the current request and update the checks state to pending then success or failure. Contains verbose TRACE
- * logging for more info on the states of the validation for more information
- *
- * @param request information about the request from the GH webhook on what resource requested revalidation. Used to
- * target the commit status of the resources
- * @param vr the pseudo request generated from the contextual webhook data. Used to make use of existing validation
- * logic.
- * @return true if the validation passed, false otherwise.
- */
- boolean handleGithubWebhookValidation(GithubWebhookRequest request, ValidationRequest vr) {
- // update the status before processing
- LOGGER
- .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
- GithubCommitStatuses.PENDING);
- updateCommitStatus(request, GithubCommitStatuses.PENDING);
-
- // validate the response
- LOGGER
- .trace("Begining validation of request for {}/#{}", request.getRepository().getFullName(),
- request.getPullRequest().getNumber());
- ValidationResponse r = validation.validateIncomingRequest(vr, wrapper);
- if (r.getPassed()) {
- LOGGER
- .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
- GithubCommitStatuses.SUCCESS);
- updateCommitStatus(request, GithubCommitStatuses.SUCCESS);
- return true;
- }
- LOGGER
- .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
- GithubCommitStatuses.FAILURE);
- updateCommitStatus(request, GithubCommitStatuses.FAILURE);
- return false;
- }
-
- /**
- * Sends off a POST to update the commit status given a context for the current PR.
- *
- * @param request the current webhook update request
- * @param state the state to set the status to
- * @param fingerprint the internal unique string for the set of commits being processed
- */
- private void updateCommitStatus(GithubWebhookRequest request, GithubCommitStatuses state) {
- LOGGER
- .trace("Generated access token for installation {}: {}", request.getInstallation().getId(),
- jwtHelper.getGithubAccessToken(request.getInstallation().getId()).getToken());
- ghApi
- .updateStatus(jwtHelper.getGhBearerString(request.getInstallation().getId()), apiVersion,
- request.getRepository().getFullName(), request.getPullRequest().getHead().getSha(),
- GithubCommitStatusRequest
- .builder()
- .setDescription(state.getMessage())
- .setState(state.toString())
- .setTargetUrl(webhooksConfig.github().serverTarget() + "/git/eca/status/gh/" + request.getRepository().getFullName() + '/'
- + request.getPullRequest().getNumber())
- .setContext(webhooksConfig.github().context())
- .build());
- }
-
- /**
- * Wraps a nullable value fetch to handle errors and will return null if the value can't be retrieved.
- *
- * @param supplier the method with potentially nullable values
- * @return the value if it can be found, or null
- */
- private String getNullableString(Supplier<String> supplier) {
- try {
- return supplier.get();
- } catch (NullPointerException e) {
- // suppress, as we don't care at this point if its null
- }
- return null;
- }
-
- /**
- * Validates required fields for processing requests.
- *
- * @param installationId the installation ID for the ECA app in the given repository
- * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
- * @param pullRequestNumber the pull request number that is being processed currently
- * @throws BadRequestException if at least one of the parameters is in an invalid state.
- */
- private void checkRequestParameters(String installationId, String repositoryFullName, int pullRequestNumber) {
- List<String> missingFields = new ArrayList<>();
- if (StringUtils.isBlank(installationId)) {
- missingFields.add(GitEcaParameterNames.INSTALLATION_ID_RAW);
- }
- if (StringUtils.isBlank(repositoryFullName)) {
- missingFields.add(GitEcaParameterNames.REPOSITORY_FULL_NAME_RAW);
- }
- if (pullRequestNumber < 1) {
- missingFields.add(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW);
- }
-
- // throw exception if some fields are missing as we can't continue to process the request
- if (!missingFields.isEmpty()) {
- throw new BadRequestException("Missing fields in order to prepare request: " + StringUtils.join(missingFields, ' '));
- }
- }
-
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java b/src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java
index f02a4015b2c7a0ca52dccc5431185caefd30ab62..86125bbbba346109e726dbe00ceb950909e6899f 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java
@@ -25,15 +25,18 @@ import javax.ws.rs.QueryParam;
import javax.ws.rs.ServerErrorException;
import javax.ws.rs.core.Response;
+import org.apache.http.HttpStatus;
import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
import org.eclipsefoundation.git.eca.helper.CaptchaHelper;
+import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
import org.eclipsefoundation.git.eca.model.RevalidationResponse;
import org.eclipsefoundation.git.eca.model.ValidationRequest;
import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
import org.eclipsefoundation.git.eca.namespace.HCaptchaErrorCodes;
import org.eclipsefoundation.git.eca.namespace.WebhookHeaders;
+import org.eclipsefoundation.git.eca.service.GithubApplicationService;
import org.jboss.resteasy.annotations.jaxrs.HeaderParam;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,6 +51,11 @@ import org.slf4j.LoggerFactory;
public class GithubWebhooksResource extends GithubAdjacentResource {
private static final Logger LOGGER = LoggerFactory.getLogger(GithubWebhooksResource.class);
+ @Inject
+ GithubApplicationService ghAppService;
+
+ @Inject
+ GithubValidationHelper validationHelper;
@Inject
CaptchaHelper captchaHelper;
@@ -70,14 +78,23 @@ public class GithubWebhooksResource extends GithubAdjacentResource {
if (!"pull_request".equalsIgnoreCase(eventType)) {
return Response.ok().build();
}
- LOGGER.trace("Processing PR event for install {} with delivery ID of {}", request.getInstallation().getId(), deliveryId);
- // prepare for validation process by pre-processing into standard format
- ValidationRequest vr = generateRequest(request);
+
// track the request before we start processing
- retrieveAndUpdateTrackingInformation(request.getInstallation().getId(), request.getRepository().getFullName(),
- request.getPullRequest());
- // process the request
- handleGithubWebhookValidation(request, vr);
+ GithubWebhookTracking tracking = validationHelper
+ .retrieveAndUpdateTrackingInformation(wrapper, request.getInstallation().getId(), request.getRepository().getFullName(),
+ request.getPullRequest());
+
+ // start processing the request
+ LOGGER.trace("Processing PR event for install {} with delivery ID of {}", request.getInstallation().getId(), deliveryId);
+ try {
+ // prepare for validation process by pre-processing into standard format
+ ValidationRequest vr = generateRequest(request);
+ // process the request
+ validationHelper.handleGithubWebhookValidation(request, vr, wrapper);
+ } catch (Exception e) {
+ // set the revalidation flag to reprocess if the initial attempt fails before exiting
+ setRevalidationFlagForTracking(tracking);
+ }
return Response.ok().build();
}
@@ -105,11 +122,12 @@ public class GithubWebhooksResource extends GithubAdjacentResource {
}
// get the tracking if it exists, create it if it doesn't, and fail out if there is an issue
- GithubWebhookTracking tracking = retrieveAndUpdateTrackingInformation(installationId, fullRepoName, prResponse.get());
+ GithubWebhookTracking tracking = validationHelper
+ .retrieveAndUpdateTrackingInformation(wrapper, installationId, fullRepoName, prResponse.get());
if (tracking == null) {
throw new ServerErrorException(
String.format("Cannot find a tracked pull request with for repo '%s', pull request number '%d'", fullRepoName, prNo),
- 500);
+ HttpStatus.SC_INTERNAL_SERVER_ERROR);
} else if (!"open".equalsIgnoreCase(tracking.getLastKnownState())) {
// we do not want to reprocess non-open pull requests
throw new BadRequestException("Cannot revalidate a non-open pull request");
@@ -125,19 +143,25 @@ public class GithubWebhooksResource extends GithubAdjacentResource {
throw new BadRequestException("hCaptcha challenge response failed for this request");
}
- // get the tracking class, convert back to a GH webhook request, and validate the request
- GithubWebhookRequest request = GithubWebhookRequest.buildFromTracking(tracking);
- boolean isSuccessful = handleGithubWebhookValidation(request, generateRequest(request));
- LOGGER.debug("Revalidation for request for '{}#{}' was {}successful.", fullRepoName, prNo, isSuccessful ? "" : " not");
-
- // build the url for pull request page
- StringBuilder sb = new StringBuilder();
- sb.append("https://github.com/");
- sb.append(tracking.getRepositoryFullName());
- sb.append("/pull/");
- sb.append(tracking.getPullRequestNumber());
- // respond with a URL to the new location in a standard request
- return Response.ok(RevalidationResponse.builder().setLocation(URI.create(sb.toString())).build()).build();
+ // wrap the processing in a try-catch to catch upstream errors and recast them as server errors
+ try {
+ // get the tracking class, convert back to a GH webhook request, and validate the request
+ GithubWebhookRequest request = GithubWebhookRequest.buildFromTracking(tracking);
+ boolean isSuccessful = validationHelper.handleGithubWebhookValidation(request, generateRequest(request), wrapper).getPassed();
+ LOGGER.debug("Revalidation for request for '{}#{}' was {}successful.", fullRepoName, prNo, isSuccessful ? "" : " not");
+
+ // build the url for pull request page
+ StringBuilder sb = new StringBuilder();
+ sb.append("https://github.com/");
+ sb.append(fullRepoName);
+ sb.append("/pull/");
+ sb.append(prNo);
+ // respond with a URL to the new location in a standard request
+ return Response.ok(RevalidationResponse.builder().setLocation(URI.create(sb.toString())).build()).build();
+ } catch (Exception e) {
+ // rewrap exception, as some of the Github stuff can fail w/o explanation
+ throw new ServerErrorException(HttpStatus.SC_INTERNAL_SERVER_ERROR, e);
+ }
}
/**
@@ -147,8 +171,9 @@ public class GithubWebhooksResource extends GithubAdjacentResource {
* @return the Validation Request body to be used in validating data
*/
private ValidationRequest generateRequest(GithubWebhookRequest request) {
- return generateRequest(request.getInstallation().getId(), request.getRepository().getFullName(),
- request.getPullRequest().getNumber(), request.getRepository().getHtmlUrl());
+ return validationHelper
+ .generateRequest(request.getInstallation().getId(), request.getRepository().getFullName(),
+ request.getPullRequest().getNumber(), request.getRepository().getHtmlUrl());
}
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java b/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
index 8fd1b44f975684f587066fbad69bf7b836aaafbd..aeef62979e8229cbbb52fdce5d5b89de391eeb17 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
@@ -12,31 +12,24 @@
package org.eclipsefoundation.git.eca.resource;
import java.util.List;
-import java.util.Optional;
import java.util.stream.Collectors;
import javax.inject.Inject;
-import javax.ws.rs.BadRequestException;
import javax.ws.rs.GET;
import javax.ws.rs.NotFoundException;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
-import javax.ws.rs.ServerErrorException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import org.apache.commons.lang3.StringUtils;
import org.eclipsefoundation.efservices.api.models.Project;
-import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
-import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
import org.eclipsefoundation.git.eca.dto.CommitValidationStatus;
-import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
+import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
import org.eclipsefoundation.git.eca.helper.ProjectHelper;
-import org.eclipsefoundation.git.eca.model.Commit;
-import org.eclipsefoundation.git.eca.model.ValidationRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.eclipsefoundation.git.eca.model.ValidationResponse;
+import org.eclipsefoundation.git.eca.service.GithubApplicationService;
+import org.eclipsefoundation.git.eca.service.ValidationService;
import io.quarkus.qute.Location;
import io.quarkus.qute.Template;
@@ -49,8 +42,14 @@ import io.quarkus.qute.Template;
*/
@Path("eca/status")
public class StatusResource extends GithubAdjacentResource {
- private static final Logger LOGGER = LoggerFactory.getLogger(StatusResource.class);
+ @Inject
+ GithubApplicationService ghAppService;
+ @Inject
+ ValidationService validation;
+
+ @Inject
+ GithubValidationHelper validationHelper;
@Inject
ProjectHelper projects;
@@ -113,49 +112,12 @@ public class StatusResource extends GithubAdjacentResource {
@Path("gh/{org}/{repoName}/{prNo}")
public Response getCommitValidationForGithub(@PathParam("org") String org, @PathParam("repoName") String repoName,
@PathParam("prNo") Integer prNo) {
- String fullRepoName = org + '/' + repoName;
- // get the installation ID for the given repo if it exists, and if the PR noted exists
- String installationId = ghAppService.getInstallationForRepo(fullRepoName);
- Optional<PullRequest> prResponse = ghAppService.getPullRequest(installationId, fullRepoName, prNo);
- if (StringUtils.isBlank(installationId)) {
- throw new BadRequestException("Could not find an ECA app installation for repo name: " + fullRepoName);
- } else if (prResponse.isEmpty()) {
- throw new NotFoundException(String.format("Could not find PR '%d' in repo name '%s'", prNo, fullRepoName));
- }
-
- // prepare the request for consumption
- String repoUrl = "https://github.com/" + fullRepoName;
- ValidationRequest vr = generateRequest(installationId, fullRepoName, prNo, repoUrl);
-
- // build the commit sha list based on the prepared request
- List<String> commitShas = vr.getCommits().stream().map(Commit::getHash).collect(Collectors.toList());
- // there should always be commits for a PR, but in case, lets check
- if (commitShas.isEmpty()) {
- throw new BadRequestException(String.format("Could not find any commits for %s#%d", fullRepoName, prNo));
- }
- LOGGER.debug("Found {} commits for '{}#{}'", commitShas.size(), fullRepoName, prNo);
-
- // retrieve the webhook tracking info, or generate an entry to track this PR if it's missing.
- GithubWebhookTracking updatedTracking = retrieveAndUpdateTrackingInformation(installationId, fullRepoName, prResponse.get());
- if (updatedTracking == null) {
- throw new ServerErrorException("Error while attempting to revalidate request, try again later.", 500);
- }
-
- // get the commit status of commits to use for checking historic validation
- List<CommitValidationStatus> statuses = validation.getHistoricValidationStatusByShas(wrapper, commitShas);
- if (!"open".equalsIgnoreCase(prResponse.get().getState()) && statuses.isEmpty()) {
- throw new BadRequestException("Cannot find validation history for current non-open PR, cannot provide validation status");
- }
-
- // we only want to update/revalidate for open PRs, so don't do this check if the PR is merged/closed
- if ("open".equalsIgnoreCase(prResponse.get().getState()) && commitShas.size() != statuses.size()) {
- LOGGER.debug("Validation for {}#{} does not seem to be current, revalidating commits", fullRepoName, prNo);
- // using the updated tracking, perform the validation
- handleGithubWebhookValidation(GithubWebhookRequest.buildFromTracking(updatedTracking), vr);
- // call to retrieve the statuses once again since they will have changed at this point
- statuses = validation.getHistoricValidationStatusByShas(wrapper, commitShas);
- }
+ // run the validation for the current request
+ ValidationResponse r = validationHelper.validateIncomingRequest(wrapper, org, repoName, prNo);
+ // retrieve the status of the commits to display on the status page
+ List<CommitValidationStatus> statuses = validation
+ .getHistoricValidationStatusByShas(wrapper, r.getCommits().keySet().stream().collect(Collectors.toList()));
// get projects for use in status UI
List<Project> ps = projects.retrieveProjectsForRepoURL(statuses.get(0).getRepoUrl(), statuses.get(0).getProvider());
// render and return the status UI
@@ -164,10 +126,10 @@ public class StatusResource extends GithubAdjacentResource {
.entity(statusUiTemplate
.data("statuses", statuses)
.data("pullRequestNumber", prNo)
- .data("fullRepoName", fullRepoName)
+ .data("fullRepoName", org + '/' + repoName)
.data("project", ps.isEmpty() ? null : ps.get(0))
- .data("repoUrl", repoUrl)
- .data("installationId", installationId)
+ .data("repoUrl", GithubValidationHelper.getRepoUrl(org, repoName))
+ .data("installationId", ghAppService.getInstallationForRepo(org + '/' + repoName))
.render())
.build();
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java b/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java
new file mode 100644
index 0000000000000000000000000000000000000000..cfaedf155b5713945bbcabbd231045b1dbc2013b
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java
@@ -0,0 +1,143 @@
+/**
+ * Copyright (c) 2023 Eclipse Foundation
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * Author: Martin Lowe <martin.lowe@eclipse-foundation.org>
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.git.eca.tasks;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.List;
+
+import javax.annotation.PostConstruct;
+import javax.enterprise.context.ApplicationScoped;
+import javax.enterprise.context.control.ActivateRequestContext;
+import javax.enterprise.inject.Instance;
+import javax.inject.Inject;
+import javax.ws.rs.core.MultivaluedMap;
+
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+import org.eclipsefoundation.core.helper.DateTimeHelper;
+import org.eclipsefoundation.core.model.FlatRequestWrapper;
+import org.eclipsefoundation.core.model.RequestWrapper;
+import org.eclipsefoundation.core.namespace.DefaultUrlParameterNames;
+import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
+import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
+import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
+import org.eclipsefoundation.persistence.dao.PersistenceDao;
+import org.eclipsefoundation.persistence.model.RDBMSQuery;
+import org.eclipsefoundation.persistence.namespace.PersistenceUrlParameterNames;
+import org.eclipsefoundation.persistence.service.FilterService;
+import org.jboss.resteasy.specimpl.MultivaluedMapImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.quarkus.scheduler.Scheduled;
+
+/**
+ * Scheduled regular task that will interact with the backend persistence to look for requests that are in a
+ * failed/unvalidated state after an error while processing that could not be recovered. These requests will be
+ * reprocessed using the same logic as the standard validation, updating the timestamp on completion and either setting
+ * the revalidation flag to false or incrementing the number of repeated revalidations needed for the request for
+ * tracking, depending on the succcess of the revalidation.
+ */
+@ApplicationScoped
+public class GithubRevalidationQueue {
+ private static final Logger LOGGER = LoggerFactory.getLogger(GithubRevalidationQueue.class);
+
+ @ConfigProperty(name = "eclipse.git-eca.tasks.gh-revalidation.enabled", defaultValue = "true")
+ Instance<Boolean> isEnabled;
+
+ @Inject
+ PersistenceDao dao;
+ @Inject
+ FilterService filters;
+
+ @Inject
+ GithubValidationHelper validationHelper;
+
+ @PostConstruct
+ void init() {
+ // indicate to log whether enabled to reduce log spam
+ LOGGER.info("Github revalidation queue task is{} enabled.", Boolean.TRUE.equals(isEnabled.get()) ? "" : " not");
+ }
+
+ /**
+ * Every 5s, this method will attempt to load a Github webhook validation request that has the needs revalidation flag
+ * set to true. This will retrieve the oldest request in queue and will attempt to revalidate it.
+ */
+ @Scheduled(every = "5s")
+ @ActivateRequestContext
+ public void revalidate() {
+ // if not enabled, don't process any potentially OOD records
+ if (!Boolean.TRUE.equals(isEnabled.get())) {
+ return;
+ }
+
+ // set up params for looking up the top of the revalidation queue
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
+ params.add(GitEcaParameterNames.NEEDS_REVALIDATION_RAW, "true");
+ params.add(PersistenceUrlParameterNames.SORT.getName(), "lastUpdated");
+ params.add(DefaultUrlParameterNames.PAGESIZE.getName(), "1");
+
+ // build the request and query to lookup the longest standing request that needs revalidation
+ RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/eca/revalidation-queue"));
+ RDBMSQuery<GithubWebhookTracking> trackingQuery = new RDBMSQuery<>(wrap, filters.get(GithubWebhookTracking.class), params);
+
+ List<GithubWebhookTracking> oldestRevalidation = dao.get(trackingQuery);
+ if (oldestRevalidation.isEmpty()) {
+ LOGGER.debug("No queued revalidation requests found");
+ } else {
+ reprocessRequest(oldestRevalidation.get(0), wrap);
+ }
+ }
+
+ /**
+ * Reprocess the given record, attempting to run the ECA validation logic again. If it passes, the revalidation flag is
+ * set to false and the time code is updated. If the processing fails again, the failure count gets incremented and the
+ * updated time is set so that another entry can be updated, as to not block on potentially broken records.
+ *
+ * @param requestToRevalidate the webhook tracking request to attempt to revalidate
+ * @param wrap the current stubbed request wrapper used for queries.
+ */
+ private void reprocessRequest(GithubWebhookTracking requestToRevalidate, RequestWrapper wrap) {
+ LOGGER
+ .debug("Attempting revalidation of request w/ ID {}, in repo {}#{}", requestToRevalidate.getId(),
+ requestToRevalidate.getRepositoryFullName(), requestToRevalidate.getPullRequestNumber());
+
+ // wrap in try-catch to avoid errors from stopping the record updates
+ try {
+ // split the full repo name into the org and repo name
+ String[] repoFullNameParts = requestToRevalidate.getRepositoryFullName().split("/");
+ if (repoFullNameParts.length != 2) {
+ throw new IllegalStateException("Record with ID '" + Long.toString(requestToRevalidate.getId())
+ + "' is in an invalid state (repository full name is not valid)");
+ }
+
+ // run the validation and then check if it succeeded
+ validationHelper
+ .validateIncomingRequest(wrap, repoFullNameParts[0], repoFullNameParts[1], requestToRevalidate.getPullRequestNumber());
+ // if we have gotten here, then the validation has completed and can be removed from queue
+ requestToRevalidate.setNeedsRevalidation(false);
+ LOGGER.debug("Sucessfully revalidated request w/ ID {}", requestToRevalidate.getId());
+ } catch (RuntimeException e) {
+ // update the number of times this status has revalidated (tracking)
+ requestToRevalidate
+ .setManualRevalidationCount(requestToRevalidate.getManualRevalidationCount() == null ? 1
+ : requestToRevalidate.getManualRevalidationCount() + 1);
+ // log the message so we can see what happened
+ LOGGER.error("Error while revalidating request w/ ID {}", requestToRevalidate.getId(), e);
+ } finally {
+ // whether successful or failed, update the updated field
+ requestToRevalidate.setLastUpdated(DateTimeHelper.now());
+ // push the update with the potentially updated validation flag or error count
+ dao.add(new RDBMSQuery<>(wrap, filters.get(GithubWebhookTracking.class)), Arrays.asList(requestToRevalidate));
+ }
+ }
+}
diff --git a/src/test/java/org/eclipsefoundation/git/eca/resource/ReportsResourceTest.java b/src/test/java/org/eclipsefoundation/git/eca/resource/ReportsResourceTest.java
index 8def7dcf42cb21cacafe2cbd42a365eb1537380a..06c6ee66a0b4191e24c887db1951b20ce4c3ad9d 100644
--- a/src/test/java/org/eclipsefoundation/git/eca/resource/ReportsResourceTest.java
+++ b/src/test/java/org/eclipsefoundation/git/eca/resource/ReportsResourceTest.java
@@ -13,6 +13,7 @@ package org.eclipsefoundation.git.eca.resource;
import java.util.Optional;
+import org.apache.http.HttpStatus;
import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
import org.eclipsefoundation.git.eca.test.namespaces.SchemaNamespaceHelper;
import org.eclipsefoundation.testing.helpers.TestCaseHelper;
diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties
index ed2bc8fb08d1992cd53873ad2ad94b65c3eb3d10..4b362d9386195a064dfb14940077dce551c3236d 100644
--- a/src/test/resources/application.properties
+++ b/src/test/resources/application.properties
@@ -34,6 +34,7 @@ eclipse.git-eca.mail.allow-list=noreply@github.com
## Reports configs
eclipse.git-eca.reports.access-key=samplekey
+eclipse.git-eca.tasks.gh-revalidation.enabled=false
## Misc
eclipse.gitlab.access-token=token_val
diff --git a/src/test/resources/database/default/V1.0.0__default.sql b/src/test/resources/database/default/V1.0.0__default.sql
index 100590d363360f140356726c5d366cfe12a9e56d..25e97153916cc74045080f8e33aaacad9c09fb8f 100644
--- a/src/test/resources/database/default/V1.0.0__default.sql
+++ b/src/test/resources/database/default/V1.0.0__default.sql
@@ -56,5 +56,7 @@ CREATE TABLE GithubWebhookTracking (
repositoryFullName varchar(127) NOT NULL,
pullRequestNumber int NOT NULL,
lastUpdated datetime DEFAULT NULL,
+ needsRevalidation BOOLEAN DEFAULT FALSE,
+ manualRevalidationCount int DEFAULT 0,
PRIMARY KEY (id)
);
\ No newline at end of file