From 1f3ec1b1344a7c339ec7e4513b68d0fbcb1930c2 Mon Sep 17 00:00:00 2001
From: Martin Lowe <martin.lowe@eclipse-foundation.org>
Date: Mon, 14 Aug 2023 12:04:24 -0400
Subject: [PATCH 1/2] Iss #115 - Add a reprocessing queue for failed GH webhook
calls
To resolve an issue where some GH calls are timing out unexpectedly
during validation, a new scheduled task was created, along with some
updates to the DB models.
The new task uses the new revalidation flag on the tracking model to
look up requests to see if they need to be revalidated, and rerun
validation on the oldest set entry. When run, the last updated field
will be updated regardless of success of validation. This will allow us
to not get stuck on a single tracking request that is in a bad state
potentially.
The DB model changes add 2 new fields to webhook tracking entries,
allowing for tracking of if a scheduled revalidation needs to be run and
how many times a given webhook validation needed to be rerun. This
should give us both the flexibility to rerun as needed, manually flag
errant entries, as well as capture data on how often this happens.
---
.../git/eca/dto/GithubWebhookTracking.java | 44 +-
.../eca/helper/GithubValidationHelper.java | 391 ++++++++++++++++++
.../eca/namespace/GitEcaParameterNames.java | 4 +-
.../eca/resource/GithubAdjacentResource.java | 280 +------------
.../eca/resource/GithubWebhooksResource.java | 73 ++--
.../git/eca/resource/StatusResource.java | 77 +---
.../eca/tasks/GithubRevalidationQueue.java | 142 +++++++
.../git/eca/resource/ReportsResourceTest.java | 1 +
.../database/default/V1.0.0__default.sql | 2 +
9 files changed, 652 insertions(+), 362 deletions(-)
create mode 100644 src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java
create mode 100644 src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java
diff --git a/src/main/java/org/eclipsefoundation/git/eca/dto/GithubWebhookTracking.java b/src/main/java/org/eclipsefoundation/git/eca/dto/GithubWebhookTracking.java
index 72cb98ce..e40f3f33 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/dto/GithubWebhookTracking.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/dto/GithubWebhookTracking.java
@@ -30,6 +30,7 @@ import org.eclipsefoundation.persistence.dto.filter.DtoFilter;
import org.eclipsefoundation.persistence.model.DtoTable;
import org.eclipsefoundation.persistence.model.ParameterizedSQLStatement;
import org.eclipsefoundation.persistence.model.ParameterizedSQLStatementBuilder;
+import org.eclipsefoundation.persistence.model.SortableField;
/**
* Github Webhook request tracking info. Tracking is required to trigger revalidation through the Github API from this
@@ -51,7 +52,10 @@ public class GithubWebhookTracking extends BareNode {
private String headSha;
private Integer pullRequestNumber;
private String lastKnownState;
+ @SortableField
private ZonedDateTime lastUpdated;
+ private boolean needsRevalidation;
+ private Integer manualRevalidationCount;
@Override
public Long getId() {
@@ -149,12 +153,41 @@ public class GithubWebhookTracking extends BareNode {
this.lastUpdated = lastUpdated;
}
+ /**
+ * @return the needsRevalidation
+ */
+ public boolean isNeedsRevalidation() {
+ return needsRevalidation;
+ }
+
+ /**
+ * @param needsRevalidation the needsRevalidation to set
+ */
+ public void setNeedsRevalidation(boolean needsRevalidation) {
+ this.needsRevalidation = needsRevalidation;
+ }
+
+ /**
+ * @return the manualRevalidationCount
+ */
+ public Integer getManualRevalidationCount() {
+ return manualRevalidationCount;
+ }
+
+ /**
+ * @param manualRevalidationCount the manualRevalidationCount to set
+ */
+ public void setManualRevalidationCount(Integer manualRevalidationCount) {
+ this.manualRevalidationCount = manualRevalidationCount;
+ }
+
@Override
public int hashCode() {
final int prime = 31;
int result = super.hashCode();
- result = prime * result
- + Objects.hash(headSha, id, installationId, lastKnownState, lastUpdated, pullRequestNumber, repositoryFullName);
+ result = prime * result + Objects
+ .hash(headSha, id, installationId, lastKnownState, lastUpdated, manualRevalidationCount, needsRevalidation,
+ pullRequestNumber, repositoryFullName);
return result;
}
@@ -172,7 +205,8 @@ public class GithubWebhookTracking extends BareNode {
GithubWebhookTracking other = (GithubWebhookTracking) obj;
return Objects.equals(headSha, other.headSha) && Objects.equals(id, other.id)
&& Objects.equals(installationId, other.installationId) && Objects.equals(lastKnownState, other.lastKnownState)
- && Objects.equals(lastUpdated, other.lastUpdated) && Objects.equals(pullRequestNumber, other.pullRequestNumber)
+ && Objects.equals(lastUpdated, other.lastUpdated) && Objects.equals(manualRevalidationCount, other.manualRevalidationCount)
+ && needsRevalidation == other.needsRevalidation && Objects.equals(pullRequestNumber, other.pullRequestNumber)
&& Objects.equals(repositoryFullName, other.repositoryFullName);
}
@@ -204,6 +238,10 @@ public class GithubWebhookTracking extends BareNode {
.addClause(new ParameterizedSQLStatement.Clause(TABLE.getAlias() + ".repositoryFullName = ?",
new Object[] { repositoryFullName }));
}
+ String needsRevalidation = params.getFirst(GitEcaParameterNames.NEEDS_REVALIDATION_RAW);
+ if (Boolean.TRUE.equals(Boolean.valueOf(needsRevalidation))) {
+ statement.addClause(new ParameterizedSQLStatement.Clause(TABLE.getAlias() + ".needsRevalidation = TRUE", new Object[] {}));
+ }
return statement;
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java b/src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java
new file mode 100644
index 00000000..9f4c34a0
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java
@@ -0,0 +1,391 @@
+/**
+ * Copyright (c) 2023 Eclipse Foundation
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * Author: Martin Lowe <martin.lowe@eclipse-foundation.org>
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.git.eca.helper;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import javax.enterprise.context.ApplicationScoped;
+import javax.inject.Inject;
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.ServerErrorException;
+import javax.ws.rs.core.MultivaluedMap;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpStatus;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+import org.eclipse.microprofile.rest.client.inject.RestClient;
+import org.eclipsefoundation.core.helper.DateTimeHelper;
+import org.eclipsefoundation.core.model.RequestWrapper;
+import org.eclipsefoundation.core.service.APIMiddleware;
+import org.eclipsefoundation.git.eca.api.GithubAPI;
+import org.eclipsefoundation.git.eca.api.models.GithubCommit;
+import org.eclipsefoundation.git.eca.api.models.GithubCommit.ParentCommit;
+import org.eclipsefoundation.git.eca.api.models.GithubCommitStatusRequest;
+import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
+import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
+import org.eclipsefoundation.git.eca.config.WebhooksConfig;
+import org.eclipsefoundation.git.eca.dto.CommitValidationStatus;
+import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
+import org.eclipsefoundation.git.eca.model.Commit;
+import org.eclipsefoundation.git.eca.model.GitUser;
+import org.eclipsefoundation.git.eca.model.ValidationRequest;
+import org.eclipsefoundation.git.eca.model.ValidationResponse;
+import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
+import org.eclipsefoundation.git.eca.namespace.GithubCommitStatuses;
+import org.eclipsefoundation.git.eca.namespace.ProviderType;
+import org.eclipsefoundation.git.eca.service.GithubApplicationService;
+import org.eclipsefoundation.git.eca.service.ValidationService;
+import org.eclipsefoundation.persistence.dao.PersistenceDao;
+import org.eclipsefoundation.persistence.model.RDBMSQuery;
+import org.eclipsefoundation.persistence.service.FilterService;
+import org.jboss.resteasy.specimpl.MultivaluedMapImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class is used to adapt Github requests to the standard validation workflow in a way that could be reused by both
+ * resource calls and scheduled tasks for revalidation.
+ */
+@ApplicationScoped
+public class GithubValidationHelper {
+ private static final Logger LOGGER = LoggerFactory.getLogger(GithubValidationHelper.class);
+
+ private static final String VALIDATION_LOGGING_MESSAGE = "Setting validation state for {}/#{} to {}";
+
+ @ConfigProperty(name = "eclipse.github.default-api-version", defaultValue = "2022-11-28")
+ String apiVersion;
+
+ @Inject
+ WebhooksConfig webhooksConfig;
+
+ @Inject
+ JwtHelper jwtHelper;
+ @Inject
+ APIMiddleware middleware;
+ @Inject
+ PersistenceDao dao;
+ @Inject
+ FilterService filters;
+
+ @Inject
+ ValidationService validation;
+ @Inject
+ GithubApplicationService ghAppService;
+
+ @RestClient
+ GithubAPI ghApi;
+
+ /**
+ * Using the wrapper and the passed unique information about a Github validation instance, lookup or create the tracking
+ * request and validate the data.
+ *
+ * @param wrapper the wrapper for the current request
+ * @param org the name of the GH organization
+ * @param repoName the slug of the repo that has the PR to be validated
+ * @param prNo the PR number for the current request.
+ * @return the validated response if it is a valid request, or throws a web exception if there is a problem validating
+ * the request.
+ */
+ public ValidationResponse validateIncomingRequest(RequestWrapper wrapper, String org, String repoName, Integer prNo) {
+ String fullRepoName = org + '/' + repoName;
+ // get the installation ID for the given repo if it exists, and if the PR noted exists
+ String installationId = ghAppService.getInstallationForRepo(fullRepoName);
+ Optional<PullRequest> prResponse = ghAppService.getPullRequest(installationId, fullRepoName, prNo);
+ if (StringUtils.isBlank(installationId)) {
+ throw new BadRequestException("Could not find an ECA app installation for repo name: " + fullRepoName);
+ } else if (prResponse.isEmpty()) {
+ throw new NotFoundException(String.format("Could not find PR '%d' in repo name '%s'", prNo, fullRepoName));
+ }
+
+ // prepare the request for consumption
+ String repoUrl = getRepoUrl(org, repoName);
+ ValidationRequest vr = generateRequest(installationId, fullRepoName, prNo, repoUrl);
+
+ // build the commit sha list based on the prepared request
+ List<String> commitShas = vr.getCommits().stream().map(Commit::getHash).collect(Collectors.toList());
+ // there should always be commits for a PR, but in case, lets check
+ if (commitShas.isEmpty()) {
+ throw new BadRequestException(String.format("Could not find any commits for %s#%d", fullRepoName, prNo));
+ }
+ LOGGER.debug("Found {} commits for '{}#{}'", commitShas.size(), fullRepoName, prNo);
+
+ // retrieve the webhook tracking info, or generate an entry to track this PR if it's missing.
+ GithubWebhookTracking updatedTracking = retrieveAndUpdateTrackingInformation(wrapper, installationId, fullRepoName,
+ prResponse.get());
+ if (updatedTracking == null) {
+ throw new ServerErrorException("Error while attempting to revalidate request, try again later.",
+ HttpStatus.SC_INTERNAL_SERVER_ERROR);
+ }
+
+ // get the commit status of commits to use for checking historic validation
+ List<CommitValidationStatus> statuses = validation.getHistoricValidationStatusByShas(wrapper, commitShas);
+ if (!"open".equalsIgnoreCase(prResponse.get().getState()) && statuses.isEmpty()) {
+ throw new BadRequestException("Cannot find validation history for current non-open PR, cannot provide validation status");
+ }
+
+ // we only want to update/revalidate for open PRs, so don't do this check if the PR is merged/closed
+ if ("open".equalsIgnoreCase(prResponse.get().getState()) && commitShas.size() != statuses.size()) {
+ LOGGER.debug("Validation for {}#{} does not seem to be current, revalidating commits", fullRepoName, prNo);
+ // using the updated tracking, perform the validation
+ return handleGithubWebhookValidation(GithubWebhookRequest.buildFromTracking(updatedTracking), vr, wrapper);
+ }
+ return null;
+ }
+
+ /**
+ * Generate a ValidationRequest object based on data pulled from Github, grabbing commits from the noted pull request
+ * using the installation ID for access/authorization.
+ *
+ * @param installationId the ECA app installation ID for the organization
+ * @param repositoryFullName the full name of the repository where the PR resides
+ * @param pullRequestNumber the pull request number that is being validated
+ * @param repositoryUrl the URL of the repository that contains the commits to validate
+ * @return the populated validation request for the Github request information
+ */
+ @SuppressWarnings("null")
+ public ValidationRequest generateRequest(String installationId, String repositoryFullName, int pullRequestNumber,
+ String repositoryUrl) {
+ checkRequestParameters(installationId, repositoryFullName, pullRequestNumber);
+ // get the commits that will be validated, don't cache as changes can come in too fast for it to be useful
+ List<GithubCommit> commits = middleware
+ .getAll(i -> ghApi
+ .getCommits(jwtHelper.getGhBearerString(installationId), apiVersion, repositoryFullName, pullRequestNumber),
+ GithubCommit.class);
+ LOGGER.trace("Retrieved {} commits for PR {} in repo {}", commits.size(), pullRequestNumber, repositoryUrl);
+ // set up the validation request from current data
+ return ValidationRequest
+ .builder()
+ .setProvider(ProviderType.GITHUB)
+ .setRepoUrl(URI.create(repositoryUrl))
+ .setStrictMode(true)
+ .setCommits(commits
+ .stream()
+ .map(c -> Commit
+ .builder()
+ .setHash(c.getSha())
+ .setAuthor(GitUser
+ .builder()
+ .setMail(getNullableString(() -> c.getCommit().getAuthor().getEmail()))
+ .setName(getNullableString(() -> c.getCommit().getAuthor().getName()))
+ .setExternalId(getNullableString(() -> c.getAuthor().getLogin()))
+ .build())
+ .setCommitter(GitUser
+ .builder()
+ .setMail(getNullableString(() -> c.getCommit().getCommitter().getEmail()))
+ .setName(getNullableString(() -> c.getCommit().getCommitter().getName()))
+ .setExternalId(getNullableString(() -> c.getCommitter().getLogin()))
+ .build())
+ .setParents(c.getParents().stream().map(ParentCommit::getSha).collect(Collectors.toList()))
+ .build())
+ .collect(Collectors.toList()))
+ .build();
+ }
+
+ /**
+ * Process the current request and update the checks state to pending then success or failure. Contains verbose TRACE
+ * logging for more info on the states of the validation for more information
+ *
+ * @param request information about the request from the GH webhook on what resource requested revalidation. Used to
+ * target the commit status of the resources
+ * @param vr the pseudo request generated from the contextual webhook data. Used to make use of existing validation
+ * logic.
+ * @return true if the validation passed, false otherwise.
+ */
+ public ValidationResponse handleGithubWebhookValidation(GithubWebhookRequest request, ValidationRequest vr, RequestWrapper wrapper) {
+ // update the status before processing
+ LOGGER
+ .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
+ GithubCommitStatuses.PENDING);
+ updateCommitStatus(request, GithubCommitStatuses.PENDING);
+
+ // validate the response
+ LOGGER
+ .trace("Begining validation of request for {}/#{}", request.getRepository().getFullName(),
+ request.getPullRequest().getNumber());
+ ValidationResponse r = validation.validateIncomingRequest(vr, wrapper);
+ if (r.getPassed()) {
+ LOGGER
+ .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
+ GithubCommitStatuses.SUCCESS);
+ updateCommitStatus(request, GithubCommitStatuses.SUCCESS);
+ } else {
+ LOGGER
+ .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
+ GithubCommitStatuses.FAILURE);
+ updateCommitStatus(request, GithubCommitStatuses.FAILURE);
+ }
+ return r;
+ }
+
+ /**
+ * Shortcut method that will retrieve existing GH tracking info, create new entries if missing, and will update the
+ * state of existing requests as well.
+ *
+ * @param installationId the installation ID for the ECA app in the given repository
+ * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
+ * @param pr the pull request targeted by the validation request.
+ * @return a new or updated tracking object, or null if there was an error in saving the information
+ */
+ public GithubWebhookTracking retrieveAndUpdateTrackingInformation(RequestWrapper wrapper, String installationId,
+ String repositoryFullName, PullRequest pr) {
+ return updateGithubTrackingIfMissing(wrapper,
+ getExistingRequestInformation(wrapper, installationId, repositoryFullName, pr.getNumber()), pr, installationId,
+ repositoryFullName);
+ }
+
+ /**
+ * Checks if the Github tracking is present for the current request, and if missing will generate a new record and save
+ * it.
+ *
+ * @param tracking the optional tracking entry for the current request
+ * @param request the pull request that is being validated
+ * @param installationId the ECA app installation ID for the current request
+ * @param fullRepoName the full repo name for the validation request
+ */
+ public GithubWebhookTracking updateGithubTrackingIfMissing(RequestWrapper wrapper, Optional<GithubWebhookTracking> tracking,
+ PullRequest request, String installationId, String fullRepoName) {
+ // if there is no tracking present, create the missing tracking and persist it
+ GithubWebhookTracking updatedTracking;
+ if (tracking.isEmpty()) {
+ updatedTracking = new GithubWebhookTracking();
+ updatedTracking.setHeadSha(request.getHead().getSha());
+ updatedTracking.setInstallationId(installationId);
+ updatedTracking.setLastUpdated(DateTimeHelper.now());
+ updatedTracking.setPullRequestNumber(request.getNumber());
+ updatedTracking.setRepositoryFullName(fullRepoName);
+ updatedTracking.setLastKnownState(request.getState());
+ updatedTracking.setNeedsRevalidation(false);
+ updatedTracking.setManualRevalidationCount(0);
+ if (!"open".equalsIgnoreCase(request.getState())) {
+ LOGGER
+ .warn("The PR {} in {} is not in an open state, and will not be validated to follow our validation practice",
+ updatedTracking.getPullRequestNumber(), fullRepoName);
+ }
+ } else {
+ // at least update the state on every run
+ updatedTracking = tracking.get();
+ updatedTracking.setLastKnownState(request.getState());
+ }
+
+ // save the data, and log on its success or failure
+ List<GithubWebhookTracking> savedTracking = dao
+ .add(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class)), Arrays.asList(updatedTracking));
+ if (savedTracking.isEmpty()) {
+ LOGGER.warn("Unable to create new GH tracking record for request to validate {}#{}", fullRepoName, request.getNumber());
+ return null;
+ }
+ // return the updated tracking when successful
+ LOGGER.debug("Created/updated GH tracking record for request to validate {}#{}", fullRepoName, request.getNumber());
+ return savedTracking.get(0);
+ }
+
+ /**
+ * Attempts to retrieve a webhook tracking record given the installation, repository, and pull request number.
+ *
+ * @param installationId the installation ID for the ECA app in the given repository
+ * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
+ * @param pullRequestNumber the pull request number that is being processed currently
+ * @return the webhook tracking record if it can be found, or an empty optional.
+ */
+ public Optional<GithubWebhookTracking> getExistingRequestInformation(RequestWrapper wrapper, String installationId,
+ String repositoryFullName, int pullRequestNumber) {
+ checkRequestParameters(installationId, repositoryFullName, pullRequestNumber);
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
+ params.add(GitEcaParameterNames.INSTALLATION_ID_RAW, installationId);
+ params.add(GitEcaParameterNames.REPOSITORY_FULL_NAME_RAW, repositoryFullName);
+ params.add(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW, Integer.toString(pullRequestNumber));
+ return dao.get(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class), params)).stream().findFirst();
+ }
+
+ /**
+ * Simple helper method so we don't have to repeat static strings in multiple places.
+ *
+ * @param fullRepoName the full repo name including org for the target PR.
+ * @return the full repo URL on Github for the request
+ */
+ public static String getRepoUrl(String org, String repoName) {
+ return "https://github.com/" + org + '/' + repoName;
+ }
+
+ /**
+ * Sends off a POST to update the commit status given a context for the current PR.
+ *
+ * @param request the current webhook update request
+ * @param state the state to set the status to
+ * @param fingerprint the internal unique string for the set of commits being processed
+ */
+ private void updateCommitStatus(GithubWebhookRequest request, GithubCommitStatuses state) {
+ LOGGER
+ .trace("Generated access token for installation {}: {}", request.getInstallation().getId(),
+ jwtHelper.getGithubAccessToken(request.getInstallation().getId()).getToken());
+ ghApi
+ .updateStatus(jwtHelper.getGhBearerString(request.getInstallation().getId()), apiVersion,
+ request.getRepository().getFullName(), request.getPullRequest().getHead().getSha(),
+ GithubCommitStatusRequest
+ .builder()
+ .setDescription(state.getMessage())
+ .setState(state.toString())
+ .setTargetUrl(webhooksConfig.github().serverTarget() + "/git/eca/status/gh/"
+ + request.getRepository().getFullName() + '/' + request.getPullRequest().getNumber())
+ .setContext(webhooksConfig.github().context())
+ .build());
+ }
+
+ /**
+ * Wraps a nullable value fetch to handle errors and will return null if the value can't be retrieved.
+ *
+ * @param supplier the method with potentially nullable values
+ * @return the value if it can be found, or null
+ */
+ private String getNullableString(Supplier<String> supplier) {
+ try {
+ return supplier.get();
+ } catch (NullPointerException e) {
+ // suppress, as we don't care at this point if its null
+ }
+ return null;
+ }
+
+ /**
+ * Validates required fields for processing requests.
+ *
+ * @param installationId the installation ID for the ECA app in the given repository
+ * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
+ * @param pullRequestNumber the pull request number that is being processed currently
+ * @throws BadRequestException if at least one of the parameters is in an invalid state.
+ */
+ private void checkRequestParameters(String installationId, String repositoryFullName, int pullRequestNumber) {
+ List<String> missingFields = new ArrayList<>();
+ if (StringUtils.isBlank(installationId)) {
+ missingFields.add(GitEcaParameterNames.INSTALLATION_ID_RAW);
+ }
+ if (StringUtils.isBlank(repositoryFullName)) {
+ missingFields.add(GitEcaParameterNames.REPOSITORY_FULL_NAME_RAW);
+ }
+ if (pullRequestNumber < 1) {
+ missingFields.add(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW);
+ }
+
+ // throw exception if some fields are missing as we can't continue to process the request
+ if (!missingFields.isEmpty()) {
+ throw new BadRequestException("Missing fields in order to prepare request: " + StringUtils.join(missingFields, ' '));
+ }
+ }
+
+}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java b/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java
index a1649923..8577baaa 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java
@@ -39,6 +39,7 @@ public final class GitEcaParameterNames implements UrlParameterNamespace {
public static final String INSTALLATION_ID_RAW = "installation_id";
public static final String PULL_REQUEST_NUMBER_RAW = "pull_request_number";
public static final String USER_MAIL_RAW = "user_mail";
+ public static final String NEEDS_REVALIDATION_RAW = "user_mail";
public static final UrlParameter COMMIT_ID = new UrlParameter(COMMIT_ID_RAW);
public static final UrlParameter SHA = new UrlParameter(SHA_RAW);
public static final UrlParameter SHAS = new UrlParameter(SHAS_RAW);
@@ -58,13 +59,14 @@ public final class GitEcaParameterNames implements UrlParameterNamespace {
public static final UrlParameter INSTALLATION_ID = new UrlParameter(INSTALLATION_ID_RAW);
public static final UrlParameter PULL_REQUEST_NUMBER = new UrlParameter(PULL_REQUEST_NUMBER_RAW);
public static final UrlParameter USER_MAIL = new UrlParameter(USER_MAIL_RAW);
+ public static final UrlParameter NEEDS_REVALIDATION = new UrlParameter(NEEDS_REVALIDATION_RAW);
@Override
public List<UrlParameter> getParameters() {
return Arrays
.asList(COMMIT_ID, SHA, SHAS, PROJECT_ID, PROJECT_IDS, NOT_IN_PROJECT_IDS, REPO_URL, FINGERPRINT, USER_ID, PROJECT_PATH,
PARENT_PROJECT, STATUS_ACTIVE, STATUS_DELETED, SINCE, UNTIL, REPOSITORY_FULL_NAME, INSTALLATION_ID,
- PULL_REQUEST_NUMBER, USER_MAIL);
+ PULL_REQUEST_NUMBER, USER_MAIL, NEEDS_REVALIDATION);
}
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/resource/GithubAdjacentResource.java b/src/main/java/org/eclipsefoundation/git/eca/resource/GithubAdjacentResource.java
index a7fc852e..6fd5d3a1 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/resource/GithubAdjacentResource.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/resource/GithubAdjacentResource.java
@@ -11,48 +11,16 @@
*/
package org.eclipsefoundation.git.eca.resource;
-import java.net.URI;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
import javax.inject.Inject;
-import javax.ws.rs.BadRequestException;
-import javax.ws.rs.core.MultivaluedMap;
-import org.apache.commons.lang3.StringUtils;
-import org.eclipse.microprofile.config.inject.ConfigProperty;
-import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.core.helper.DateTimeHelper;
import org.eclipsefoundation.core.model.RequestWrapper;
-import org.eclipsefoundation.core.service.APIMiddleware;
-import org.eclipsefoundation.git.eca.api.GithubAPI;
-import org.eclipsefoundation.git.eca.api.models.GithubCommit;
-import org.eclipsefoundation.git.eca.api.models.GithubCommit.ParentCommit;
-import org.eclipsefoundation.git.eca.api.models.GithubCommitStatusRequest;
-import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
-import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
-import org.eclipsefoundation.git.eca.config.WebhooksConfig;
import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
-import org.eclipsefoundation.git.eca.helper.JwtHelper;
-import org.eclipsefoundation.git.eca.model.Commit;
-import org.eclipsefoundation.git.eca.model.GitUser;
-import org.eclipsefoundation.git.eca.model.ValidationRequest;
-import org.eclipsefoundation.git.eca.model.ValidationResponse;
-import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
-import org.eclipsefoundation.git.eca.namespace.GithubCommitStatuses;
-import org.eclipsefoundation.git.eca.namespace.ProviderType;
-import org.eclipsefoundation.git.eca.service.GithubApplicationService;
-import org.eclipsefoundation.git.eca.service.ValidationService;
import org.eclipsefoundation.persistence.dao.PersistenceDao;
import org.eclipsefoundation.persistence.model.RDBMSQuery;
import org.eclipsefoundation.persistence.service.FilterService;
-import org.jboss.resteasy.specimpl.MultivaluedMapImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Contains operations and properties that are common to resources that interact with Github validation.
@@ -61,258 +29,18 @@ import org.slf4j.LoggerFactory;
*
*/
public abstract class GithubAdjacentResource {
- private static final Logger LOGGER = LoggerFactory.getLogger(GithubAdjacentResource.class);
-
- private static final String VALIDATION_LOGGING_MESSAGE = "Setting validation state for {}/#{} to {}";
-
- @ConfigProperty(name = "eclipse.github.default-api-version", defaultValue = "2022-11-28")
- String apiVersion;
-
- @Inject
- WebhooksConfig webhooksConfig;
- @Inject
- JwtHelper jwtHelper;
- @Inject
- APIMiddleware middleware;
@Inject
PersistenceDao dao;
@Inject
FilterService filters;
- @Inject
- ValidationService validation;
- @Inject
- GithubApplicationService ghAppService;
-
@Inject
RequestWrapper wrapper;
- @RestClient
- GithubAPI ghApi;
-
- /**
- * Generate a ValidationRequest object based on data pulled from Github, grabbing commits from the noted pull request
- * using the installation ID for access/authorization.
- *
- * @param installationId the ECA app installation ID for the organization
- * @param repositoryFullName the full name of the repository where the PR resides
- * @param pullRequestNumber the pull request number that is being validated
- * @param repositoryUrl the URL of the repository that contains the commits to validate
- * @return the populated validation request for the Github request information
- */
- @SuppressWarnings("null")
- ValidationRequest generateRequest(String installationId, String repositoryFullName, int pullRequestNumber, String repositoryUrl) {
- checkRequestParameters(installationId, repositoryFullName, pullRequestNumber);
- // get the commits that will be validated, don't cache as changes can come in too fast for it to be useful
- List<GithubCommit> commits = middleware
- .getAll(i -> ghApi
- .getCommits(jwtHelper.getGhBearerString(installationId), apiVersion, repositoryFullName, pullRequestNumber),
- GithubCommit.class);
- LOGGER.trace("Retrieved {} commits for PR {} in repo {}", commits.size(), pullRequestNumber, repositoryUrl);
- // set up the validation request from current data
- return ValidationRequest
- .builder()
- .setProvider(ProviderType.GITHUB)
- .setRepoUrl(URI.create(repositoryUrl))
- .setStrictMode(true)
- .setCommits(commits
- .stream()
- .map(c -> Commit
- .builder()
- .setHash(c.getSha())
- .setAuthor(GitUser
- .builder()
- .setMail(getNullableString(() -> c.getCommit().getAuthor().getEmail()))
- .setName(getNullableString(() -> c.getCommit().getAuthor().getName()))
- .setExternalId(getNullableString(() -> c.getAuthor().getLogin()))
- .build())
- .setCommitter(GitUser
- .builder()
- .setMail(getNullableString(() -> c.getCommit().getCommitter().getEmail()))
- .setName(getNullableString(() -> c.getCommit().getCommitter().getName()))
- .setExternalId(getNullableString(() -> c.getCommitter().getLogin()))
- .build())
- .setParents(c.getParents().stream().map(ParentCommit::getSha).collect(Collectors.toList()))
- .build())
- .collect(Collectors.toList()))
- .build();
- }
-
- /**
- * Shortcut method that will retrieve existing GH tracking info, create new entries if missing, and will update the
- * state of existing requests as well.
- *
- * @param installationId the installation ID for the ECA app in the given repository
- * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
- * @param pr the pull request targeted by the validation request.
- * @return a new or updated tracking object, or null if there was an error in saving the information
- */
- GithubWebhookTracking retrieveAndUpdateTrackingInformation(String installationId, String repositoryFullName, PullRequest pr) {
- return updateGithubTrackingIfMissing(getExistingRequestInformation(installationId, repositoryFullName, pr.getNumber()), pr,
- installationId, repositoryFullName);
- }
-
- /**
- * Attempts to retrieve a webhook tracking record given the installation, repository, and pull request number.
- *
- * @param installationId the installation ID for the ECA app in the given repository
- * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
- * @param pullRequestNumber the pull request number that is being processed currently
- * @return the webhook tracking record if it can be found, or an empty optional.
- */
- Optional<GithubWebhookTracking> getExistingRequestInformation(String installationId, String repositoryFullName, int pullRequestNumber) {
- checkRequestParameters(installationId, repositoryFullName, pullRequestNumber);
- MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
- params.add(GitEcaParameterNames.INSTALLATION_ID_RAW, installationId);
- params.add(GitEcaParameterNames.REPOSITORY_FULL_NAME_RAW, repositoryFullName);
- params.add(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW, Integer.toString(pullRequestNumber));
- return dao.get(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class), params)).stream().findFirst();
+ void setRevalidationFlagForTracking(GithubWebhookTracking tracking) {
+ tracking.setNeedsRevalidation(true);
+ tracking.setLastUpdated(DateTimeHelper.now());
+ dao.add(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class)), Arrays.asList(tracking));
}
-
- /**
- * Checks if the Github tracking is present for the current request, and if missing will generate a new record and save
- * it.
- *
- * @param tracking the optional tracking entry for the current request
- * @param request the pull request that is being validated
- * @param installationId the ECA app installation ID for the current request
- * @param fullRepoName the full repo name for the validation request
- */
- GithubWebhookTracking updateGithubTrackingIfMissing(Optional<GithubWebhookTracking> tracking, PullRequest request,
- String installationId, String fullRepoName) {
- // if there is no tracking present, create the missing tracking and persist it
- GithubWebhookTracking updatedTracking;
- if (tracking.isEmpty()) {
- updatedTracking = new GithubWebhookTracking();
- updatedTracking.setHeadSha(request.getHead().getSha());
- updatedTracking.setInstallationId(installationId);
- updatedTracking.setLastUpdated(DateTimeHelper.now());
- updatedTracking.setPullRequestNumber(request.getNumber());
- updatedTracking.setRepositoryFullName(fullRepoName);
- updatedTracking.setLastKnownState(request.getState());
- if (!"open".equalsIgnoreCase(request.getState())) {
- LOGGER
- .warn("The PR {} in {} is not in an open state, and will not be validated to follow our validation practice",
- updatedTracking.getPullRequestNumber(), fullRepoName);
- }
- } else {
- // at least update the state on every run
- updatedTracking = tracking.get();
- updatedTracking.setLastKnownState(request.getState());
- }
-
- // save the data, and log on its success or failure
- List<GithubWebhookTracking> savedTracking = dao
- .add(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class)), Arrays.asList(updatedTracking));
- if (savedTracking.isEmpty()) {
- LOGGER.warn("Unable to create new GH tracking record for request to validate {}#{}", fullRepoName, request.getNumber());
- return null;
- }
- // return the updated tracking when successful
- LOGGER.debug("Created new GH tracking record for request to validate {}#{}", fullRepoName, request.getNumber());
- return savedTracking.get(0);
- }
-
- /**
- * Process the current request and update the checks state to pending then success or failure. Contains verbose TRACE
- * logging for more info on the states of the validation for more information
- *
- * @param request information about the request from the GH webhook on what resource requested revalidation. Used to
- * target the commit status of the resources
- * @param vr the pseudo request generated from the contextual webhook data. Used to make use of existing validation
- * logic.
- * @return true if the validation passed, false otherwise.
- */
- boolean handleGithubWebhookValidation(GithubWebhookRequest request, ValidationRequest vr) {
- // update the status before processing
- LOGGER
- .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
- GithubCommitStatuses.PENDING);
- updateCommitStatus(request, GithubCommitStatuses.PENDING);
-
- // validate the response
- LOGGER
- .trace("Begining validation of request for {}/#{}", request.getRepository().getFullName(),
- request.getPullRequest().getNumber());
- ValidationResponse r = validation.validateIncomingRequest(vr, wrapper);
- if (r.getPassed()) {
- LOGGER
- .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
- GithubCommitStatuses.SUCCESS);
- updateCommitStatus(request, GithubCommitStatuses.SUCCESS);
- return true;
- }
- LOGGER
- .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
- GithubCommitStatuses.FAILURE);
- updateCommitStatus(request, GithubCommitStatuses.FAILURE);
- return false;
- }
-
- /**
- * Sends off a POST to update the commit status given a context for the current PR.
- *
- * @param request the current webhook update request
- * @param state the state to set the status to
- * @param fingerprint the internal unique string for the set of commits being processed
- */
- private void updateCommitStatus(GithubWebhookRequest request, GithubCommitStatuses state) {
- LOGGER
- .trace("Generated access token for installation {}: {}", request.getInstallation().getId(),
- jwtHelper.getGithubAccessToken(request.getInstallation().getId()).getToken());
- ghApi
- .updateStatus(jwtHelper.getGhBearerString(request.getInstallation().getId()), apiVersion,
- request.getRepository().getFullName(), request.getPullRequest().getHead().getSha(),
- GithubCommitStatusRequest
- .builder()
- .setDescription(state.getMessage())
- .setState(state.toString())
- .setTargetUrl(webhooksConfig.github().serverTarget() + "/git/eca/status/gh/" + request.getRepository().getFullName() + '/'
- + request.getPullRequest().getNumber())
- .setContext(webhooksConfig.github().context())
- .build());
- }
-
- /**
- * Wraps a nullable value fetch to handle errors and will return null if the value can't be retrieved.
- *
- * @param supplier the method with potentially nullable values
- * @return the value if it can be found, or null
- */
- private String getNullableString(Supplier<String> supplier) {
- try {
- return supplier.get();
- } catch (NullPointerException e) {
- // suppress, as we don't care at this point if its null
- }
- return null;
- }
-
- /**
- * Validates required fields for processing requests.
- *
- * @param installationId the installation ID for the ECA app in the given repository
- * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
- * @param pullRequestNumber the pull request number that is being processed currently
- * @throws BadRequestException if at least one of the parameters is in an invalid state.
- */
- private void checkRequestParameters(String installationId, String repositoryFullName, int pullRequestNumber) {
- List<String> missingFields = new ArrayList<>();
- if (StringUtils.isBlank(installationId)) {
- missingFields.add(GitEcaParameterNames.INSTALLATION_ID_RAW);
- }
- if (StringUtils.isBlank(repositoryFullName)) {
- missingFields.add(GitEcaParameterNames.REPOSITORY_FULL_NAME_RAW);
- }
- if (pullRequestNumber < 1) {
- missingFields.add(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW);
- }
-
- // throw exception if some fields are missing as we can't continue to process the request
- if (!missingFields.isEmpty()) {
- throw new BadRequestException("Missing fields in order to prepare request: " + StringUtils.join(missingFields, ' '));
- }
- }
-
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java b/src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java
index f02a4015..86125bbb 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java
@@ -25,15 +25,18 @@ import javax.ws.rs.QueryParam;
import javax.ws.rs.ServerErrorException;
import javax.ws.rs.core.Response;
+import org.apache.http.HttpStatus;
import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
import org.eclipsefoundation.git.eca.helper.CaptchaHelper;
+import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
import org.eclipsefoundation.git.eca.model.RevalidationResponse;
import org.eclipsefoundation.git.eca.model.ValidationRequest;
import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
import org.eclipsefoundation.git.eca.namespace.HCaptchaErrorCodes;
import org.eclipsefoundation.git.eca.namespace.WebhookHeaders;
+import org.eclipsefoundation.git.eca.service.GithubApplicationService;
import org.jboss.resteasy.annotations.jaxrs.HeaderParam;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,6 +51,11 @@ import org.slf4j.LoggerFactory;
public class GithubWebhooksResource extends GithubAdjacentResource {
private static final Logger LOGGER = LoggerFactory.getLogger(GithubWebhooksResource.class);
+ @Inject
+ GithubApplicationService ghAppService;
+
+ @Inject
+ GithubValidationHelper validationHelper;
@Inject
CaptchaHelper captchaHelper;
@@ -70,14 +78,23 @@ public class GithubWebhooksResource extends GithubAdjacentResource {
if (!"pull_request".equalsIgnoreCase(eventType)) {
return Response.ok().build();
}
- LOGGER.trace("Processing PR event for install {} with delivery ID of {}", request.getInstallation().getId(), deliveryId);
- // prepare for validation process by pre-processing into standard format
- ValidationRequest vr = generateRequest(request);
+
// track the request before we start processing
- retrieveAndUpdateTrackingInformation(request.getInstallation().getId(), request.getRepository().getFullName(),
- request.getPullRequest());
- // process the request
- handleGithubWebhookValidation(request, vr);
+ GithubWebhookTracking tracking = validationHelper
+ .retrieveAndUpdateTrackingInformation(wrapper, request.getInstallation().getId(), request.getRepository().getFullName(),
+ request.getPullRequest());
+
+ // start processing the request
+ LOGGER.trace("Processing PR event for install {} with delivery ID of {}", request.getInstallation().getId(), deliveryId);
+ try {
+ // prepare for validation process by pre-processing into standard format
+ ValidationRequest vr = generateRequest(request);
+ // process the request
+ validationHelper.handleGithubWebhookValidation(request, vr, wrapper);
+ } catch (Exception e) {
+ // set the revalidation flag to reprocess if the initial attempt fails before exiting
+ setRevalidationFlagForTracking(tracking);
+ }
return Response.ok().build();
}
@@ -105,11 +122,12 @@ public class GithubWebhooksResource extends GithubAdjacentResource {
}
// get the tracking if it exists, create it if it doesn't, and fail out if there is an issue
- GithubWebhookTracking tracking = retrieveAndUpdateTrackingInformation(installationId, fullRepoName, prResponse.get());
+ GithubWebhookTracking tracking = validationHelper
+ .retrieveAndUpdateTrackingInformation(wrapper, installationId, fullRepoName, prResponse.get());
if (tracking == null) {
throw new ServerErrorException(
String.format("Cannot find a tracked pull request with for repo '%s', pull request number '%d'", fullRepoName, prNo),
- 500);
+ HttpStatus.SC_INTERNAL_SERVER_ERROR);
} else if (!"open".equalsIgnoreCase(tracking.getLastKnownState())) {
// we do not want to reprocess non-open pull requests
throw new BadRequestException("Cannot revalidate a non-open pull request");
@@ -125,19 +143,25 @@ public class GithubWebhooksResource extends GithubAdjacentResource {
throw new BadRequestException("hCaptcha challenge response failed for this request");
}
- // get the tracking class, convert back to a GH webhook request, and validate the request
- GithubWebhookRequest request = GithubWebhookRequest.buildFromTracking(tracking);
- boolean isSuccessful = handleGithubWebhookValidation(request, generateRequest(request));
- LOGGER.debug("Revalidation for request for '{}#{}' was {}successful.", fullRepoName, prNo, isSuccessful ? "" : " not");
-
- // build the url for pull request page
- StringBuilder sb = new StringBuilder();
- sb.append("https://github.com/");
- sb.append(tracking.getRepositoryFullName());
- sb.append("/pull/");
- sb.append(tracking.getPullRequestNumber());
- // respond with a URL to the new location in a standard request
- return Response.ok(RevalidationResponse.builder().setLocation(URI.create(sb.toString())).build()).build();
+ // wrap the processing in a try-catch to catch upstream errors and recast them as server errors
+ try {
+ // get the tracking class, convert back to a GH webhook request, and validate the request
+ GithubWebhookRequest request = GithubWebhookRequest.buildFromTracking(tracking);
+ boolean isSuccessful = validationHelper.handleGithubWebhookValidation(request, generateRequest(request), wrapper).getPassed();
+ LOGGER.debug("Revalidation for request for '{}#{}' was {}successful.", fullRepoName, prNo, isSuccessful ? "" : " not");
+
+ // build the url for pull request page
+ StringBuilder sb = new StringBuilder();
+ sb.append("https://github.com/");
+ sb.append(fullRepoName);
+ sb.append("/pull/");
+ sb.append(prNo);
+ // respond with a URL to the new location in a standard request
+ return Response.ok(RevalidationResponse.builder().setLocation(URI.create(sb.toString())).build()).build();
+ } catch (Exception e) {
+ // rewrap exception, as some of the Github stuff can fail w/o explanation
+ throw new ServerErrorException(HttpStatus.SC_INTERNAL_SERVER_ERROR, e);
+ }
}
/**
@@ -147,8 +171,9 @@ public class GithubWebhooksResource extends GithubAdjacentResource {
* @return the Validation Request body to be used in validating data
*/
private ValidationRequest generateRequest(GithubWebhookRequest request) {
- return generateRequest(request.getInstallation().getId(), request.getRepository().getFullName(),
- request.getPullRequest().getNumber(), request.getRepository().getHtmlUrl());
+ return validationHelper
+ .generateRequest(request.getInstallation().getId(), request.getRepository().getFullName(),
+ request.getPullRequest().getNumber(), request.getRepository().getHtmlUrl());
}
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java b/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
index 8fd1b44f..df588b92 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
@@ -12,31 +12,23 @@
package org.eclipsefoundation.git.eca.resource;
import java.util.List;
-import java.util.Optional;
import java.util.stream.Collectors;
import javax.inject.Inject;
-import javax.ws.rs.BadRequestException;
import javax.ws.rs.GET;
-import javax.ws.rs.NotFoundException;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
-import javax.ws.rs.ServerErrorException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import org.apache.commons.lang3.StringUtils;
import org.eclipsefoundation.efservices.api.models.Project;
-import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
-import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
import org.eclipsefoundation.git.eca.dto.CommitValidationStatus;
-import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
+import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
import org.eclipsefoundation.git.eca.helper.ProjectHelper;
-import org.eclipsefoundation.git.eca.model.Commit;
-import org.eclipsefoundation.git.eca.model.ValidationRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.eclipsefoundation.git.eca.model.ValidationResponse;
+import org.eclipsefoundation.git.eca.service.GithubApplicationService;
+import org.eclipsefoundation.git.eca.service.ValidationService;
import io.quarkus.qute.Location;
import io.quarkus.qute.Template;
@@ -49,8 +41,14 @@ import io.quarkus.qute.Template;
*/
@Path("eca/status")
public class StatusResource extends GithubAdjacentResource {
- private static final Logger LOGGER = LoggerFactory.getLogger(StatusResource.class);
+ @Inject
+ GithubApplicationService ghAppService;
+ @Inject
+ ValidationService validation;
+
+ @Inject
+ GithubValidationHelper validationHelper;
@Inject
ProjectHelper projects;
@@ -113,49 +111,12 @@ public class StatusResource extends GithubAdjacentResource {
@Path("gh/{org}/{repoName}/{prNo}")
public Response getCommitValidationForGithub(@PathParam("org") String org, @PathParam("repoName") String repoName,
@PathParam("prNo") Integer prNo) {
- String fullRepoName = org + '/' + repoName;
- // get the installation ID for the given repo if it exists, and if the PR noted exists
- String installationId = ghAppService.getInstallationForRepo(fullRepoName);
- Optional<PullRequest> prResponse = ghAppService.getPullRequest(installationId, fullRepoName, prNo);
- if (StringUtils.isBlank(installationId)) {
- throw new BadRequestException("Could not find an ECA app installation for repo name: " + fullRepoName);
- } else if (prResponse.isEmpty()) {
- throw new NotFoundException(String.format("Could not find PR '%d' in repo name '%s'", prNo, fullRepoName));
- }
-
- // prepare the request for consumption
- String repoUrl = "https://github.com/" + fullRepoName;
- ValidationRequest vr = generateRequest(installationId, fullRepoName, prNo, repoUrl);
-
- // build the commit sha list based on the prepared request
- List<String> commitShas = vr.getCommits().stream().map(Commit::getHash).collect(Collectors.toList());
- // there should always be commits for a PR, but in case, lets check
- if (commitShas.isEmpty()) {
- throw new BadRequestException(String.format("Could not find any commits for %s#%d", fullRepoName, prNo));
- }
- LOGGER.debug("Found {} commits for '{}#{}'", commitShas.size(), fullRepoName, prNo);
-
- // retrieve the webhook tracking info, or generate an entry to track this PR if it's missing.
- GithubWebhookTracking updatedTracking = retrieveAndUpdateTrackingInformation(installationId, fullRepoName, prResponse.get());
- if (updatedTracking == null) {
- throw new ServerErrorException("Error while attempting to revalidate request, try again later.", 500);
- }
-
- // get the commit status of commits to use for checking historic validation
- List<CommitValidationStatus> statuses = validation.getHistoricValidationStatusByShas(wrapper, commitShas);
- if (!"open".equalsIgnoreCase(prResponse.get().getState()) && statuses.isEmpty()) {
- throw new BadRequestException("Cannot find validation history for current non-open PR, cannot provide validation status");
- }
-
- // we only want to update/revalidate for open PRs, so don't do this check if the PR is merged/closed
- if ("open".equalsIgnoreCase(prResponse.get().getState()) && commitShas.size() != statuses.size()) {
- LOGGER.debug("Validation for {}#{} does not seem to be current, revalidating commits", fullRepoName, prNo);
- // using the updated tracking, perform the validation
- handleGithubWebhookValidation(GithubWebhookRequest.buildFromTracking(updatedTracking), vr);
- // call to retrieve the statuses once again since they will have changed at this point
- statuses = validation.getHistoricValidationStatusByShas(wrapper, commitShas);
- }
+ // run the validation for the current request
+ ValidationResponse r = validationHelper.validateIncomingRequest(wrapper, org, repoName, prNo);
+ // retrieve the status of the commits to display on the status page
+ List<CommitValidationStatus> statuses = validation
+ .getHistoricValidationStatusByShas(wrapper, r.getCommits().keySet().stream().collect(Collectors.toList()));
// get projects for use in status UI
List<Project> ps = projects.retrieveProjectsForRepoURL(statuses.get(0).getRepoUrl(), statuses.get(0).getProvider());
// render and return the status UI
@@ -164,10 +125,10 @@ public class StatusResource extends GithubAdjacentResource {
.entity(statusUiTemplate
.data("statuses", statuses)
.data("pullRequestNumber", prNo)
- .data("fullRepoName", fullRepoName)
+ .data("fullRepoName", org + '/' + repoName)
.data("project", ps.isEmpty() ? null : ps.get(0))
- .data("repoUrl", repoUrl)
- .data("installationId", installationId)
+ .data("repoUrl", GithubValidationHelper.getRepoUrl(org, repoName))
+ .data("installationId", ghAppService.getInstallationForRepo(org + '/' + repoName))
.render())
.build();
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java b/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java
new file mode 100644
index 00000000..5158e4aa
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java
@@ -0,0 +1,142 @@
+/**
+ * Copyright (c) 2023 Eclipse Foundation
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * Author: Martin Lowe <martin.lowe@eclipse-foundation.org>
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.git.eca.tasks;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.List;
+
+import javax.annotation.PostConstruct;
+import javax.enterprise.context.ApplicationScoped;
+import javax.enterprise.context.control.ActivateRequestContext;
+import javax.inject.Inject;
+import javax.ws.rs.core.MultivaluedMap;
+
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+import org.eclipsefoundation.core.helper.DateTimeHelper;
+import org.eclipsefoundation.core.model.FlatRequestWrapper;
+import org.eclipsefoundation.core.model.RequestWrapper;
+import org.eclipsefoundation.core.namespace.DefaultUrlParameterNames;
+import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
+import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
+import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
+import org.eclipsefoundation.persistence.dao.PersistenceDao;
+import org.eclipsefoundation.persistence.model.RDBMSQuery;
+import org.eclipsefoundation.persistence.namespace.PersistenceUrlParameterNames;
+import org.eclipsefoundation.persistence.service.FilterService;
+import org.jboss.resteasy.specimpl.MultivaluedMapImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.quarkus.scheduler.Scheduled;
+
+/**
+ * Scheduled regular task that will interact with the backend persistence to look for requests that are in a
+ * failed/unvalidated state after an error while processing that could not be recovered. These requests will be
+ * reprocessed using the same logic as the standard validation, updating the timestamp on completion and either setting
+ * the revalidation flag to false or incrementing the number of repeated revalidations needed for the request for
+ * tracking, depending on the succcess of the revalidation.
+ */
+@ApplicationScoped
+public class GithubRevalidationQueue {
+ private static final Logger LOGGER = LoggerFactory.getLogger(GithubRevalidationQueue.class);
+
+ @ConfigProperty(name = "eclipse.git-eca.tasks.gh-revalidation.enabled", defaultValue = "true")
+ boolean isEnabled;
+
+ @Inject
+ PersistenceDao dao;
+ @Inject
+ FilterService filters;
+
+ @Inject
+ GithubValidationHelper validationHelper;
+
+ @PostConstruct
+ void init() {
+ // indicate to log whether enabled to reduce log spam
+ LOGGER.info("Github revalidation queue task is{} enabled.", isEnabled ? "" : " not");
+ }
+
+ /**
+ * Every 5s, this method will attempt to load a Github webhook validation request that has the needs revalidation flag
+ * set to true. This will retrieve the oldest request in queue and will attempt to revalidate it.
+ */
+ @Scheduled(every = "5s")
+ @ActivateRequestContext
+ public void revalidate() {
+ // if not enabled, don't process any potentially OOD records
+ if (!isEnabled) {
+ return;
+ }
+
+ // set up params for looking up the top of the revalidation queue
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
+ params.add(GitEcaParameterNames.NEEDS_REVALIDATION_RAW, "true");
+ params.add(PersistenceUrlParameterNames.SORT.getName(), "lastUpdated");
+ params.add(DefaultUrlParameterNames.PAGESIZE.getName(), "1");
+
+ // build the request and query to lookup the longest standing request that needs revalidation
+ RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/eca/revalidation-queue"));
+ RDBMSQuery<GithubWebhookTracking> trackingQuery = new RDBMSQuery<>(wrap, filters.get(GithubWebhookTracking.class), params);
+
+ List<GithubWebhookTracking> oldestRevalidation = dao.get(trackingQuery);
+ if (oldestRevalidation.isEmpty()) {
+ LOGGER.debug("No queued revalidation requests found");
+ } else {
+ reprocessRequest(oldestRevalidation.get(0), wrap);
+ }
+ }
+
+ /**
+ * Reprocess the given record, attempting to run the ECA validation logic again. If it passes, the revalidation flag is
+ * set to false and the time code is updated. If the processing fails again, the failure count gets incremented and the
+ * updated time is set so that another entry can be updated, as to not block on potentially broken records.
+ *
+ * @param requestToRevalidate the webhook tracking request to attempt to revalidate
+ * @param wrap the current stubbed request wrapper used for queries.
+ */
+ private void reprocessRequest(GithubWebhookTracking requestToRevalidate, RequestWrapper wrap) {
+ LOGGER
+ .debug("Attempting revalidation of request w/ ID {}, in repo {}#{}", requestToRevalidate.getId(),
+ requestToRevalidate.getRepositoryFullName(), requestToRevalidate.getPullRequestNumber());
+
+ // wrap in try-catch to avoid errors from stopping the record updates
+ try {
+ // split the full repo name into the org and repo name
+ String[] repoFullNameParts = requestToRevalidate.getRepositoryFullName().split("/");
+ if (repoFullNameParts.length != 2) {
+ throw new IllegalStateException("Record with ID '" + Long.toString(requestToRevalidate.getId())
+ + "' is in an invalid state (repository full name is not valid)");
+ }
+
+ // run the validation and then check if it succeeded
+ validationHelper
+ .validateIncomingRequest(wrap, repoFullNameParts[0], repoFullNameParts[1], requestToRevalidate.getPullRequestNumber());
+ // if we have gotten here, then the validation has completed and can be removed from queue
+ requestToRevalidate.setNeedsRevalidation(false);
+ LOGGER.debug("Sucessfully revalidated request w/ ID {}", requestToRevalidate.getId());
+ } catch (RuntimeException e) {
+ // update the number of times this status has revalidated (tracking)
+ requestToRevalidate
+ .setManualRevalidationCount(requestToRevalidate.getManualRevalidationCount() == null ? 1
+ : requestToRevalidate.getManualRevalidationCount() + 1);
+ // log the message so we can see what happened
+ LOGGER.error("Error while revalidating request w/ ID {}", requestToRevalidate.getId(), e);
+ } finally {
+ // whether successful or failed, update the updated field
+ requestToRevalidate.setLastUpdated(DateTimeHelper.now());
+ // push the update with the potentially updated validation flag or error count
+ dao.add(new RDBMSQuery<>(wrap, filters.get(GithubWebhookTracking.class)), Arrays.asList(requestToRevalidate));
+ }
+ }
+}
diff --git a/src/test/java/org/eclipsefoundation/git/eca/resource/ReportsResourceTest.java b/src/test/java/org/eclipsefoundation/git/eca/resource/ReportsResourceTest.java
index 8def7dcf..06c6ee66 100644
--- a/src/test/java/org/eclipsefoundation/git/eca/resource/ReportsResourceTest.java
+++ b/src/test/java/org/eclipsefoundation/git/eca/resource/ReportsResourceTest.java
@@ -13,6 +13,7 @@ package org.eclipsefoundation.git.eca.resource;
import java.util.Optional;
+import org.apache.http.HttpStatus;
import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
import org.eclipsefoundation.git.eca.test.namespaces.SchemaNamespaceHelper;
import org.eclipsefoundation.testing.helpers.TestCaseHelper;
diff --git a/src/test/resources/database/default/V1.0.0__default.sql b/src/test/resources/database/default/V1.0.0__default.sql
index 100590d3..25e97153 100644
--- a/src/test/resources/database/default/V1.0.0__default.sql
+++ b/src/test/resources/database/default/V1.0.0__default.sql
@@ -56,5 +56,7 @@ CREATE TABLE GithubWebhookTracking (
repositoryFullName varchar(127) NOT NULL,
pullRequestNumber int NOT NULL,
lastUpdated datetime DEFAULT NULL,
+ needsRevalidation BOOLEAN DEFAULT FALSE,
+ manualRevalidationCount int DEFAULT 0,
PRIMARY KEY (id)
);
\ No newline at end of file
--
GitLab
From f2817301f1b9c5b884bd819856c4fa88baab2976 Mon Sep 17 00:00:00 2001
From: Martin Lowe <martin.lowe@eclipse-foundation.org>
Date: Wed, 16 Aug 2023 11:22:21 -0400
Subject: [PATCH 2/2] Fix revalidation param value, disable revalidation task
on test
Additionally has some config patches to update internal SQL to include
new fields. They will still need to be patched in manually for existing
DBs.
---
config/mariadb/initdb.d/init.sql | 2 ++
.../git/eca/namespace/GitEcaParameterNames.java | 2 +-
.../eclipsefoundation/git/eca/resource/StatusResource.java | 1 +
.../git/eca/tasks/GithubRevalidationQueue.java | 7 ++++---
src/test/resources/application.properties | 1 +
5 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/config/mariadb/initdb.d/init.sql b/config/mariadb/initdb.d/init.sql
index 45938283..002b10e7 100644
--- a/config/mariadb/initdb.d/init.sql
+++ b/config/mariadb/initdb.d/init.sql
@@ -49,5 +49,7 @@ CREATE TABLE GithubWebhookTracking (
repositoryFullName varchar(127) NOT NULL,
pullRequestNumber int NOT NULL,
lastUpdated datetime DEFAULT NULL,
+ needsRevalidation tinyint(1) DEFAULT 0,
+ manualRevalidationCount int DEFAULT 0,
PRIMARY KEY (id)
);
\ No newline at end of file
diff --git a/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java b/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java
index 8577baaa..cbfaa0ee 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java
@@ -39,7 +39,7 @@ public final class GitEcaParameterNames implements UrlParameterNamespace {
public static final String INSTALLATION_ID_RAW = "installation_id";
public static final String PULL_REQUEST_NUMBER_RAW = "pull_request_number";
public static final String USER_MAIL_RAW = "user_mail";
- public static final String NEEDS_REVALIDATION_RAW = "user_mail";
+ public static final String NEEDS_REVALIDATION_RAW = "needs_revalidation";
public static final UrlParameter COMMIT_ID = new UrlParameter(COMMIT_ID_RAW);
public static final UrlParameter SHA = new UrlParameter(SHA_RAW);
public static final UrlParameter SHAS = new UrlParameter(SHAS_RAW);
diff --git a/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java b/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
index df588b92..aeef6297 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
@@ -16,6 +16,7 @@ import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.ws.rs.GET;
+import javax.ws.rs.NotFoundException;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
diff --git a/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java b/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java
index 5158e4aa..cfaedf15 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java
@@ -18,6 +18,7 @@ import java.util.List;
import javax.annotation.PostConstruct;
import javax.enterprise.context.ApplicationScoped;
import javax.enterprise.context.control.ActivateRequestContext;
+import javax.enterprise.inject.Instance;
import javax.inject.Inject;
import javax.ws.rs.core.MultivaluedMap;
@@ -51,7 +52,7 @@ public class GithubRevalidationQueue {
private static final Logger LOGGER = LoggerFactory.getLogger(GithubRevalidationQueue.class);
@ConfigProperty(name = "eclipse.git-eca.tasks.gh-revalidation.enabled", defaultValue = "true")
- boolean isEnabled;
+ Instance<Boolean> isEnabled;
@Inject
PersistenceDao dao;
@@ -64,7 +65,7 @@ public class GithubRevalidationQueue {
@PostConstruct
void init() {
// indicate to log whether enabled to reduce log spam
- LOGGER.info("Github revalidation queue task is{} enabled.", isEnabled ? "" : " not");
+ LOGGER.info("Github revalidation queue task is{} enabled.", Boolean.TRUE.equals(isEnabled.get()) ? "" : " not");
}
/**
@@ -75,7 +76,7 @@ public class GithubRevalidationQueue {
@ActivateRequestContext
public void revalidate() {
// if not enabled, don't process any potentially OOD records
- if (!isEnabled) {
+ if (!Boolean.TRUE.equals(isEnabled.get())) {
return;
}
diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties
index ed2bc8fb..4b362d93 100644
--- a/src/test/resources/application.properties
+++ b/src/test/resources/application.properties
@@ -34,6 +34,7 @@ eclipse.git-eca.mail.allow-list=noreply@github.com
## Reports configs
eclipse.git-eca.reports.access-key=samplekey
+eclipse.git-eca.tasks.gh-revalidation.enabled=false
## Misc
eclipse.gitlab.access-token=token_val
--
GitLab