Skip to content
Snippets Groups Projects
Commit 789ba5e6 authored by Martin Lowe's avatar Martin Lowe :flag_ca:
Browse files

Merge branch 'malowe/main/115' into 'main' 

Iss #115 - Add a reprocessing queue for failed GH webhook calls

See merge request !151
parents fd433fec f2817301
No related branches found
No related tags found
1 merge request!151Iss #115 - Add a reprocessing queue for failed GH webhook calls
Pipeline #30717 passed
Stage: external
    Expand all Hide whitespace changes
    Inline Side-by-side
    Showing
    with 656 additions and 361 deletions
    config/mariadb/initdb.d/init.sql
    + 2
    − 0
    View file @ 789ba5e6
    ......@@ -49,5 +49,7 @@ CREATE TABLE GithubWebhookTracking (
    repositoryFullName varchar(127) NOT NULL,
    pullRequestNumber int NOT NULL,
    lastUpdated datetime DEFAULT NULL,
    needsRevalidation tinyint(1) DEFAULT 0,
    manualRevalidationCount int DEFAULT 0,
    PRIMARY KEY (id)
    );
    \ No newline at end of file
    src/main/java/org/eclipsefoundation/git/eca/dto/GithubWebhookTracking.java
    + 41
    − 3
    View file @ 789ba5e6
    ......@@ -30,6 +30,7 @@ import org.eclipsefoundation.persistence.dto.filter.DtoFilter;
    import org.eclipsefoundation.persistence.model.DtoTable;
    import org.eclipsefoundation.persistence.model.ParameterizedSQLStatement;
    import org.eclipsefoundation.persistence.model.ParameterizedSQLStatementBuilder;
    import org.eclipsefoundation.persistence.model.SortableField;
    /**
    * Github Webhook request tracking info. Tracking is required to trigger revalidation through the Github API from this
    ......@@ -51,7 +52,10 @@ public class GithubWebhookTracking extends BareNode {
    private String headSha;
    private Integer pullRequestNumber;
    private String lastKnownState;
    @SortableField
    private ZonedDateTime lastUpdated;
    private boolean needsRevalidation;
    private Integer manualRevalidationCount;
    @Override
    public Long getId() {
    ......@@ -149,12 +153,41 @@ public class GithubWebhookTracking extends BareNode {
    this.lastUpdated = lastUpdated;
    }
    /**
    * @return the needsRevalidation
    */
    public boolean isNeedsRevalidation() {
    return needsRevalidation;
    }
    /**
    * @param needsRevalidation the needsRevalidation to set
    */
    public void setNeedsRevalidation(boolean needsRevalidation) {
    this.needsRevalidation = needsRevalidation;
    }
    /**
    * @return the manualRevalidationCount
    */
    public Integer getManualRevalidationCount() {
    return manualRevalidationCount;
    }
    /**
    * @param manualRevalidationCount the manualRevalidationCount to set
    */
    public void setManualRevalidationCount(Integer manualRevalidationCount) {
    this.manualRevalidationCount = manualRevalidationCount;
    }
    @Override
    public int hashCode() {
    final int prime = 31;
    int result = super.hashCode();
    result = prime * result
    + Objects.hash(headSha, id, installationId, lastKnownState, lastUpdated, pullRequestNumber, repositoryFullName);
    result = prime * result + Objects
    .hash(headSha, id, installationId, lastKnownState, lastUpdated, manualRevalidationCount, needsRevalidation,
    pullRequestNumber, repositoryFullName);
    return result;
    }
    ......@@ -172,7 +205,8 @@ public class GithubWebhookTracking extends BareNode {
    GithubWebhookTracking other = (GithubWebhookTracking) obj;
    return Objects.equals(headSha, other.headSha) && Objects.equals(id, other.id)
    && Objects.equals(installationId, other.installationId) && Objects.equals(lastKnownState, other.lastKnownState)
    && Objects.equals(lastUpdated, other.lastUpdated) && Objects.equals(pullRequestNumber, other.pullRequestNumber)
    && Objects.equals(lastUpdated, other.lastUpdated) && Objects.equals(manualRevalidationCount, other.manualRevalidationCount)
    && needsRevalidation == other.needsRevalidation && Objects.equals(pullRequestNumber, other.pullRequestNumber)
    && Objects.equals(repositoryFullName, other.repositoryFullName);
    }
    ......@@ -204,6 +238,10 @@ public class GithubWebhookTracking extends BareNode {
    .addClause(new ParameterizedSQLStatement.Clause(TABLE.getAlias() + ".repositoryFullName = ?",
    new Object[] { repositoryFullName }));
    }
    String needsRevalidation = params.getFirst(GitEcaParameterNames.NEEDS_REVALIDATION_RAW);
    if (Boolean.TRUE.equals(Boolean.valueOf(needsRevalidation))) {
    statement.addClause(new ParameterizedSQLStatement.Clause(TABLE.getAlias() + ".needsRevalidation = TRUE", new Object[] {}));
    }
    return statement;
    }
    ......
    src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java
    + 3
    − 1
    View file @ 789ba5e6
    ......@@ -39,6 +39,7 @@ public final class GitEcaParameterNames implements UrlParameterNamespace {
    public static final String INSTALLATION_ID_RAW = "installation_id";
    public static final String PULL_REQUEST_NUMBER_RAW = "pull_request_number";
    public static final String USER_MAIL_RAW = "user_mail";
    public static final String NEEDS_REVALIDATION_RAW = "needs_revalidation";
    public static final UrlParameter COMMIT_ID = new UrlParameter(COMMIT_ID_RAW);
    public static final UrlParameter SHA = new UrlParameter(SHA_RAW);
    public static final UrlParameter SHAS = new UrlParameter(SHAS_RAW);
    ......@@ -58,13 +59,14 @@ public final class GitEcaParameterNames implements UrlParameterNamespace {
    public static final UrlParameter INSTALLATION_ID = new UrlParameter(INSTALLATION_ID_RAW);
    public static final UrlParameter PULL_REQUEST_NUMBER = new UrlParameter(PULL_REQUEST_NUMBER_RAW);
    public static final UrlParameter USER_MAIL = new UrlParameter(USER_MAIL_RAW);
    public static final UrlParameter NEEDS_REVALIDATION = new UrlParameter(NEEDS_REVALIDATION_RAW);
    @Override
    public List<UrlParameter> getParameters() {
    return Arrays
    .asList(COMMIT_ID, SHA, SHAS, PROJECT_ID, PROJECT_IDS, NOT_IN_PROJECT_IDS, REPO_URL, FINGERPRINT, USER_ID, PROJECT_PATH,
    PARENT_PROJECT, STATUS_ACTIVE, STATUS_DELETED, SINCE, UNTIL, REPOSITORY_FULL_NAME, INSTALLATION_ID,
    PULL_REQUEST_NUMBER, USER_MAIL);
    PULL_REQUEST_NUMBER, USER_MAIL, NEEDS_REVALIDATION);
    }
    }
    src/main/java/org/eclipsefoundation/git/eca/resource/GithubAdjacentResource.java
    + 4
    − 276
    View file @ 789ba5e6
    ......@@ -11,48 +11,16 @@
    */
    package org.eclipsefoundation.git.eca.resource;
    import java.net.URI;
    import java.util.ArrayList;
    import java.util.Arrays;
    import java.util.List;
    import java.util.Optional;
    import java.util.function.Supplier;
    import java.util.stream.Collectors;
    import javax.inject.Inject;
    import javax.ws.rs.BadRequestException;
    import javax.ws.rs.core.MultivaluedMap;
    import org.apache.commons.lang3.StringUtils;
    import org.eclipse.microprofile.config.inject.ConfigProperty;
    import org.eclipse.microprofile.rest.client.inject.RestClient;
    import org.eclipsefoundation.core.helper.DateTimeHelper;
    import org.eclipsefoundation.core.model.RequestWrapper;
    import org.eclipsefoundation.core.service.APIMiddleware;
    import org.eclipsefoundation.git.eca.api.GithubAPI;
    import org.eclipsefoundation.git.eca.api.models.GithubCommit;
    import org.eclipsefoundation.git.eca.api.models.GithubCommit.ParentCommit;
    import org.eclipsefoundation.git.eca.api.models.GithubCommitStatusRequest;
    import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
    import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
    import org.eclipsefoundation.git.eca.config.WebhooksConfig;
    import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
    import org.eclipsefoundation.git.eca.helper.JwtHelper;
    import org.eclipsefoundation.git.eca.model.Commit;
    import org.eclipsefoundation.git.eca.model.GitUser;
    import org.eclipsefoundation.git.eca.model.ValidationRequest;
    import org.eclipsefoundation.git.eca.model.ValidationResponse;
    import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
    import org.eclipsefoundation.git.eca.namespace.GithubCommitStatuses;
    import org.eclipsefoundation.git.eca.namespace.ProviderType;
    import org.eclipsefoundation.git.eca.service.GithubApplicationService;
    import org.eclipsefoundation.git.eca.service.ValidationService;
    import org.eclipsefoundation.persistence.dao.PersistenceDao;
    import org.eclipsefoundation.persistence.model.RDBMSQuery;
    import org.eclipsefoundation.persistence.service.FilterService;
    import org.jboss.resteasy.specimpl.MultivaluedMapImpl;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    /**
    * Contains operations and properties that are common to resources that interact with Github validation.
    ......@@ -61,258 +29,18 @@ import org.slf4j.LoggerFactory;
    *
    */
    public abstract class GithubAdjacentResource {
    private static final Logger LOGGER = LoggerFactory.getLogger(GithubAdjacentResource.class);
    private static final String VALIDATION_LOGGING_MESSAGE = "Setting validation state for {}/#{} to {}";
    @ConfigProperty(name = "eclipse.github.default-api-version", defaultValue = "2022-11-28")
    String apiVersion;
    @Inject
    WebhooksConfig webhooksConfig;
    @Inject
    JwtHelper jwtHelper;
    @Inject
    APIMiddleware middleware;
    @Inject
    PersistenceDao dao;
    @Inject
    FilterService filters;
    @Inject
    ValidationService validation;
    @Inject
    GithubApplicationService ghAppService;
    @Inject
    RequestWrapper wrapper;
    @RestClient
    GithubAPI ghApi;
    /**
    * Generate a ValidationRequest object based on data pulled from Github, grabbing commits from the noted pull request
    * using the installation ID for access/authorization.
    *
    * @param installationId the ECA app installation ID for the organization
    * @param repositoryFullName the full name of the repository where the PR resides
    * @param pullRequestNumber the pull request number that is being validated
    * @param repositoryUrl the URL of the repository that contains the commits to validate
    * @return the populated validation request for the Github request information
    */
    @SuppressWarnings("null")
    ValidationRequest generateRequest(String installationId, String repositoryFullName, int pullRequestNumber, String repositoryUrl) {
    checkRequestParameters(installationId, repositoryFullName, pullRequestNumber);
    // get the commits that will be validated, don't cache as changes can come in too fast for it to be useful
    List<GithubCommit> commits = middleware
    .getAll(i -> ghApi
    .getCommits(jwtHelper.getGhBearerString(installationId), apiVersion, repositoryFullName, pullRequestNumber),
    GithubCommit.class);
    LOGGER.trace("Retrieved {} commits for PR {} in repo {}", commits.size(), pullRequestNumber, repositoryUrl);
    // set up the validation request from current data
    return ValidationRequest
    .builder()
    .setProvider(ProviderType.GITHUB)
    .setRepoUrl(URI.create(repositoryUrl))
    .setStrictMode(true)
    .setCommits(commits
    .stream()
    .map(c -> Commit
    .builder()
    .setHash(c.getSha())
    .setAuthor(GitUser
    .builder()
    .setMail(getNullableString(() -> c.getCommit().getAuthor().getEmail()))
    .setName(getNullableString(() -> c.getCommit().getAuthor().getName()))
    .setExternalId(getNullableString(() -> c.getAuthor().getLogin()))
    .build())
    .setCommitter(GitUser
    .builder()
    .setMail(getNullableString(() -> c.getCommit().getCommitter().getEmail()))
    .setName(getNullableString(() -> c.getCommit().getCommitter().getName()))
    .setExternalId(getNullableString(() -> c.getCommitter().getLogin()))
    .build())
    .setParents(c.getParents().stream().map(ParentCommit::getSha).collect(Collectors.toList()))
    .build())
    .collect(Collectors.toList()))
    .build();
    }
    /**
    * Shortcut method that will retrieve existing GH tracking info, create new entries if missing, and will update the
    * state of existing requests as well.
    *
    * @param installationId the installation ID for the ECA app in the given repository
    * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
    * @param pr the pull request targeted by the validation request.
    * @return a new or updated tracking object, or null if there was an error in saving the information
    */
    GithubWebhookTracking retrieveAndUpdateTrackingInformation(String installationId, String repositoryFullName, PullRequest pr) {
    return updateGithubTrackingIfMissing(getExistingRequestInformation(installationId, repositoryFullName, pr.getNumber()), pr,
    installationId, repositoryFullName);
    }
    /**
    * Attempts to retrieve a webhook tracking record given the installation, repository, and pull request number.
    *
    * @param installationId the installation ID for the ECA app in the given repository
    * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
    * @param pullRequestNumber the pull request number that is being processed currently
    * @return the webhook tracking record if it can be found, or an empty optional.
    */
    Optional<GithubWebhookTracking> getExistingRequestInformation(String installationId, String repositoryFullName, int pullRequestNumber) {
    checkRequestParameters(installationId, repositoryFullName, pullRequestNumber);
    MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
    params.add(GitEcaParameterNames.INSTALLATION_ID_RAW, installationId);
    params.add(GitEcaParameterNames.REPOSITORY_FULL_NAME_RAW, repositoryFullName);
    params.add(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW, Integer.toString(pullRequestNumber));
    return dao.get(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class), params)).stream().findFirst();
    void setRevalidationFlagForTracking(GithubWebhookTracking tracking) {
    tracking.setNeedsRevalidation(true);
    tracking.setLastUpdated(DateTimeHelper.now());
    dao.add(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class)), Arrays.asList(tracking));
    }
    /**
    * Checks if the Github tracking is present for the current request, and if missing will generate a new record and save
    * it.
    *
    * @param tracking the optional tracking entry for the current request
    * @param request the pull request that is being validated
    * @param installationId the ECA app installation ID for the current request
    * @param fullRepoName the full repo name for the validation request
    */
    GithubWebhookTracking updateGithubTrackingIfMissing(Optional<GithubWebhookTracking> tracking, PullRequest request,
    String installationId, String fullRepoName) {
    // if there is no tracking present, create the missing tracking and persist it
    GithubWebhookTracking updatedTracking;
    if (tracking.isEmpty()) {
    updatedTracking = new GithubWebhookTracking();
    updatedTracking.setHeadSha(request.getHead().getSha());
    updatedTracking.setInstallationId(installationId);
    updatedTracking.setLastUpdated(DateTimeHelper.now());
    updatedTracking.setPullRequestNumber(request.getNumber());
    updatedTracking.setRepositoryFullName(fullRepoName);
    updatedTracking.setLastKnownState(request.getState());
    if (!"open".equalsIgnoreCase(request.getState())) {
    LOGGER
    .warn("The PR {} in {} is not in an open state, and will not be validated to follow our validation practice",
    updatedTracking.getPullRequestNumber(), fullRepoName);
    }
    } else {
    // at least update the state on every run
    updatedTracking = tracking.get();
    updatedTracking.setLastKnownState(request.getState());
    }
    // save the data, and log on its success or failure
    List<GithubWebhookTracking> savedTracking = dao
    .add(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class)), Arrays.asList(updatedTracking));
    if (savedTracking.isEmpty()) {
    LOGGER.warn("Unable to create new GH tracking record for request to validate {}#{}", fullRepoName, request.getNumber());
    return null;
    }
    // return the updated tracking when successful
    LOGGER.debug("Created new GH tracking record for request to validate {}#{}", fullRepoName, request.getNumber());
    return savedTracking.get(0);
    }
    /**
    * Process the current request and update the checks state to pending then success or failure. Contains verbose TRACE
    * logging for more info on the states of the validation for more information
    *
    * @param request information about the request from the GH webhook on what resource requested revalidation. Used to
    * target the commit status of the resources
    * @param vr the pseudo request generated from the contextual webhook data. Used to make use of existing validation
    * logic.
    * @return true if the validation passed, false otherwise.
    */
    boolean handleGithubWebhookValidation(GithubWebhookRequest request, ValidationRequest vr) {
    // update the status before processing
    LOGGER
    .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
    GithubCommitStatuses.PENDING);
    updateCommitStatus(request, GithubCommitStatuses.PENDING);
    // validate the response
    LOGGER
    .trace("Begining validation of request for {}/#{}", request.getRepository().getFullName(),
    request.getPullRequest().getNumber());
    ValidationResponse r = validation.validateIncomingRequest(vr, wrapper);
    if (r.getPassed()) {
    LOGGER
    .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
    GithubCommitStatuses.SUCCESS);
    updateCommitStatus(request, GithubCommitStatuses.SUCCESS);
    return true;
    }
    LOGGER
    .trace(VALIDATION_LOGGING_MESSAGE, request.getRepository().getFullName(), request.getPullRequest().getNumber(),
    GithubCommitStatuses.FAILURE);
    updateCommitStatus(request, GithubCommitStatuses.FAILURE);
    return false;
    }
    /**
    * Sends off a POST to update the commit status given a context for the current PR.
    *
    * @param request the current webhook update request
    * @param state the state to set the status to
    * @param fingerprint the internal unique string for the set of commits being processed
    */
    private void updateCommitStatus(GithubWebhookRequest request, GithubCommitStatuses state) {
    LOGGER
    .trace("Generated access token for installation {}: {}", request.getInstallation().getId(),
    jwtHelper.getGithubAccessToken(request.getInstallation().getId()).getToken());
    ghApi
    .updateStatus(jwtHelper.getGhBearerString(request.getInstallation().getId()), apiVersion,
    request.getRepository().getFullName(), request.getPullRequest().getHead().getSha(),
    GithubCommitStatusRequest
    .builder()
    .setDescription(state.getMessage())
    .setState(state.toString())
    .setTargetUrl(webhooksConfig.github().serverTarget() + "/git/eca/status/gh/" + request.getRepository().getFullName() + '/'
    + request.getPullRequest().getNumber())
    .setContext(webhooksConfig.github().context())
    .build());
    }
    /**
    * Wraps a nullable value fetch to handle errors and will return null if the value can't be retrieved.
    *
    * @param supplier the method with potentially nullable values
    * @return the value if it can be found, or null
    */
    private String getNullableString(Supplier<String> supplier) {
    try {
    return supplier.get();
    } catch (NullPointerException e) {
    // suppress, as we don't care at this point if its null
    }
    return null;
    }
    /**
    * Validates required fields for processing requests.
    *
    * @param installationId the installation ID for the ECA app in the given repository
    * @param repositoryFullName the full repository name for the target repo, e.g. eclipse/jetty
    * @param pullRequestNumber the pull request number that is being processed currently
    * @throws BadRequestException if at least one of the parameters is in an invalid state.
    */
    private void checkRequestParameters(String installationId, String repositoryFullName, int pullRequestNumber) {
    List<String> missingFields = new ArrayList<>();
    if (StringUtils.isBlank(installationId)) {
    missingFields.add(GitEcaParameterNames.INSTALLATION_ID_RAW);
    }
    if (StringUtils.isBlank(repositoryFullName)) {
    missingFields.add(GitEcaParameterNames.REPOSITORY_FULL_NAME_RAW);
    }
    if (pullRequestNumber < 1) {
    missingFields.add(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW);
    }
    // throw exception if some fields are missing as we can't continue to process the request
    if (!missingFields.isEmpty()) {
    throw new BadRequestException("Missing fields in order to prepare request: " + StringUtils.join(missingFields, ' '));
    }
    }
    }
    src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java
    + 49
    − 24
    View file @ 789ba5e6
    ......@@ -25,15 +25,18 @@ import javax.ws.rs.QueryParam;
    import javax.ws.rs.ServerErrorException;
    import javax.ws.rs.core.Response;
    import org.apache.http.HttpStatus;
    import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
    import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
    import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
    import org.eclipsefoundation.git.eca.helper.CaptchaHelper;
    import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
    import org.eclipsefoundation.git.eca.model.RevalidationResponse;
    import org.eclipsefoundation.git.eca.model.ValidationRequest;
    import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
    import org.eclipsefoundation.git.eca.namespace.HCaptchaErrorCodes;
    import org.eclipsefoundation.git.eca.namespace.WebhookHeaders;
    import org.eclipsefoundation.git.eca.service.GithubApplicationService;
    import org.jboss.resteasy.annotations.jaxrs.HeaderParam;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    ......@@ -48,6 +51,11 @@ import org.slf4j.LoggerFactory;
    public class GithubWebhooksResource extends GithubAdjacentResource {
    private static final Logger LOGGER = LoggerFactory.getLogger(GithubWebhooksResource.class);
    @Inject
    GithubApplicationService ghAppService;
    @Inject
    GithubValidationHelper validationHelper;
    @Inject
    CaptchaHelper captchaHelper;
    ......@@ -70,14 +78,23 @@ public class GithubWebhooksResource extends GithubAdjacentResource {
    if (!"pull_request".equalsIgnoreCase(eventType)) {
    return Response.ok().build();
    }
    LOGGER.trace("Processing PR event for install {} with delivery ID of {}", request.getInstallation().getId(), deliveryId);
    // prepare for validation process by pre-processing into standard format
    ValidationRequest vr = generateRequest(request);
    // track the request before we start processing
    retrieveAndUpdateTrackingInformation(request.getInstallation().getId(), request.getRepository().getFullName(),
    request.getPullRequest());
    // process the request
    handleGithubWebhookValidation(request, vr);
    GithubWebhookTracking tracking = validationHelper
    .retrieveAndUpdateTrackingInformation(wrapper, request.getInstallation().getId(), request.getRepository().getFullName(),
    request.getPullRequest());
    // start processing the request
    LOGGER.trace("Processing PR event for install {} with delivery ID of {}", request.getInstallation().getId(), deliveryId);
    try {
    // prepare for validation process by pre-processing into standard format
    ValidationRequest vr = generateRequest(request);
    // process the request
    validationHelper.handleGithubWebhookValidation(request, vr, wrapper);
    } catch (Exception e) {
    // set the revalidation flag to reprocess if the initial attempt fails before exiting
    setRevalidationFlagForTracking(tracking);
    }
    return Response.ok().build();
    }
    ......@@ -105,11 +122,12 @@ public class GithubWebhooksResource extends GithubAdjacentResource {
    }
    // get the tracking if it exists, create it if it doesn't, and fail out if there is an issue
    GithubWebhookTracking tracking = retrieveAndUpdateTrackingInformation(installationId, fullRepoName, prResponse.get());
    GithubWebhookTracking tracking = validationHelper
    .retrieveAndUpdateTrackingInformation(wrapper, installationId, fullRepoName, prResponse.get());
    if (tracking == null) {
    throw new ServerErrorException(
    String.format("Cannot find a tracked pull request with for repo '%s', pull request number '%d'", fullRepoName, prNo),
    500);
    HttpStatus.SC_INTERNAL_SERVER_ERROR);
    } else if (!"open".equalsIgnoreCase(tracking.getLastKnownState())) {
    // we do not want to reprocess non-open pull requests
    throw new BadRequestException("Cannot revalidate a non-open pull request");
    ......@@ -125,19 +143,25 @@ public class GithubWebhooksResource extends GithubAdjacentResource {
    throw new BadRequestException("hCaptcha challenge response failed for this request");
    }
    // get the tracking class, convert back to a GH webhook request, and validate the request
    GithubWebhookRequest request = GithubWebhookRequest.buildFromTracking(tracking);
    boolean isSuccessful = handleGithubWebhookValidation(request, generateRequest(request));
    LOGGER.debug("Revalidation for request for '{}#{}' was {}successful.", fullRepoName, prNo, isSuccessful ? "" : " not");
    // build the url for pull request page
    StringBuilder sb = new StringBuilder();
    sb.append("https://github.com/");
    sb.append(tracking.getRepositoryFullName());
    sb.append("/pull/");
    sb.append(tracking.getPullRequestNumber());
    // respond with a URL to the new location in a standard request
    return Response.ok(RevalidationResponse.builder().setLocation(URI.create(sb.toString())).build()).build();
    // wrap the processing in a try-catch to catch upstream errors and recast them as server errors
    try {
    // get the tracking class, convert back to a GH webhook request, and validate the request
    GithubWebhookRequest request = GithubWebhookRequest.buildFromTracking(tracking);
    boolean isSuccessful = validationHelper.handleGithubWebhookValidation(request, generateRequest(request), wrapper).getPassed();
    LOGGER.debug("Revalidation for request for '{}#{}' was {}successful.", fullRepoName, prNo, isSuccessful ? "" : " not");
    // build the url for pull request page
    StringBuilder sb = new StringBuilder();
    sb.append("https://github.com/");
    sb.append(fullRepoName);
    sb.append("/pull/");
    sb.append(prNo);
    // respond with a URL to the new location in a standard request
    return Response.ok(RevalidationResponse.builder().setLocation(URI.create(sb.toString())).build()).build();
    } catch (Exception e) {
    // rewrap exception, as some of the Github stuff can fail w/o explanation
    throw new ServerErrorException(HttpStatus.SC_INTERNAL_SERVER_ERROR, e);
    }
    }
    /**
    ......@@ -147,8 +171,9 @@ public class GithubWebhooksResource extends GithubAdjacentResource {
    * @return the Validation Request body to be used in validating data
    */
    private ValidationRequest generateRequest(GithubWebhookRequest request) {
    return generateRequest(request.getInstallation().getId(), request.getRepository().getFullName(),
    request.getPullRequest().getNumber(), request.getRepository().getHtmlUrl());
    return validationHelper
    .generateRequest(request.getInstallation().getId(), request.getRepository().getFullName(),
    request.getPullRequest().getNumber(), request.getRepository().getHtmlUrl());
    }
    }
    src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
    + 19
    − 57
    View file @ 789ba5e6
    ......@@ -12,31 +12,24 @@
    package org.eclipsefoundation.git.eca.resource;
    import java.util.List;
    import java.util.Optional;
    import java.util.stream.Collectors;
    import javax.inject.Inject;
    import javax.ws.rs.BadRequestException;
    import javax.ws.rs.GET;
    import javax.ws.rs.NotFoundException;
    import javax.ws.rs.Path;
    import javax.ws.rs.PathParam;
    import javax.ws.rs.Produces;
    import javax.ws.rs.ServerErrorException;
    import javax.ws.rs.core.MediaType;
    import javax.ws.rs.core.Response;
    import org.apache.commons.lang3.StringUtils;
    import org.eclipsefoundation.efservices.api.models.Project;
    import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
    import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
    import org.eclipsefoundation.git.eca.dto.CommitValidationStatus;
    import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
    import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
    import org.eclipsefoundation.git.eca.helper.ProjectHelper;
    import org.eclipsefoundation.git.eca.model.Commit;
    import org.eclipsefoundation.git.eca.model.ValidationRequest;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import org.eclipsefoundation.git.eca.model.ValidationResponse;
    import org.eclipsefoundation.git.eca.service.GithubApplicationService;
    import org.eclipsefoundation.git.eca.service.ValidationService;
    import io.quarkus.qute.Location;
    import io.quarkus.qute.Template;
    ......@@ -49,8 +42,14 @@ import io.quarkus.qute.Template;
    */
    @Path("eca/status")
    public class StatusResource extends GithubAdjacentResource {
    private static final Logger LOGGER = LoggerFactory.getLogger(StatusResource.class);
    @Inject
    GithubApplicationService ghAppService;
    @Inject
    ValidationService validation;
    @Inject
    GithubValidationHelper validationHelper;
    @Inject
    ProjectHelper projects;
    ......@@ -113,49 +112,12 @@ public class StatusResource extends GithubAdjacentResource {
    @Path("gh/{org}/{repoName}/{prNo}")
    public Response getCommitValidationForGithub(@PathParam("org") String org, @PathParam("repoName") String repoName,
    @PathParam("prNo") Integer prNo) {
    String fullRepoName = org + '/' + repoName;
    // get the installation ID for the given repo if it exists, and if the PR noted exists
    String installationId = ghAppService.getInstallationForRepo(fullRepoName);
    Optional<PullRequest> prResponse = ghAppService.getPullRequest(installationId, fullRepoName, prNo);
    if (StringUtils.isBlank(installationId)) {
    throw new BadRequestException("Could not find an ECA app installation for repo name: " + fullRepoName);
    } else if (prResponse.isEmpty()) {
    throw new NotFoundException(String.format("Could not find PR '%d' in repo name '%s'", prNo, fullRepoName));
    }
    // prepare the request for consumption
    String repoUrl = "https://github.com/" + fullRepoName;
    ValidationRequest vr = generateRequest(installationId, fullRepoName, prNo, repoUrl);
    // build the commit sha list based on the prepared request
    List<String> commitShas = vr.getCommits().stream().map(Commit::getHash).collect(Collectors.toList());
    // there should always be commits for a PR, but in case, lets check
    if (commitShas.isEmpty()) {
    throw new BadRequestException(String.format("Could not find any commits for %s#%d", fullRepoName, prNo));
    }
    LOGGER.debug("Found {} commits for '{}#{}'", commitShas.size(), fullRepoName, prNo);
    // retrieve the webhook tracking info, or generate an entry to track this PR if it's missing.
    GithubWebhookTracking updatedTracking = retrieveAndUpdateTrackingInformation(installationId, fullRepoName, prResponse.get());
    if (updatedTracking == null) {
    throw new ServerErrorException("Error while attempting to revalidate request, try again later.", 500);
    }
    // get the commit status of commits to use for checking historic validation
    List<CommitValidationStatus> statuses = validation.getHistoricValidationStatusByShas(wrapper, commitShas);
    if (!"open".equalsIgnoreCase(prResponse.get().getState()) && statuses.isEmpty()) {
    throw new BadRequestException("Cannot find validation history for current non-open PR, cannot provide validation status");
    }
    // we only want to update/revalidate for open PRs, so don't do this check if the PR is merged/closed
    if ("open".equalsIgnoreCase(prResponse.get().getState()) && commitShas.size() != statuses.size()) {
    LOGGER.debug("Validation for {}#{} does not seem to be current, revalidating commits", fullRepoName, prNo);
    // using the updated tracking, perform the validation
    handleGithubWebhookValidation(GithubWebhookRequest.buildFromTracking(updatedTracking), vr);
    // call to retrieve the statuses once again since they will have changed at this point
    statuses = validation.getHistoricValidationStatusByShas(wrapper, commitShas);
    }
    // run the validation for the current request
    ValidationResponse r = validationHelper.validateIncomingRequest(wrapper, org, repoName, prNo);
    // retrieve the status of the commits to display on the status page
    List<CommitValidationStatus> statuses = validation
    .getHistoricValidationStatusByShas(wrapper, r.getCommits().keySet().stream().collect(Collectors.toList()));
    // get projects for use in status UI
    List<Project> ps = projects.retrieveProjectsForRepoURL(statuses.get(0).getRepoUrl(), statuses.get(0).getProvider());
    // render and return the status UI
    ......@@ -164,10 +126,10 @@ public class StatusResource extends GithubAdjacentResource {
    .entity(statusUiTemplate
    .data("statuses", statuses)
    .data("pullRequestNumber", prNo)
    .data("fullRepoName", fullRepoName)
    .data("fullRepoName", org + '/' + repoName)
    .data("project", ps.isEmpty() ? null : ps.get(0))
    .data("repoUrl", repoUrl)
    .data("installationId", installationId)
    .data("repoUrl", GithubValidationHelper.getRepoUrl(org, repoName))
    .data("installationId", ghAppService.getInstallationForRepo(org + '/' + repoName))
    .render())
    .build();
    }
    ......
    src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java 0 → 100644
    + 143
    − 0
    View file @ 789ba5e6
    /**
    * Copyright (c) 2023 Eclipse Foundation
    *
    * This program and the accompanying materials are made
    * available under the terms of the Eclipse Public License 2.0
    * which is available at https://www.eclipse.org/legal/epl-2.0/
    *
    * Author: Martin Lowe <martin.lowe@eclipse-foundation.org>
    *
    * SPDX-License-Identifier: EPL-2.0
    */
    package org.eclipsefoundation.git.eca.tasks;
    import java.net.URI;
    import java.util.Arrays;
    import java.util.List;
    import javax.annotation.PostConstruct;
    import javax.enterprise.context.ApplicationScoped;
    import javax.enterprise.context.control.ActivateRequestContext;
    import javax.enterprise.inject.Instance;
    import javax.inject.Inject;
    import javax.ws.rs.core.MultivaluedMap;
    import org.eclipse.microprofile.config.inject.ConfigProperty;
    import org.eclipsefoundation.core.helper.DateTimeHelper;
    import org.eclipsefoundation.core.model.FlatRequestWrapper;
    import org.eclipsefoundation.core.model.RequestWrapper;
    import org.eclipsefoundation.core.namespace.DefaultUrlParameterNames;
    import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
    import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
    import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
    import org.eclipsefoundation.persistence.dao.PersistenceDao;
    import org.eclipsefoundation.persistence.model.RDBMSQuery;
    import org.eclipsefoundation.persistence.namespace.PersistenceUrlParameterNames;
    import org.eclipsefoundation.persistence.service.FilterService;
    import org.jboss.resteasy.specimpl.MultivaluedMapImpl;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import io.quarkus.scheduler.Scheduled;
    /**
    * Scheduled regular task that will interact with the backend persistence to look for requests that are in a
    * failed/unvalidated state after an error while processing that could not be recovered. These requests will be
    * reprocessed using the same logic as the standard validation, updating the timestamp on completion and either setting
    * the revalidation flag to false or incrementing the number of repeated revalidations needed for the request for
    * tracking, depending on the succcess of the revalidation.
    */
    @ApplicationScoped
    public class GithubRevalidationQueue {
    private static final Logger LOGGER = LoggerFactory.getLogger(GithubRevalidationQueue.class);
    @ConfigProperty(name = "eclipse.git-eca.tasks.gh-revalidation.enabled", defaultValue = "true")
    Instance<Boolean> isEnabled;
    @Inject
    PersistenceDao dao;
    @Inject
    FilterService filters;
    @Inject
    GithubValidationHelper validationHelper;
    @PostConstruct
    void init() {
    // indicate to log whether enabled to reduce log spam
    LOGGER.info("Github revalidation queue task is{} enabled.", Boolean.TRUE.equals(isEnabled.get()) ? "" : " not");
    }
    /**
    * Every 5s, this method will attempt to load a Github webhook validation request that has the needs revalidation flag
    * set to true. This will retrieve the oldest request in queue and will attempt to revalidate it.
    */
    @Scheduled(every = "5s")
    @ActivateRequestContext
    public void revalidate() {
    // if not enabled, don't process any potentially OOD records
    if (!Boolean.TRUE.equals(isEnabled.get())) {
    return;
    }
    // set up params for looking up the top of the revalidation queue
    MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
    params.add(GitEcaParameterNames.NEEDS_REVALIDATION_RAW, "true");
    params.add(PersistenceUrlParameterNames.SORT.getName(), "lastUpdated");
    params.add(DefaultUrlParameterNames.PAGESIZE.getName(), "1");
    // build the request and query to lookup the longest standing request that needs revalidation
    RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/eca/revalidation-queue"));
    RDBMSQuery<GithubWebhookTracking> trackingQuery = new RDBMSQuery<>(wrap, filters.get(GithubWebhookTracking.class), params);
    List<GithubWebhookTracking> oldestRevalidation = dao.get(trackingQuery);
    if (oldestRevalidation.isEmpty()) {
    LOGGER.debug("No queued revalidation requests found");
    } else {
    reprocessRequest(oldestRevalidation.get(0), wrap);
    }
    }
    /**
    * Reprocess the given record, attempting to run the ECA validation logic again. If it passes, the revalidation flag is
    * set to false and the time code is updated. If the processing fails again, the failure count gets incremented and the
    * updated time is set so that another entry can be updated, as to not block on potentially broken records.
    *
    * @param requestToRevalidate the webhook tracking request to attempt to revalidate
    * @param wrap the current stubbed request wrapper used for queries.
    */
    private void reprocessRequest(GithubWebhookTracking requestToRevalidate, RequestWrapper wrap) {
    LOGGER
    .debug("Attempting revalidation of request w/ ID {}, in repo {}#{}", requestToRevalidate.getId(),
    requestToRevalidate.getRepositoryFullName(), requestToRevalidate.getPullRequestNumber());
    // wrap in try-catch to avoid errors from stopping the record updates
    try {
    // split the full repo name into the org and repo name
    String[] repoFullNameParts = requestToRevalidate.getRepositoryFullName().split("/");
    if (repoFullNameParts.length != 2) {
    throw new IllegalStateException("Record with ID '" + Long.toString(requestToRevalidate.getId())
    + "' is in an invalid state (repository full name is not valid)");
    }
    // run the validation and then check if it succeeded
    validationHelper
    .validateIncomingRequest(wrap, repoFullNameParts[0], repoFullNameParts[1], requestToRevalidate.getPullRequestNumber());
    // if we have gotten here, then the validation has completed and can be removed from queue
    requestToRevalidate.setNeedsRevalidation(false);
    LOGGER.debug("Sucessfully revalidated request w/ ID {}", requestToRevalidate.getId());
    } catch (RuntimeException e) {
    // update the number of times this status has revalidated (tracking)
    requestToRevalidate
    .setManualRevalidationCount(requestToRevalidate.getManualRevalidationCount() == null ? 1
    : requestToRevalidate.getManualRevalidationCount() + 1);
    // log the message so we can see what happened
    LOGGER.error("Error while revalidating request w/ ID {}", requestToRevalidate.getId(), e);
    } finally {
    // whether successful or failed, update the updated field
    requestToRevalidate.setLastUpdated(DateTimeHelper.now());
    // push the update with the potentially updated validation flag or error count
    dao.add(new RDBMSQuery<>(wrap, filters.get(GithubWebhookTracking.class)), Arrays.asList(requestToRevalidate));
    }
    }
    }
    src/test/java/org/eclipsefoundation/git/eca/resource/ReportsResourceTest.java
    + 1
    − 0
    View file @ 789ba5e6
    ......@@ -13,6 +13,7 @@ package org.eclipsefoundation.git.eca.resource;
    import java.util.Optional;
    import org.apache.http.HttpStatus;
    import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
    import org.eclipsefoundation.git.eca.test.namespaces.SchemaNamespaceHelper;
    import org.eclipsefoundation.testing.helpers.TestCaseHelper;
    ......
    src/test/resources/application.properties
    + 1
    − 0
    View file @ 789ba5e6
    ......@@ -34,6 +34,7 @@ eclipse.git-eca.mail.allow-list=noreply@github.com
    ## Reports configs
    eclipse.git-eca.reports.access-key=samplekey
    eclipse.git-eca.tasks.gh-revalidation.enabled=false
    ## Misc
    eclipse.gitlab.access-token=token_val
    ......
    src/test/resources/database/default/V1.0.0__default.sql
    + 2
    − 0
    View file @ 789ba5e6
    ......@@ -56,5 +56,7 @@ CREATE TABLE GithubWebhookTracking (
    repositoryFullName varchar(127) NOT NULL,
    pullRequestNumber int NOT NULL,
    lastUpdated datetime DEFAULT NULL,
    needsRevalidation BOOLEAN DEFAULT FALSE,
    manualRevalidationCount int DEFAULT 0,
    PRIMARY KEY (id)
    );
    \ No newline at end of file
    0% or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment

    Copyright © Eclipse Foundation, Inc. All Rights Reserved.     Privacy Policy | Terms of Use | Copyright Agent