diff --git a/src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java b/src/main/java/org/eclipsefoundation/git/eca/helper/GithubHelper.java
similarity index 81%
rename from src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java
rename to src/main/java/org/eclipsefoundation/git/eca/helper/GithubHelper.java
index 054ebad6739c2b353c7ba933051b4fde7b5fd717..803567edcab00ce75083e3d28e1bf6c4ab47613b 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/helper/GithubHelper.java
@@ -12,8 +12,11 @@
package org.eclipsefoundation.git.eca.helper;
import java.net.URI;
+import java.time.ZonedDateTime;
+import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.function.Supplier;
@@ -23,6 +26,7 @@ import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.core.service.APIMiddleware;
import org.eclipsefoundation.git.eca.api.GithubAPI;
+import org.eclipsefoundation.git.eca.api.models.GithubApplicationInstallationData;
import org.eclipsefoundation.git.eca.api.models.GithubCommit;
import org.eclipsefoundation.git.eca.api.models.GithubCommit.ParentCommit;
import org.eclipsefoundation.git.eca.api.models.GithubCommitStatusRequest;
@@ -30,6 +34,7 @@ import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
import org.eclipsefoundation.git.eca.config.WebhooksConfig;
import org.eclipsefoundation.git.eca.dto.CommitValidationStatus;
+import org.eclipsefoundation.git.eca.dto.GithubApplicationInstallation;
import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
import org.eclipsefoundation.git.eca.model.Commit;
import org.eclipsefoundation.git.eca.model.GitUser;
@@ -41,6 +46,7 @@ import org.eclipsefoundation.git.eca.namespace.ProviderType;
import org.eclipsefoundation.git.eca.service.GithubApplicationService;
import org.eclipsefoundation.git.eca.service.ValidationService;
import org.eclipsefoundation.git.eca.service.ValidationStatusService;
+import org.eclipsefoundation.http.model.FlatRequestWrapper;
import org.eclipsefoundation.http.model.RequestWrapper;
import org.eclipsefoundation.persistence.dao.PersistenceDao;
import org.eclipsefoundation.persistence.model.RDBMSQuery;
@@ -64,8 +70,8 @@ import jakarta.ws.rs.core.Response;
* scheduled tasks for revalidation.
*/
@ApplicationScoped
-public class GithubValidationHelper {
- private static final Logger LOGGER = LoggerFactory.getLogger(GithubValidationHelper.class);
+public class GithubHelper {
+ private static final Logger LOGGER = LoggerFactory.getLogger(GithubHelper.class);
private static final String VALIDATION_LOGGING_MESSAGE = "Setting validation state for {}/#{} to {}";
@@ -83,6 +89,8 @@ public class GithubValidationHelper {
PersistenceDao dao;
@Inject
FilterService filters;
+ @Inject
+ JwtHelper jwt;
@Inject
ValidationService validation;
@@ -352,6 +360,49 @@ public class GithubValidationHelper {
return dao.get(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class), params)).stream().findFirst();
}
+ /**
+ * Using the configured GitHub application JWT and application ID, fetches all active installations and updates the DB cache containing
+ * the installation data. After all records are updated, the starting timestamp is used to delete stale records that were updated before
+ * the starting time.
+ *
+ * This does not use locking, but with the way that updates are done to look for long stale entries, multiple concurrent runs would not
+ * lead to a loss of data/integrity.
+ */
+ public void updateAppInstallationRecords() {
+ // get the installations for the currently configured app
+ List<GithubApplicationInstallationData> installations = middleware
+ .getAll(i -> ghApi.getInstallations(i, "Bearer " + jwt.generateJwt()));
+ // check that there are installations, none may indicate an issue, and better to fail pessimistically
+ if (installations.isEmpty()) {
+ LOGGER.warn("Did not find any installations for the currently configured Github application");
+ return;
+ }
+ // trace log the installations for more context
+ LOGGER.debug("Found {} installations to cache", installations.size());
+
+ // create a common timestamp that looks for entries stale for more than a day
+ Date startingTimestamp = new Date(ZonedDateTime.now().minus(1, ChronoUnit.DAYS).toInstant().toEpochMilli());
+ // from installations, build records and start the processing for each entry
+ List<GithubApplicationInstallation> installationRecords = installations.stream().map(this::processInstallation).toList();
+
+ // once records are prepared, persist them back to the database with updates where necessary as a batch
+ RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/webhooks/github/installations"));
+ List<GithubApplicationInstallation> repoRecords = dao
+ .add(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class)), installationRecords);
+ if (repoRecords.size() != installationRecords.size()) {
+ LOGGER.warn("Background update to installation records had a size mismatch, cleaning will be skipped for this run");
+ return;
+ }
+
+ // build query to do cleanup of stale records
+ MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+ params.add(GitEcaParameterNames.APPLICATION_ID_RAW, Integer.toString(webhooksConfig.github().appId()));
+ params.add(GitEcaParameterNames.LAST_UPDATED_BEFORE_RAW, DateTimeHelper.toRFC3339(startingTimestamp));
+
+ // run the delete call, removing stale entries
+ dao.delete(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class), params));
+ }
+
/**
* Simple helper method so we don't have to repeat static strings in multiple places.
*
@@ -435,7 +486,43 @@ public class GithubValidationHelper {
throw new BadRequestException("Missing fields in order to prepare request: " + StringUtils.join(missingFields, ' '));
}
}
-
+
+ /**
+ * Converts the raw installation data from Github into a short record to be persisted to database as a form of persistent caching.
+ * Checks database for existing record, and returns record with a touched date for existing entries.
+ *
+ * @param ghInstallation raw Github installation record for current application
+ * @return the new or updated installation record to be persisted to the database.
+ */
+ private GithubApplicationInstallation processInstallation(GithubApplicationInstallationData ghInstallation) {
+ RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/webhooks/github/installations"));
+ // build the lookup query for the current installation record
+ MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+ params.add(GitEcaParameterNames.APPLICATION_ID_RAW, Integer.toString(webhooksConfig.github().appId()));
+ params.add(GitEcaParameterNames.INSTALLATION_ID_RAW, Integer.toString(ghInstallation.getId()));
+
+ // lookup existing records in the database
+ List<GithubApplicationInstallation> existingRecords = dao
+ .get(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class), params));
+
+ // check for existing entry, creating if missing
+ GithubApplicationInstallation installation;
+ if (existingRecords == null || existingRecords.isEmpty()) {
+ installation = new GithubApplicationInstallation();
+ installation.setAppId(webhooksConfig.github().appId());
+ installation.setInstallationId(ghInstallation.getId());
+ } else {
+ installation = existingRecords.get(0);
+ }
+ // update the basic stats to handle renames, and update last updated time
+ // login is technically nullable, so this might be missing. This is best we can do, as we can't look up by id
+ installation
+ .setName(StringUtils.isNotBlank(ghInstallation.getAccount().getLogin()) ? ghInstallation.getAccount().getLogin()
+ : "UNKNOWN");
+ installation.setLastUpdated(new Date());
+ return installation;
+ }
+
/**
* Retrieves the full repo name for a given org and repo name, used for storage and legacy support.
*
diff --git a/src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java b/src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java
index 453188aa05016b455ac25acbcd623f9b8a4d4a73..a39308123c2a31ae3411474cb9eaf86a2dd452f0 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java
@@ -19,13 +19,14 @@ import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
import org.eclipsefoundation.git.eca.helper.CaptchaHelper;
-import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
+import org.eclipsefoundation.git.eca.helper.GithubHelper;
import org.eclipsefoundation.git.eca.model.RevalidationResponse;
import org.eclipsefoundation.git.eca.model.ValidationRequest;
import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
import org.eclipsefoundation.git.eca.namespace.HCaptchaErrorCodes;
import org.eclipsefoundation.git.eca.namespace.WebhookHeaders;
import org.eclipsefoundation.git.eca.service.GithubApplicationService;
+import org.eclipsefoundation.git.eca.tasks.GithubInstallationUpdateTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,9 +59,11 @@ public class GithubWebhooksResource extends CommonResource {
GithubApplicationService ghAppService;
@Inject
- GithubValidationHelper validationHelper;
+ GithubHelper validationHelper;
@Inject
CaptchaHelper captchaHelper;
+ @Inject
+ GithubInstallationUpdateTask updateTask;
/**
* Entry point for processing Github webhook requests. Accepts standard fields as described in the <a href=
@@ -104,6 +107,9 @@ public class GithubWebhooksResource extends CommonResource {
// set the revalidation flag to reprocess if the initial attempt fails before exiting
setRevalidationFlagForTracking(tracking);
}
+ } else if ("installation".equalsIgnoreCase(eventType)) {
+ // manually trigger the run early.
+ updateTask.revalidate();
}
return Response.ok().build();
@@ -133,7 +139,7 @@ public class GithubWebhooksResource extends CommonResource {
@QueryParam(GitEcaParameterNames.INSTALLATION_ID_RAW) String installationId,
@QueryParam(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW) Integer prNo,
@FormParam("h-captcha-response") String captchaResponse) {
- String sanitizedRepoName = GithubValidationHelper.getFullRepoName(orgName, repoName);
+ String sanitizedRepoName = GithubHelper.getFullRepoName(orgName, repoName);
// retrieve and check that the PR exists
Optional<PullRequest> prResponse = ghAppService.getPullRequest(installationId, orgName, repoName, prNo);
if (prResponse.isEmpty()) {
diff --git a/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java b/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
index 2625fa163f4ef8edcebddaaa313e7d2f67f72dba..da2958222bd8ca5567b1d71b252cfc5e68a3d8bc 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
@@ -17,7 +17,7 @@ import org.apache.commons.lang3.StringUtils;
import org.eclipsefoundation.efservices.api.models.Project;
import org.eclipsefoundation.git.eca.config.EclipseQuteTemplateExtensions;
import org.eclipsefoundation.git.eca.dto.CommitValidationStatus;
-import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
+import org.eclipsefoundation.git.eca.helper.GithubHelper;
import org.eclipsefoundation.git.eca.helper.ProjectHelper;
import org.eclipsefoundation.git.eca.model.Commit;
import org.eclipsefoundation.git.eca.model.ValidationRequest;
@@ -65,7 +65,7 @@ public class StatusResource extends CommonResource {
ValidationStatusService validationStatus;
@Inject
- GithubValidationHelper validationHelper;
+ GithubHelper validationHelper;
@Inject
ProjectHelper projects;
@@ -146,7 +146,7 @@ public class StatusResource extends CommonResource {
public Response getCommitValidationForGithub(@PathParam("org") String org, @PathParam("repoName") String repoName,
@PathParam("prNo") Integer prNo) {
// generate the URL used to retrieve valid projects
- String repoUrl = GithubValidationHelper.getRepoUrl(org, repoName);
+ String repoUrl = GithubHelper.getRepoUrl(org, repoName);
try {
// check that the passed repo has a valid installation
diff --git a/src/main/java/org/eclipsefoundation/git/eca/service/impl/DefaultGithubApplicationService.java b/src/main/java/org/eclipsefoundation/git/eca/service/impl/DefaultGithubApplicationService.java
index 02bbc0f38d58c4933d907eba327c2fcb9450e926..6720a561ba2526b6779bca408955cf762c40bd3a 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/service/impl/DefaultGithubApplicationService.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/service/impl/DefaultGithubApplicationService.java
@@ -28,7 +28,7 @@ import org.eclipsefoundation.git.eca.api.GithubAPI;
import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
import org.eclipsefoundation.git.eca.config.WebhooksConfig;
import org.eclipsefoundation.git.eca.dto.GithubApplicationInstallation;
-import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
+import org.eclipsefoundation.git.eca.helper.GithubHelper;
import org.eclipsefoundation.git.eca.helper.JwtHelper;
import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
import org.eclipsefoundation.git.eca.service.GithubApplicationService;
@@ -118,7 +118,7 @@ public class DefaultGithubApplicationService implements GithubApplicationService
@Override
public Optional<PullRequest> getPullRequest(String installationId, String org, String repoName, Integer pullRequest) {
- String fullRepoName = GithubValidationHelper.getFullRepoName(org, repoName);
+ String fullRepoName = GithubHelper.getFullRepoName(org, repoName);
// create param map to better handle multiple PRs from a single repo in short time span
MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
params.add(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW, Integer.toString(pullRequest));
diff --git a/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubInstallationUpdateTask.java b/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubInstallationUpdateTask.java
index 5b24ae85a00893e52c5458aaddbdd57bab0340b2..5215090c759d6a5399dc7fc4ed640a20ab61e195 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubInstallationUpdateTask.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubInstallationUpdateTask.java
@@ -11,26 +11,8 @@
*/
package org.eclipsefoundation.git.eca.tasks;
-import java.net.URI;
-import java.util.Date;
-import java.util.List;
-
-import org.apache.commons.lang3.StringUtils;
import org.eclipse.microprofile.config.inject.ConfigProperty;
-import org.eclipse.microprofile.rest.client.inject.RestClient;
-import org.eclipsefoundation.core.service.APIMiddleware;
-import org.eclipsefoundation.git.eca.api.GithubAPI;
-import org.eclipsefoundation.git.eca.api.models.GithubApplicationInstallationData;
-import org.eclipsefoundation.git.eca.config.WebhooksConfig;
-import org.eclipsefoundation.git.eca.dto.GithubApplicationInstallation;
-import org.eclipsefoundation.git.eca.helper.JwtHelper;
-import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
-import org.eclipsefoundation.http.model.FlatRequestWrapper;
-import org.eclipsefoundation.http.model.RequestWrapper;
-import org.eclipsefoundation.persistence.dao.PersistenceDao;
-import org.eclipsefoundation.persistence.model.RDBMSQuery;
-import org.eclipsefoundation.persistence.service.FilterService;
-import org.eclipsefoundation.utils.helper.DateTimeHelper;
+import org.eclipsefoundation.git.eca.helper.GithubHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,8 +22,6 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.context.control.ActivateRequestContext;
import jakarta.enterprise.inject.Instance;
import jakarta.inject.Inject;
-import jakarta.ws.rs.core.MultivaluedHashMap;
-import jakarta.ws.rs.core.MultivaluedMap;
/**
* Using the configured database, Github installation records are tracked for all installations available for the
@@ -54,20 +34,9 @@ public class GithubInstallationUpdateTask {
@ConfigProperty(name = "eclipse.git-eca.tasks.gh-installation.enabled", defaultValue = "true")
Instance<Boolean> isEnabled;
+
@Inject
- WebhooksConfig config;
-
- @RestClient
- GithubAPI gh;
-
- @Inject
- JwtHelper jwt;
- @Inject
- APIMiddleware middle;
- @Inject
- PersistenceDao dao;
- @Inject
- FilterService filters;
+ GithubHelper helper;
@PostConstruct
void init() {
@@ -86,78 +55,6 @@ public class GithubInstallationUpdateTask {
if (!Boolean.TRUE.equals(isEnabled.get())) {
return;
}
-
- // get the installations for the currently configured app
- List<GithubApplicationInstallationData> installations = middle
- .getAll(i -> gh.getInstallations(i, "Bearer " + jwt.generateJwt()));
- // check that there are installations
- if (installations.isEmpty()) {
- LOGGER.warn("Did not find any installations for the currently configured Github application");
- return;
- }
- // trace log the installations for more context
- LOGGER.debug("Found {} installations to cache", installations.size());
-
- // create a common timestamp for easier lookups of stale entries
- Date startingTimestamp = new Date();
- // from installations, build records and start the processing for each entry
- List<GithubApplicationInstallation> installationRecords = installations
- .stream()
- .map(this::processInstallation)
- .toList();
-
- // once records are prepared, persist them back to the database with updates where necessary as a batch
- RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/webhooks/github/installations"));
- List<GithubApplicationInstallation> repoRecords = dao
- .add(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class)), installationRecords);
- if (repoRecords.size() != installationRecords.size()) {
- LOGGER.warn("Background update to installation records had a size mismatch, cleaning will be skipped for this run");
- return;
- }
-
- // build query to do cleanup of stale records
- MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
- params.add(GitEcaParameterNames.APPLICATION_ID_RAW, Integer.toString(config.github().appId()));
- params.add(GitEcaParameterNames.LAST_UPDATED_BEFORE_RAW, DateTimeHelper.toRFC3339(startingTimestamp));
-
- // run the delete call, removing stale entries
- dao.delete(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class), params));
- }
-
- /**
- * Converts the raw installation data from Github into a short record to be persisted to database as a form of
- * persistent caching. Checks database for existing record, and returns record with a touched date for existing entries.
- *
- * @param ghInstallation raw Github installation record for current application
- * @return the new or updated installation record to be persisted to the database.
- */
- private GithubApplicationInstallation processInstallation(GithubApplicationInstallationData ghInstallation) {
- RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/webhooks/github/installations"));
- // build the lookup query for the current installation record
- MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
- params.add(GitEcaParameterNames.APPLICATION_ID_RAW, Integer.toString(config.github().appId()));
- params.add(GitEcaParameterNames.INSTALLATION_ID_RAW, Integer.toString(ghInstallation.getId()));
-
- // lookup existing records in the database
- List<GithubApplicationInstallation> existingRecords = dao
- .get(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class), params));
-
- // check for existing entry, creating if missing
- GithubApplicationInstallation installation;
- if (existingRecords == null || existingRecords.isEmpty()) {
- installation = new GithubApplicationInstallation();
- installation.setAppId(config.github().appId());
- installation.setInstallationId(ghInstallation.getId());
- } else {
- installation = existingRecords.get(0);
- }
- // update the basic stats to handle renames, and update last updated time
- // login is technically nullable, so this might be missing. This is best we can do, as we can't look up by id
- installation
- .setName(StringUtils.isNotBlank(ghInstallation.getAccount().getLogin()) ? ghInstallation.getAccount().getLogin()
- : "UNKNOWN");
- installation.setLastUpdated(new Date());
- return installation;
-
+ helper.updateAppInstallationRecords();
}
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java b/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java
index 35bddfdf45b0b842362ad2a0cc9bdd82fdc7b7d8..bdb9f0c621dd57aefa27e304b5f233401a19adb7 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java
@@ -17,7 +17,7 @@ import java.util.List;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
-import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
+import org.eclipsefoundation.git.eca.helper.GithubHelper;
import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
import org.eclipsefoundation.http.model.FlatRequestWrapper;
import org.eclipsefoundation.http.model.RequestWrapper;
@@ -62,7 +62,7 @@ public class GithubRevalidationQueue {
FilterService filters;
@Inject
- GithubValidationHelper validationHelper;
+ GithubHelper validationHelper;
@PostConstruct
void init() {