update: add GH webhook event check for "installation" type events

Enabled by default, installation type events get triggered when the
associated GH application is installed into an organization or
repository. By extracting the current scheduled task to the GitHub
helper, we can call this arbitrarily when there are new installations
as well as handle catch-up on a regular schedule.

Resolves #140

src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java → src/main/java/org/eclipsefoundation/git/eca/helper/GithubHelper.java

@@ -12,8 +12,11 @@
 package org.eclipsefoundation.git.eca.helper;
 
 import java.net.URI;
+import java.time.ZonedDateTime;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Date;
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Supplier;
@@ -23,6 +26,7 @@ import org.eclipse.microprofile.config.inject.ConfigProperty;
 import org.eclipse.microprofile.rest.client.inject.RestClient;
 import org.eclipsefoundation.core.service.APIMiddleware;
 import org.eclipsefoundation.git.eca.api.GithubAPI;
+import org.eclipsefoundation.git.eca.api.models.GithubApplicationInstallationData;
 import org.eclipsefoundation.git.eca.api.models.GithubCommit;
 import org.eclipsefoundation.git.eca.api.models.GithubCommit.ParentCommit;
 import org.eclipsefoundation.git.eca.api.models.GithubCommitStatusRequest;
@@ -30,6 +34,7 @@ import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
 import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
 import org.eclipsefoundation.git.eca.config.WebhooksConfig;
 import org.eclipsefoundation.git.eca.dto.CommitValidationStatus;
+import org.eclipsefoundation.git.eca.dto.GithubApplicationInstallation;
 import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
 import org.eclipsefoundation.git.eca.model.Commit;
 import org.eclipsefoundation.git.eca.model.GitUser;
@@ -41,6 +46,7 @@ import org.eclipsefoundation.git.eca.namespace.ProviderType;
 import org.eclipsefoundation.git.eca.service.GithubApplicationService;
 import org.eclipsefoundation.git.eca.service.ValidationService;
 import org.eclipsefoundation.git.eca.service.ValidationStatusService;
+import org.eclipsefoundation.http.model.FlatRequestWrapper;
 import org.eclipsefoundation.http.model.RequestWrapper;
 import org.eclipsefoundation.persistence.dao.PersistenceDao;
 import org.eclipsefoundation.persistence.model.RDBMSQuery;
@@ -64,8 +70,8 @@ import jakarta.ws.rs.core.Response;
  * scheduled tasks for revalidation.
  */
 @ApplicationScoped
-public class GithubValidationHelper {
-    private static final Logger LOGGER = LoggerFactory.getLogger(GithubValidationHelper.class);
+public class GithubHelper {
+    private static final Logger LOGGER = LoggerFactory.getLogger(GithubHelper.class);
 
     private static final String VALIDATION_LOGGING_MESSAGE = "Setting validation state for {}/#{} to {}";
 
@@ -83,6 +89,8 @@ public class GithubValidationHelper {
     PersistenceDao dao;
     @Inject
     FilterService filters;
+    @Inject
+    JwtHelper jwt;
 
     @Inject
     ValidationService validation;
@@ -352,6 +360,49 @@ public class GithubValidationHelper {
         return dao.get(new RDBMSQuery<>(wrapper, filters.get(GithubWebhookTracking.class), params)).stream().findFirst();
     }
 
+    /**
+     * Using the configured GitHub application JWT and application ID, fetches all active installations and updates the DB cache containing
+     * the installation data. After all records are updated, the starting timestamp is used to delete stale records that were updated before
+     * the starting time.
+     * 
+     * This does not use locking, but with the way that updates are done to look for long stale entries, multiple concurrent runs would not
+     * lead to a loss of data/integrity.
+     */
+    public void updateAppInstallationRecords() {
+        // get the installations for the currently configured app
+        List<GithubApplicationInstallationData> installations = middleware
+                .getAll(i -> ghApi.getInstallations(i, "Bearer " + jwt.generateJwt()));
+        // check that there are installations, none may indicate an issue, and better to fail pessimistically
+        if (installations.isEmpty()) {
+            LOGGER.warn("Did not find any installations for the currently configured Github application");
+            return;
+        }
+        // trace log the installations for more context
+        LOGGER.debug("Found {} installations to cache", installations.size());
+
+        // create a common timestamp that looks for entries stale for more than a day
+        Date startingTimestamp = new Date(ZonedDateTime.now().minus(1, ChronoUnit.DAYS).toInstant().toEpochMilli());
+        // from installations, build records and start the processing for each entry
+        List<GithubApplicationInstallation> installationRecords = installations.stream().map(this::processInstallation).toList();
+
+        // once records are prepared, persist them back to the database with updates where necessary as a batch
+        RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/webhooks/github/installations"));
+        List<GithubApplicationInstallation> repoRecords = dao
+                .add(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class)), installationRecords);
+        if (repoRecords.size() != installationRecords.size()) {
+            LOGGER.warn("Background update to installation records had a size mismatch, cleaning will be skipped for this run");
+            return;
+        }
+
+        // build query to do cleanup of stale records
+        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+        params.add(GitEcaParameterNames.APPLICATION_ID_RAW, Integer.toString(webhooksConfig.github().appId()));
+        params.add(GitEcaParameterNames.LAST_UPDATED_BEFORE_RAW, DateTimeHelper.toRFC3339(startingTimestamp));
+
+        // run the delete call, removing stale entries
+        dao.delete(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class), params));
+    }
+
     /**
      * Simple helper method so we don't have to repeat static strings in multiple places.
      * 
@@ -435,7 +486,43 @@ public class GithubValidationHelper {
             throw new BadRequestException("Missing fields in order to prepare request: " + StringUtils.join(missingFields, ' '));
         }
     }
-    
+
+    /**
+     * Converts the raw installation data from Github into a short record to be persisted to database as a form of persistent caching.
+     * Checks database for existing record, and returns record with a touched date for existing entries.
+     * 
+     * @param ghInstallation raw Github installation record for current application
+     * @return the new or updated installation record to be persisted to the database.
+     */
+    private GithubApplicationInstallation processInstallation(GithubApplicationInstallationData ghInstallation) {
+        RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/webhooks/github/installations"));
+        // build the lookup query for the current installation record
+        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
+        params.add(GitEcaParameterNames.APPLICATION_ID_RAW, Integer.toString(webhooksConfig.github().appId()));
+        params.add(GitEcaParameterNames.INSTALLATION_ID_RAW, Integer.toString(ghInstallation.getId()));
+
+        // lookup existing records in the database
+        List<GithubApplicationInstallation> existingRecords = dao
+                .get(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class), params));
+
+        // check for existing entry, creating if missing
+        GithubApplicationInstallation installation;
+        if (existingRecords == null || existingRecords.isEmpty()) {
+            installation = new GithubApplicationInstallation();
+            installation.setAppId(webhooksConfig.github().appId());
+            installation.setInstallationId(ghInstallation.getId());
+        } else {
+            installation = existingRecords.get(0);
+        }
+        // update the basic stats to handle renames, and update last updated time
+        // login is technically nullable, so this might be missing. This is best we can do, as we can't look up by id
+        installation
+                .setName(StringUtils.isNotBlank(ghInstallation.getAccount().getLogin()) ? ghInstallation.getAccount().getLogin()
+                        : "UNKNOWN");
+        installation.setLastUpdated(new Date());
+        return installation;
+    }
+
     /**
      * Retrieves the full repo name for a given org and repo name, used for storage and legacy support.
      *

src/main/java/org/eclipsefoundation/git/eca/resource/GithubWebhooksResource.java

@@ -19,7 +19,7 @@ import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest;
 import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
 import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
 import org.eclipsefoundation.git.eca.helper.CaptchaHelper;
-import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
+import org.eclipsefoundation.git.eca.helper.GithubHelper;
 import org.eclipsefoundation.git.eca.model.RevalidationResponse;
 import org.eclipsefoundation.git.eca.model.ValidationRequest;
 import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
@@ -58,9 +58,11 @@ public class GithubWebhooksResource extends CommonResource {
     GithubApplicationService ghAppService;
 
     @Inject
-    GithubValidationHelper validationHelper;
+    GithubHelper validationHelper;
     @Inject
     CaptchaHelper captchaHelper;
+    @Inject
+    GithubHelper ghHelper;
 
     /**
      * Entry point for processing Github webhook requests. Accepts standard fields as described in the <a href=
@@ -104,6 +106,9 @@ public class GithubWebhooksResource extends CommonResource {
                 // set the revalidation flag to reprocess if the initial attempt fails before exiting
                 setRevalidationFlagForTracking(tracking);
             }
+        } else if ("installation".equalsIgnoreCase(eventType)) {
+            // manually trigger the application installation update 
+            ghHelper.updateAppInstallationRecords();
         }
 
         return Response.ok().build();
@@ -133,7 +138,7 @@ public class GithubWebhooksResource extends CommonResource {
             @QueryParam(GitEcaParameterNames.INSTALLATION_ID_RAW) String installationId,
             @QueryParam(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW) Integer prNo,
             @FormParam("h-captcha-response") String captchaResponse) {
-        String sanitizedRepoName = GithubValidationHelper.getFullRepoName(orgName, repoName);
+        String sanitizedRepoName = GithubHelper.getFullRepoName(orgName, repoName);
         // retrieve and check that the PR exists
         Optional<PullRequest> prResponse = ghAppService.getPullRequest(installationId, orgName, repoName, prNo);
         if (prResponse.isEmpty()) {

src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java

@@ -17,7 +17,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.eclipsefoundation.efservices.api.models.Project;
 import org.eclipsefoundation.git.eca.config.EclipseQuteTemplateExtensions;
 import org.eclipsefoundation.git.eca.dto.CommitValidationStatus;
-import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
+import org.eclipsefoundation.git.eca.helper.GithubHelper;
 import org.eclipsefoundation.git.eca.helper.ProjectHelper;
 import org.eclipsefoundation.git.eca.model.Commit;
 import org.eclipsefoundation.git.eca.model.ValidationRequest;
@@ -65,7 +65,7 @@ public class StatusResource extends CommonResource {
     ValidationStatusService validationStatus;
 
     @Inject
-    GithubValidationHelper validationHelper;
+    GithubHelper validationHelper;
     @Inject
     ProjectHelper projects;
 
@@ -146,7 +146,7 @@ public class StatusResource extends CommonResource {
     public Response getCommitValidationForGithub(@PathParam("org") String org, @PathParam("repoName") String repoName,
             @PathParam("prNo") Integer prNo) {
         // generate the URL used to retrieve valid projects
-        String repoUrl = GithubValidationHelper.getRepoUrl(org, repoName);
+        String repoUrl = GithubHelper.getRepoUrl(org, repoName);
 
         try {
             // check that the passed repo has a valid installation

src/main/java/org/eclipsefoundation/git/eca/service/impl/DefaultGithubApplicationService.java

@@ -28,7 +28,7 @@ import org.eclipsefoundation.git.eca.api.GithubAPI;
 import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
 import org.eclipsefoundation.git.eca.config.WebhooksConfig;
 import org.eclipsefoundation.git.eca.dto.GithubApplicationInstallation;
-import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
+import org.eclipsefoundation.git.eca.helper.GithubHelper;
 import org.eclipsefoundation.git.eca.helper.JwtHelper;
 import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
 import org.eclipsefoundation.git.eca.service.GithubApplicationService;
@@ -118,7 +118,7 @@ public class DefaultGithubApplicationService implements GithubApplicationService
 
     @Override
     public Optional<PullRequest> getPullRequest(String installationId, String org, String repoName, Integer pullRequest) {
-        String fullRepoName = GithubValidationHelper.getFullRepoName(org, repoName);
+        String fullRepoName = GithubHelper.getFullRepoName(org, repoName);
         // create param map to better handle multiple PRs from a single repo in short time span
         MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
         params.add(GitEcaParameterNames.PULL_REQUEST_NUMBER_RAW, Integer.toString(pullRequest));

src/main/java/org/eclipsefoundation/git/eca/tasks/GithubInstallationUpdateTask.java

@@ -11,26 +11,8 @@
  */
 package org.eclipsefoundation.git.eca.tasks;
 
-import java.net.URI;
-import java.util.Date;
-import java.util.List;
-
-import org.apache.commons.lang3.StringUtils;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
-import org.eclipse.microprofile.rest.client.inject.RestClient;
-import org.eclipsefoundation.core.service.APIMiddleware;
-import org.eclipsefoundation.git.eca.api.GithubAPI;
-import org.eclipsefoundation.git.eca.api.models.GithubApplicationInstallationData;
-import org.eclipsefoundation.git.eca.config.WebhooksConfig;
-import org.eclipsefoundation.git.eca.dto.GithubApplicationInstallation;
-import org.eclipsefoundation.git.eca.helper.JwtHelper;
-import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
-import org.eclipsefoundation.http.model.FlatRequestWrapper;
-import org.eclipsefoundation.http.model.RequestWrapper;
-import org.eclipsefoundation.persistence.dao.PersistenceDao;
-import org.eclipsefoundation.persistence.model.RDBMSQuery;
-import org.eclipsefoundation.persistence.service.FilterService;
-import org.eclipsefoundation.utils.helper.DateTimeHelper;
+import org.eclipsefoundation.git.eca.helper.GithubHelper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,8 +22,6 @@ import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.enterprise.context.control.ActivateRequestContext;
 import jakarta.enterprise.inject.Instance;
 import jakarta.inject.Inject;
-import jakarta.ws.rs.core.MultivaluedHashMap;
-import jakarta.ws.rs.core.MultivaluedMap;
 
 /**
  * Using the configured database, Github installation records are tracked for all installations available for the
@@ -54,20 +34,9 @@ public class GithubInstallationUpdateTask {
 
     @ConfigProperty(name = "eclipse.git-eca.tasks.gh-installation.enabled", defaultValue = "true")
     Instance<Boolean> isEnabled;
+    
     @Inject
-    WebhooksConfig config;
-
-    @RestClient
-    GithubAPI gh;
-
-    @Inject
-    JwtHelper jwt;
-    @Inject
-    APIMiddleware middle;
-    @Inject
-    PersistenceDao dao;
-    @Inject
-    FilterService filters;
+    GithubHelper helper;
 
     @PostConstruct
     void init() {
@@ -86,78 +55,6 @@ public class GithubInstallationUpdateTask {
         if (!Boolean.TRUE.equals(isEnabled.get())) {
             return;
         }
-
-        // get the installations for the currently configured app
-        List<GithubApplicationInstallationData> installations = middle
-                .getAll(i -> gh.getInstallations(i, "Bearer " + jwt.generateJwt()));
-        // check that there are installations
-        if (installations.isEmpty()) {
-            LOGGER.warn("Did not find any installations for the currently configured Github application");
-            return;
-        }
-        // trace log the installations for more context
-        LOGGER.debug("Found {} installations to cache", installations.size());
-
-        // create a common timestamp for easier lookups of stale entries
-        Date startingTimestamp = new Date();
-        // from installations, build records and start the processing for each entry
-        List<GithubApplicationInstallation> installationRecords = installations
-                .stream()
-                .map(this::processInstallation)
-                .toList();
-
-        // once records are prepared, persist them back to the database with updates where necessary as a batch
-        RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/webhooks/github/installations"));
-        List<GithubApplicationInstallation> repoRecords = dao
-                .add(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class)), installationRecords);
-        if (repoRecords.size() != installationRecords.size()) {
-            LOGGER.warn("Background update to installation records had a size mismatch, cleaning will be skipped for this run");
-            return;
-        }
-
-        // build query to do cleanup of stale records
-        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
-        params.add(GitEcaParameterNames.APPLICATION_ID_RAW, Integer.toString(config.github().appId()));
-        params.add(GitEcaParameterNames.LAST_UPDATED_BEFORE_RAW, DateTimeHelper.toRFC3339(startingTimestamp));
-
-        // run the delete call, removing stale entries
-        dao.delete(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class), params));
-    }
-
-    /**
-     * Converts the raw installation data from Github into a short record to be persisted to database as a form of
-     * persistent caching. Checks database for existing record, and returns record with a touched date for existing entries.
-     * 
-     * @param ghInstallation raw Github installation record for current application
-     * @return the new or updated installation record to be persisted to the database.
-     */
-    private GithubApplicationInstallation processInstallation(GithubApplicationInstallationData ghInstallation) {
-        RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/webhooks/github/installations"));
-        // build the lookup query for the current installation record
-        MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
-        params.add(GitEcaParameterNames.APPLICATION_ID_RAW, Integer.toString(config.github().appId()));
-        params.add(GitEcaParameterNames.INSTALLATION_ID_RAW, Integer.toString(ghInstallation.getId()));
-
-        // lookup existing records in the database
-        List<GithubApplicationInstallation> existingRecords = dao
-                .get(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class), params));
-
-        // check for existing entry, creating if missing
-        GithubApplicationInstallation installation;
-        if (existingRecords == null || existingRecords.isEmpty()) {
-            installation = new GithubApplicationInstallation();
-            installation.setAppId(config.github().appId());
-            installation.setInstallationId(ghInstallation.getId());
-        } else {
-            installation = existingRecords.get(0);
-        }
-        // update the basic stats to handle renames, and update last updated time
-        // login is technically nullable, so this might be missing. This is best we can do, as we can't look up by id
-        installation
-                .setName(StringUtils.isNotBlank(ghInstallation.getAccount().getLogin()) ? ghInstallation.getAccount().getLogin()
-                        : "UNKNOWN");
-        installation.setLastUpdated(new Date());
-        return installation;
-
+        helper.updateAppInstallationRecords();
     }
 }

src/main/java/org/eclipsefoundation/git/eca/tasks/GithubRevalidationQueue.java

@@ -17,7 +17,7 @@ import java.util.List;
 
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 import org.eclipsefoundation.git.eca.dto.GithubWebhookTracking;
-import org.eclipsefoundation.git.eca.helper.GithubValidationHelper;
+import org.eclipsefoundation.git.eca.helper.GithubHelper;
 import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
 import org.eclipsefoundation.http.model.FlatRequestWrapper;
 import org.eclipsefoundation.http.model.RequestWrapper;
@@ -62,7 +62,7 @@ public class GithubRevalidationQueue {
     FilterService filters;
 
     @Inject
-    GithubValidationHelper validationHelper;
+    GithubHelper validationHelper;
 
     @PostConstruct
     void init() {