Improve performance by updating how statuses are fetched for validation

• Improve performance by updating how statuses are fetched for validation

A change was made to fetch each status as it is being updated to ensure we have the latest before update. This lead to a lot of performance problems and timeout for very large commit sets. To improve this where possible, it was changed to only attempt to fetch entries if they weren't already found in the initial lookup at the start of validation.

src/main/java/org/eclipsefoundation/git/eca/service/impl/DefaultValidationStatusService.java

@@ -102,7 +102,7 @@ public class DefaultValidationStatusService implements ValidationStatusService {
                 .entrySet()
                 .stream()
                 .filter(e -> !ValidationResponse.NIL_HASH_PLACEHOLDER.equalsIgnoreCase(e.getKey()))
-                .map(e -> recordUpdatedValidationStatus(req, wrapper, p, e))
+                .map(e -> recordUpdatedValidationStatus(req, statuses, wrapper, p, e))
                 .toList();
         String fingerprint = CommitHelper.generateRequestHash(req);
         // update the base commit status and messages
@@ -122,8 +122,8 @@ public class DefaultValidationStatusService implements ValidationStatusService {
      * @param e the current commit that is being updated
      * @return the new or updated commit status
      */
-    private CommitValidationStatus recordUpdatedValidationStatus(ValidationRequest req, RequestWrapper wrap, Project p,
-            Entry<String, CommitStatus> e) {
+    private CommitValidationStatus recordUpdatedValidationStatus(ValidationRequest req, List<CommitValidationStatus> statuses,
+            RequestWrapper wrap, Project p, Entry<String, CommitStatus> e) {
         // get the commit for current status
         Optional<Commit> commit = req.getCommits().stream().filter(c -> e.getKey().equals(c.getHash())).findFirst();
         if (commit.isEmpty()) {
@@ -132,31 +132,9 @@ public class DefaultValidationStatusService implements ValidationStatusService {
             return null;
         }
         Commit c = commit.get();
-        // lookup the existing status one at a time to prevent issues with near parallel requests/duplicate commit data
-        MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
-        params.add(GitEcaParameterNames.SHAS_RAW, c.getHash());
-        params.add(GitEcaParameterNames.REPO_URL_RAW, req.getRepoUrl().toString());
-        List<CommitValidationStatus> existingStatus = dao.get(new RDBMSQuery<>(wrap, filters.get(CommitValidationStatus.class), params));
 
         // update the status if present, otherwise make new one.
-        CommitValidationStatus base;
-        if (existingStatus != null && !existingStatus.isEmpty()) {
-            base = existingStatus.get(0);
-            if (LOGGER.isTraceEnabled()) {
-                LOGGER
-                        .trace("Found existing commit status for commit '{}' in repository {}, {} existing errors", c.getHash(),
-                                req.getRepoUrl(), base.getErrors().size());
-            }
-        } else {
-            base = new CommitValidationStatus();
-            base.setProject(CommitHelper.getProjectId(p));
-            base.setCommitHash(e.getKey());
-            base.setUserMail(c.getAuthor().getMail());
-            base.setProvider(req.getProvider());
-            base.setRepoUrl(req.getRepoUrl().toString());
-            base.setCreationDate(DateTimeHelper.now());
-            base.setEstimatedLoc(req.getEstimatedLoc());
-        }
+        CommitValidationStatus base = getStatusForHash(c, req, statuses, wrap, p);
         base.setLastModified(DateTimeHelper.now());
 
         // if there are errors, update validation messages
@@ -195,6 +173,51 @@ public class DefaultValidationStatusService implements ValidationStatusService {
         return base;
     }
 
+    /**
+     * Retrieves a status for a hash, first checking the existing list fetched in bulk at the start of the request, and then attempting to
+     * fetch it fresh, before finally creating a new entry.
+     * 
+     * @param c commit being processed
+     * @param req the original request for validation
+     * @param statuses the list of original statuses fetched at the start of processing
+     * @param wrap the current request
+     * @param p project associated with the request if it exists.
+     * @return the commit validation status for the passed hash.
+     */
+    private CommitValidationStatus getStatusForHash(Commit c, ValidationRequest req, List<CommitValidationStatus> statuses,
+            RequestWrapper wrap, Project p) {
+        Optional<CommitValidationStatus> status = statuses.stream().filter(s -> c.getHash().equals(s.getCommitHash())).findFirst();
+        if (!status.isEmpty()) {
+            if (LOGGER.isTraceEnabled()) {
+                LOGGER
+                        .trace("Found existing commit status for commit '{}' in repository {}, {} existing errors", c.getHash(),
+                                req.getRepoUrl(), status.get().getErrors().size());
+            }
+            return status.get();
+        } else {
+            // lookup the existing status to prevent issues with near parallel requests/duplicate commit data
+            MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
+            params.add(GitEcaParameterNames.SHAS_RAW, c.getHash());
+            params.add(GitEcaParameterNames.REPO_URL_RAW, req.getRepoUrl().toString());
+            List<CommitValidationStatus> existingStatus = dao
+                    .get(new RDBMSQuery<>(wrap, filters.get(CommitValidationStatus.class), params));
+            if (existingStatus != null && !existingStatus.isEmpty()) {
+                return existingStatus.get(0);
+            }
+
+            // create the missing validation status and return it
+            CommitValidationStatus base = new CommitValidationStatus();
+            base.setProject(CommitHelper.getProjectId(p));
+            base.setCommitHash(c.getHash());
+            base.setUserMail(c.getAuthor().getMail());
+            base.setProvider(req.getProvider());
+            base.setRepoUrl(req.getRepoUrl().toString());
+            base.setCreationDate(DateTimeHelper.now());
+            base.setEstimatedLoc(req.getEstimatedLoc());
+            return base;
+        }
+    }
+
     /**
      * Used to deduplicate error messages on outgoing status data. This is to address an issue where in the past new and previous errors
      * weren't correctly deduplicated, leading to large amounts of duplicate messaging cluttering returns and user interfaces.