From 17b75791368464faf33c64aa8da369c4100689e6 Mon Sep 17 00:00:00 2001
From: Martin Lowe <martin.lowe@eclipse-foundation.org>
Date: Wed, 13 Sep 2023 13:01:34 -0400
Subject: [PATCH] Iss #141 - Add DB-backed GH installation cache for better
uptime
Previously, we cached installation info live at server start, but this
had awful performance and due to GH API instability, was not reliable.
The new solution should be more stable and better handle disruptions.
This will initially be updated hourly by a thread, and in the future
will have updates performed by the already configured webhook.
---
...=> GithubApplicationInstallationData.java} | 37 +++-
.../git/eca/config/WebhooksConfig.java | 2 +
.../dto/GithubApplicationInstallation.java | 183 ++++++++++++++++++
.../eca/helper/GithubValidationHelper.java | 2 +-
.../eca/namespace/GitEcaParameterNames.java | 8 +-
.../git/eca/resource/StatusResource.java | 4 +-
.../eca/service/GithubApplicationService.java | 17 +-
.../impl/DefaultGithubApplicationService.java | 81 ++++----
.../tasks/GithubInstallationUpdateTask.java | 165 ++++++++++++++++
src/main/resources/application.properties | 9 +-
src/test/resources/application.properties | 1 +
11 files changed, 445 insertions(+), 64 deletions(-)
rename src/main/java/org/eclipsefoundation/git/eca/api/models/{GithubApplicationInstallation.java => GithubApplicationInstallationData.java} (53%)
create mode 100644 src/main/java/org/eclipsefoundation/git/eca/dto/GithubApplicationInstallation.java
create mode 100644 src/main/java/org/eclipsefoundation/git/eca/tasks/GithubInstallationUpdateTask.java
diff --git a/src/main/java/org/eclipsefoundation/git/eca/api/models/GithubApplicationInstallation.java b/src/main/java/org/eclipsefoundation/git/eca/api/models/GithubApplicationInstallationData.java
similarity index 53%
rename from src/main/java/org/eclipsefoundation/git/eca/api/models/GithubApplicationInstallation.java
rename to src/main/java/org/eclipsefoundation/git/eca/api/models/GithubApplicationInstallationData.java
index 3783a36b..4924c044 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/api/models/GithubApplicationInstallation.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/api/models/GithubApplicationInstallationData.java
@@ -11,6 +11,8 @@
*/
package org.eclipsefoundation.git.eca.api.models;
+import javax.annotation.Nullable;
+
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
import com.google.auto.value.AutoValue;
@@ -22,8 +24,8 @@ import com.google.auto.value.AutoValue;
*
*/
@AutoValue
-@JsonDeserialize(builder = AutoValue_GithubApplicationInstallation.Builder.class)
-public abstract class GithubApplicationInstallation {
+@JsonDeserialize(builder = AutoValue_GithubApplicationInstallationData.Builder.class)
+public abstract class GithubApplicationInstallationData {
public abstract int getId();
@@ -31,8 +33,10 @@ public abstract class GithubApplicationInstallation {
public abstract String getTargetId();
+ public abstract Account getAccount();
+
public static Builder builder() {
- return new AutoValue_GithubApplicationInstallation.Builder();
+ return new AutoValue_GithubApplicationInstallationData.Builder();
}
@AutoValue.Builder
@@ -44,6 +48,31 @@ public abstract class GithubApplicationInstallation {
public abstract Builder setTargetId(String targetId);
- public abstract GithubApplicationInstallation build();
+ public abstract Builder setAccount(Account account);
+
+ public abstract GithubApplicationInstallationData build();
+ }
+
+ @AutoValue
+ @JsonDeserialize(builder = AutoValue_GithubApplicationInstallationData_Account.Builder.class)
+ public abstract static class Account {
+ public abstract int getId();
+
+ @Nullable
+ public abstract String getLogin();
+
+ public static Builder builder() {
+ return new AutoValue_GithubApplicationInstallationData_Account.Builder();
+ }
+
+ @AutoValue.Builder
+ @JsonPOJOBuilder(withPrefix = "set")
+ public abstract static class Builder {
+ public abstract Builder setId(int id);
+
+ public abstract Builder setLogin(@Nullable String login);
+
+ public abstract Account build();
+ }
}
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/config/WebhooksConfig.java b/src/main/java/org/eclipsefoundation/git/eca/config/WebhooksConfig.java
index 2d7f00df..62cf9277 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/config/WebhooksConfig.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/config/WebhooksConfig.java
@@ -25,5 +25,7 @@ public interface WebhooksConfig {
String context();
String serverTarget();
+
+ int appId();
}
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/dto/GithubApplicationInstallation.java b/src/main/java/org/eclipsefoundation/git/eca/dto/GithubApplicationInstallation.java
new file mode 100644
index 00000000..8c05c352
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/git/eca/dto/GithubApplicationInstallation.java
@@ -0,0 +1,183 @@
+/**
+ * Copyright (c) 2023 Eclipse Foundation
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * Author: Martin Lowe <martin.lowe@eclipse-foundation.org>
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.git.eca.dto;
+
+import java.time.format.DateTimeParseException;
+import java.util.Date;
+import java.util.Objects;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.ws.rs.core.MultivaluedMap;
+
+import org.apache.commons.lang3.StringUtils;
+import org.eclipsefoundation.core.helper.DateTimeHelper;
+import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
+import org.eclipsefoundation.persistence.dto.BareNode;
+import org.eclipsefoundation.persistence.dto.filter.DtoFilter;
+import org.eclipsefoundation.persistence.model.DtoTable;
+import org.eclipsefoundation.persistence.model.ParameterizedSQLStatement;
+import org.eclipsefoundation.persistence.model.ParameterizedSQLStatementBuilder;
+
+/**
+ * Tracking for github installation data for GH ECA applications used for requesting data and updating commit statuses.
+ */
+@Table
+@Entity
+public class GithubApplicationInstallation extends BareNode {
+ public static final DtoTable TABLE = new DtoTable(GithubApplicationInstallation.class, "gai");
+
+ @Id
+ @GeneratedValue(strategy = GenerationType.IDENTITY)
+ private long id;
+ private int appId;
+ private int installationId;
+ private String name;
+ private Date lastUpdated;
+
+ @Override
+ public Long getId() {
+ return id;
+ }
+
+ /**
+ * @param id the id to set
+ */
+ public void setId(long id) {
+ this.id = id;
+ }
+
+ /**
+ * @return the appId
+ */
+ public int getAppId() {
+ return appId;
+ }
+
+ /**
+ * @param appId the appId to set
+ */
+ public void setAppId(int appId) {
+ this.appId = appId;
+ }
+
+ /**
+ * @return the installationId
+ */
+ public int getInstallationId() {
+ return installationId;
+ }
+
+ /**
+ * @param installationId the installationId to set
+ */
+ public void setInstallationId(int installationId) {
+ this.installationId = installationId;
+ }
+
+ /**
+ * @return the name
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * @param name the name to set
+ */
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * @return the lastUpdated
+ */
+ public Date getLastUpdated() {
+ return lastUpdated;
+ }
+
+ /**
+ * @param lastUpdated the lastUpdated to set
+ */
+ public void setLastUpdated(Date lastUpdated) {
+ this.lastUpdated = lastUpdated;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + Objects.hash(appId, id, installationId, lastUpdated, name);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!super.equals(obj)) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ GithubApplicationInstallation other = (GithubApplicationInstallation) obj;
+ return appId == other.appId && id == other.id && installationId == other.installationId
+ && Objects.equals(lastUpdated, other.lastUpdated) && Objects.equals(name, other.name);
+ }
+
+ @Singleton
+ public static class GithubApplicationInstallationFilter implements DtoFilter<GithubApplicationInstallation> {
+
+ @Inject
+ ParameterizedSQLStatementBuilder builder;
+
+ @Override
+ public ParameterizedSQLStatement getFilters(MultivaluedMap<String, String> params, boolean isRoot) {
+ ParameterizedSQLStatement statement = builder.build(TABLE);
+
+ String installationId = params.getFirst(GitEcaParameterNames.INSTALLATION_ID_RAW);
+ if (StringUtils.isNotBlank(installationId)) {
+ statement
+ .addClause(new ParameterizedSQLStatement.Clause(TABLE.getAlias() + ".installationId = ?",
+ new Object[] { Integer.parseInt(installationId) }));
+ }
+ String appId = params.getFirst(GitEcaParameterNames.APPLICATION_ID_RAW);
+ if (StringUtils.isNumeric(appId)) {
+ statement
+ .addClause(new ParameterizedSQLStatement.Clause(TABLE.getAlias() + ".appId = ?",
+ new Object[] { Integer.parseInt(appId) }));
+ }
+ String lastUpdated = params.getFirst(GitEcaParameterNames.LAST_UPDATED_BEFORE_RAW);
+ if (StringUtils.isNotBlank(lastUpdated)) {
+ try {
+ Date dt = DateTimeHelper.toRFC3339(lastUpdated);
+ statement.addClause(new ParameterizedSQLStatement.Clause(TABLE.getAlias() + ".lastUpdated < ?", new Object[] { dt }));
+ } catch (DateTimeParseException e) {
+ // do not do anything for badly formatted args
+ }
+ }
+ return statement;
+ }
+
+ @Override
+ public Class<GithubApplicationInstallation> getType() {
+ return GithubApplicationInstallation.class;
+ }
+ }
+}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java b/src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java
index f5ec3697..6fa08c4c 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/helper/GithubValidationHelper.java
@@ -110,7 +110,7 @@ public class GithubValidationHelper {
// use logging helper to sanitize newlines as they aren't needed here
String fullRepoName = LoggingHelper.format(org + '/' + repoName);
// get the installation ID for the given repo if it exists, and if the PR noted exists
- String installationId = ghAppService.getInstallationForRepo(fullRepoName);
+ String installationId = ghAppService.getInstallationForRepo(org, repoName);
Optional<PullRequest> prResponse = ghAppService.getPullRequest(installationId, fullRepoName, prNo);
if (StringUtils.isBlank(installationId)) {
throw new BadRequestException("Could not find an ECA app installation for repo name: " + fullRepoName);
diff --git a/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java b/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java
index 70a657c1..35d4fc24 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/namespace/GitEcaParameterNames.java
@@ -37,6 +37,8 @@ public final class GitEcaParameterNames implements UrlParameterNamespace {
public static final String UNTIL_RAW = "until";
public static final String REPOSITORY_FULL_NAME_RAW = "repo_full_name";
public static final String INSTALLATION_ID_RAW = "installation_id";
+ public static final String APPLICATION_ID_RAW = "application_id";
+ public static final String LAST_UPDATED_BEFORE_RAW = "last_updated_before";
public static final String PULL_REQUEST_NUMBER_RAW = "pull_request_number";
public static final String USER_MAIL_RAW = "user_mail";
public static final String NEEDS_REVALIDATION_RAW = "needs_revalidation";
@@ -57,6 +59,8 @@ public final class GitEcaParameterNames implements UrlParameterNamespace {
public static final UrlParameter UNTIL = new UrlParameter(UNTIL_RAW);
public static final UrlParameter REPOSITORY_FULL_NAME = new UrlParameter(REPOSITORY_FULL_NAME_RAW);
public static final UrlParameter INSTALLATION_ID = new UrlParameter(INSTALLATION_ID_RAW);
+ public static final UrlParameter APPLICATION_ID = new UrlParameter(APPLICATION_ID_RAW);
+ public static final UrlParameter LAST_UPDATED_BEFORE = new UrlParameter(LAST_UPDATED_BEFORE_RAW);
public static final UrlParameter PULL_REQUEST_NUMBER = new UrlParameter(PULL_REQUEST_NUMBER_RAW);
public static final UrlParameter USER_MAIL = new UrlParameter(USER_MAIL_RAW);
public static final UrlParameter NEEDS_REVALIDATION = new UrlParameter(NEEDS_REVALIDATION_RAW);
@@ -65,8 +69,8 @@ public final class GitEcaParameterNames implements UrlParameterNamespace {
public List<UrlParameter> getParameters() {
return Arrays
.asList(COMMIT_ID, SHA, SHAS, PROJECT_ID, PROJECT_IDS, NOT_IN_PROJECT_IDS, REPO_URL, FINGERPRINT, USER_ID, PROJECT_PATH,
- PARENT_PROJECT, STATUS_ACTIVE, STATUS_DELETED, SINCE, UNTIL, REPOSITORY_FULL_NAME, INSTALLATION_ID,
- PULL_REQUEST_NUMBER, USER_MAIL, NEEDS_REVALIDATION);
+ PARENT_PROJECT, STATUS_ACTIVE, STATUS_DELETED, SINCE, UNTIL, REPOSITORY_FULL_NAME, INSTALLATION_ID, APPLICATION_ID,
+ LAST_UPDATED_BEFORE, PULL_REQUEST_NUMBER, USER_MAIL, NEEDS_REVALIDATION);
}
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java b/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
index 0db2c74d..630ae1f1 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/resource/StatusResource.java
@@ -122,7 +122,7 @@ public class StatusResource extends GithubAdjacentResource {
@PathParam("prNo") Integer prNo) {
String repoFullName = org + '/' + repoName;
// check that the passed repo has a valid installation
- String installationId = ghAppService.getInstallationForRepo(repoFullName);
+ String installationId = ghAppService.getInstallationForRepo(org, repoName);
if (StringUtils.isBlank(installationId)) {
throw new BadRequestException("Repo " + repoFullName + " requested, but does not have visible installation, returning");
}
@@ -158,7 +158,7 @@ public class StatusResource extends GithubAdjacentResource {
.data("fullRepoName", repoFullName)
.data("project", ps.isEmpty() ? null : ps.get(0))
.data("repoUrl", repoUrl)
- .data("installationId", ghAppService.getInstallationForRepo(repoFullName))
+ .data("installationId", installationId)
.render())
.build();
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/service/GithubApplicationService.java b/src/main/java/org/eclipsefoundation/git/eca/service/GithubApplicationService.java
index 4b6cafab..ae0a120f 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/service/GithubApplicationService.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/service/GithubApplicationService.java
@@ -11,11 +11,11 @@
*/
package org.eclipsefoundation.git.eca.service;
+import java.util.List;
import java.util.Optional;
-import javax.ws.rs.core.MultivaluedMap;
-
import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
+import org.eclipsefoundation.git.eca.dto.GithubApplicationInstallation;
/**
* Service for interacting with the Github API with caching for performance.
@@ -26,19 +26,20 @@ import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest
public interface GithubApplicationService {
/**
- * Retrieve safe map of all installations that are managed by this instance of the Eclispe ECA app.
+ * Retrieve safe list of all installations that are managed by this instance of the Eclispe ECA app.
*
- * @return map containing installation IDs mapped to the list of repos they support for the current app
+ * @return list containing installation records for the current application
*/
- MultivaluedMap<String, String> getManagedInstallations();
+ List<GithubApplicationInstallation> getManagedInstallations();
/**
- * Retrieves the installation ID for the ECA app on the given repo if it exists.
+ * Retrieves the installation ID for the ECA app on the given org or repo if it exists.
*
- * @param repoFullName the full repo name to retrieve an installation ID for. E.g. eclipse/jetty
+ * @param org the name of organization to retrieve an installation ID for
+ * @param repo the name of repo to retrieve an installation ID for
* @return the numeric installation ID if it exists, or null
*/
- String getInstallationForRepo(String repoFullName);
+ String getInstallationForRepo(String org, String repo);
/**
* Retrieves a pull request given the repo, pull request, and associated installation to action the fetch.
diff --git a/src/main/java/org/eclipsefoundation/git/eca/service/impl/DefaultGithubApplicationService.java b/src/main/java/org/eclipsefoundation/git/eca/service/impl/DefaultGithubApplicationService.java
index d1f04c33..ca5cd9e7 100644
--- a/src/main/java/org/eclipsefoundation/git/eca/service/impl/DefaultGithubApplicationService.java
+++ b/src/main/java/org/eclipsefoundation/git/eca/service/impl/DefaultGithubApplicationService.java
@@ -11,8 +11,10 @@
*/
package org.eclipsefoundation.git.eca.service.impl;
+import java.net.URI;
import java.time.Duration;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
@@ -26,14 +28,20 @@ import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.eclipse.microprofile.context.ManagedExecutor;
import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.core.exception.ApplicationException;
+import org.eclipsefoundation.core.model.FlatRequestWrapper;
+import org.eclipsefoundation.core.model.RequestWrapper;
import org.eclipsefoundation.core.service.APIMiddleware;
import org.eclipsefoundation.core.service.CachingService;
import org.eclipsefoundation.git.eca.api.GithubAPI;
-import org.eclipsefoundation.git.eca.api.models.GithubApplicationInstallation;
-import org.eclipsefoundation.git.eca.api.models.GithubInstallationRepositoriesResponse;
import org.eclipsefoundation.git.eca.api.models.GithubWebhookRequest.PullRequest;
+import org.eclipsefoundation.git.eca.config.WebhooksConfig;
+import org.eclipsefoundation.git.eca.dto.GithubApplicationInstallation;
import org.eclipsefoundation.git.eca.helper.JwtHelper;
+import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
import org.eclipsefoundation.git.eca.service.GithubApplicationService;
+import org.eclipsefoundation.persistence.dao.PersistenceDao;
+import org.eclipsefoundation.persistence.model.RDBMSQuery;
+import org.eclipsefoundation.persistence.service.FilterService;
import org.jboss.resteasy.specimpl.MultivaluedMapImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,21 +66,27 @@ public class DefaultGithubApplicationService implements GithubApplicationService
@ConfigProperty(name = "eclipse.github.default-api-version", defaultValue = "2022-11-28")
String apiVersion;
+ @Inject
+ WebhooksConfig config;
@RestClient
GithubAPI gh;
@Inject
- JwtHelper jwt;
+ PersistenceDao dao;
+ @Inject
+ FilterService filters;
@Inject
CachingService cache;
@Inject
APIMiddleware middle;
+ @Inject
+ JwtHelper jwt;
@Inject
ManagedExecutor exec;
- private AsyncLoadingCache<String, MultivaluedMap<String, String>> installationRepositoriesCache;
+ private AsyncLoadingCache<String, List<GithubApplicationInstallation>> installationRepositoriesCache;
@PostConstruct
void init() {
@@ -87,17 +101,18 @@ public class DefaultGithubApplicationService implements GithubApplicationService
}
@Override
- public MultivaluedMap<String, String> getManagedInstallations() {
- // create a deep copy of the map to protect from modifications and return it
- MultivaluedMap<String, String> mapClone = new MultivaluedMapImpl<>();
- getAllInstallRepos().entrySet().stream().forEach(e -> mapClone.put(e.getKey(), new ArrayList<>(e.getValue())));
- return mapClone;
+ public List<GithubApplicationInstallation> getManagedInstallations() {
+ return new ArrayList<>(getAllInstallRepos());
}
@Override
- public String getInstallationForRepo(String repoFullName) {
- MultivaluedMap<String, String> map = getAllInstallRepos();
- return map.keySet().stream().filter(k -> map.get(k).contains(repoFullName)).findFirst().orElse(null);
+ public String getInstallationForRepo(String org, String repo) {
+ return getAllInstallRepos()
+ .stream()
+ .filter(install -> install.getName().equalsIgnoreCase(org))
+ .map(install -> Integer.toString(install.getInstallationId()))
+ .findFirst()
+ .orElse(null);
}
@Override
@@ -107,7 +122,7 @@ public class DefaultGithubApplicationService implements GithubApplicationService
() -> gh.getPullRequest(jwt.getGhBearerString(installationId), apiVersion, repoFullName, pullRequest));
}
- private MultivaluedMap<String, String> getAllInstallRepos() {
+ private List<GithubApplicationInstallation> getAllInstallRepos() {
try {
return this.installationRepositoriesCache.get("all").get(INSTALL_REPO_FETCH_MAX_TIMEOUT, TimeUnit.SECONDS);
} catch (InterruptedException e) {
@@ -118,42 +133,22 @@ public class DefaultGithubApplicationService implements GithubApplicationService
throw new ApplicationException(
"Thread interrupted while building repository cache, no entries will be available for current call", e);
}
- return new MultivaluedMapImpl<>();
+ return Collections.emptyList();
}
/**
- * Retrieves a fresh copy of installation repositories, mapped by installation ID to associated full repo names.
+ * Retrieves a fresh copy of tracked installations, which have the associated app, the installation ID, and the name of the namespace associated with it.
*
* @return a multivalued map relating installation IDs to associated full repo names.
*/
- private MultivaluedMap<String, String> loadInstallationRepositories() {
- // create map early for potential empty returns
- MultivaluedMapImpl<String, String> out = new MultivaluedMapImpl<>();
- // get JWT bearer and then get all associated installations of current app
- String auth = "Bearer " + jwt.generateJwt();
- List<GithubApplicationInstallation> installations = middle
- .getAll(i -> gh.getInstallations(i, auth), GithubApplicationInstallation.class);
- // check that there are installations
- if (installations.isEmpty()) {
- LOGGER.warn("Did not find any installations for the currently configured Github application");
- return out;
- }
- // trace log the installations for more context
- LOGGER.trace("Found the following installations: {}", installations);
-
- // from installations, get the assoc. repos and grab their full repo name and collect them
- installations
- .stream()
- .forEach(installation -> middle
- .getAll(i -> gh.getInstallationRepositories(i, jwt.getGhBearerString(Integer.toString(installation.getId()))),
- GithubInstallationRepositoriesResponse.class)
- .stream()
- .forEach(installRepo -> installRepo
- .getRepositories()
- .stream()
- .forEach(r -> out.add(Integer.toString(installation.getId()), r.getFullName()))));
- LOGGER.trace("Final results for generating installation mapping: {}", out);
- return out;
+ private List<GithubApplicationInstallation> loadInstallationRepositories() {
+ // once records are prepared, persist them back to the database with updates where necessary as a batch
+ RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/webhooks/github/installations"));
+
+ // build query to do fetch of records for currently active application
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
+ params.add(GitEcaParameterNames.APPLICATION_ID_RAW, Integer.toString(config.github().appId()));
+ return dao.get(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class), params));
}
}
diff --git a/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubInstallationUpdateTask.java b/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubInstallationUpdateTask.java
new file mode 100644
index 00000000..30c431fe
--- /dev/null
+++ b/src/main/java/org/eclipsefoundation/git/eca/tasks/GithubInstallationUpdateTask.java
@@ -0,0 +1,165 @@
+/**
+ * Copyright (c) 2023 Eclipse Foundation
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * Author: Martin Lowe <martin.lowe@eclipse-foundation.org>
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.git.eca.tasks;
+
+import java.net.URI;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import javax.annotation.PostConstruct;
+import javax.enterprise.context.ApplicationScoped;
+import javax.enterprise.context.control.ActivateRequestContext;
+import javax.enterprise.inject.Instance;
+import javax.inject.Inject;
+import javax.ws.rs.core.MultivaluedMap;
+
+import org.apache.maven.shared.utils.StringUtils;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+import org.eclipse.microprofile.rest.client.inject.RestClient;
+import org.eclipsefoundation.core.helper.DateTimeHelper;
+import org.eclipsefoundation.core.model.FlatRequestWrapper;
+import org.eclipsefoundation.core.model.RequestWrapper;
+import org.eclipsefoundation.core.service.APIMiddleware;
+import org.eclipsefoundation.git.eca.api.GithubAPI;
+import org.eclipsefoundation.git.eca.api.models.GithubApplicationInstallationData;
+import org.eclipsefoundation.git.eca.config.WebhooksConfig;
+import org.eclipsefoundation.git.eca.dto.GithubApplicationInstallation;
+import org.eclipsefoundation.git.eca.helper.JwtHelper;
+import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
+import org.eclipsefoundation.persistence.dao.PersistenceDao;
+import org.eclipsefoundation.persistence.model.RDBMSQuery;
+import org.eclipsefoundation.persistence.service.FilterService;
+import org.jboss.resteasy.specimpl.MultivaluedMapImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.quarkus.scheduler.Scheduled;
+
+/**
+ * Using the configured database, Github installation records are tracked for all installations available for the
+ * currently configured GH application. This is used over an in-memory cache as the calls to build the cache are long
+ * running and have been known to fail sporadically.
+ */
+@ApplicationScoped
+public class GithubInstallationUpdateTask {
+ private static final Logger LOGGER = LoggerFactory.getLogger(GithubInstallationUpdateTask.class);
+
+ @ConfigProperty(name = "eclipse.git-eca.tasks.gh-installation.enabled", defaultValue = "true")
+ Instance<Boolean> isEnabled;
+ @Inject
+ WebhooksConfig config;
+
+ @RestClient
+ GithubAPI gh;
+
+ @Inject
+ JwtHelper jwt;
+ @Inject
+ APIMiddleware middle;
+ @Inject
+ PersistenceDao dao;
+ @Inject
+ FilterService filters;
+
+ @PostConstruct
+ void init() {
+ // indicate to log whether enabled to reduce log spam
+ LOGGER.info("Github installation DB cache task is{} enabled.", Boolean.TRUE.equals(isEnabled.get()) ? "" : " not");
+ }
+
+ /**
+ * Every 1h, this method will query Github for installations for the currently configured application. These
+ * installations will be translated into database entries for a persistent cache to be loaded on request.
+ */
+ @Scheduled(every = "1h")
+ @ActivateRequestContext
+ public void revalidate() {
+ // if not enabled, don't process any potentially OOD records
+ if (!Boolean.TRUE.equals(isEnabled.get())) {
+ return;
+ }
+
+ // get the installations for the currently configured app
+ List<GithubApplicationInstallationData> installations = middle
+ .getAll(i -> gh.getInstallations(i, "Bearer " + jwt.generateJwt()), GithubApplicationInstallationData.class);
+ // check that there are installations
+ if (installations.isEmpty()) {
+ LOGGER.warn("Did not find any installations for the currently configured Github application");
+ return;
+ }
+ // trace log the installations for more context
+ LOGGER.debug("Found {} installations to cache", installations.size());
+
+ // create a common timestamp for easier lookups of stale entries
+ Date startingTimestamp = new Date();
+ // from installations, build records and start the processing for each entry
+ List<GithubApplicationInstallation> installationRecords = installations
+ .stream()
+ .map(this::processInstallation)
+ .collect(Collectors.toList());
+
+ // once records are prepared, persist them back to the database with updates where necessary as a batch
+ RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/webhooks/github/installations"));
+ List<GithubApplicationInstallation> repoRecords = dao
+ .add(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class)), installationRecords);
+ if (repoRecords.size() != installationRecords.size()) {
+ LOGGER.warn("Background update to installation records had a size mismatch, cleaning will be skipped for this run");
+ return;
+ }
+
+ // build query to do cleanup of stale records
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
+ params.add(GitEcaParameterNames.APPLICATION_ID_RAW, Integer.toString(config.github().appId()));
+ params.add(GitEcaParameterNames.LAST_UPDATED_BEFORE_RAW, DateTimeHelper.toRFC3339(startingTimestamp));
+
+ // run the delete call, removing stale entries
+ dao.delete(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class), params));
+ }
+
+ /**
+ * Converts the raw installation data from Github into a short record to be persisted to database as a form of
+ * persistent caching. Checks database for existing record, and returns record with a touched date for existing entries.
+ *
+ * @param ghInstallation raw Github installation record for current application
+ * @return the new or updated installation record to be persisted to the database.
+ */
+ private GithubApplicationInstallation processInstallation(GithubApplicationInstallationData ghInstallation) {
+ RequestWrapper wrap = new FlatRequestWrapper(URI.create("https://api.eclipse.org/git/webhooks/github/installations"));
+ // build the lookup query for the current installation record
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
+ params.add(GitEcaParameterNames.APPLICATION_ID_RAW, Integer.toString(config.github().appId()));
+ params.add(GitEcaParameterNames.INSTALLATION_ID_RAW, Integer.toString(ghInstallation.getId()));
+
+ // lookup existing records in the database
+ List<GithubApplicationInstallation> existingRecords = dao
+ .get(new RDBMSQuery<>(wrap, filters.get(GithubApplicationInstallation.class), params));
+
+ // check for existing entry, creating if missing
+ GithubApplicationInstallation installation;
+ if (existingRecords == null || existingRecords.isEmpty()) {
+ installation = new GithubApplicationInstallation();
+ installation.setAppId(config.github().appId());
+ installation.setInstallationId(ghInstallation.getId());
+ } else {
+ installation = existingRecords.get(0);
+ }
+ // update the basic stats to handle renames, and update last updated time
+ // login is technically nullable, so this might be missing. This is best we can do, as we can't look up by id
+ installation
+ .setName(StringUtils.isNotBlank(ghInstallation.getAccount().getLogin()) ? ghInstallation.getAccount().getLogin()
+ : "UNKNOWN");
+ installation.setLastUpdated(new Date());
+ return installation;
+
+ }
+}
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
index f113468e..08e8aee3 100644
--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -40,13 +40,14 @@ quarkus.cache.caffeine."record".expire-after-write=${quarkus.cache.caffeine."def
quarkus.cache.caffeine."accesstoken".initial-capacity=1000
quarkus.cache.caffeine."accesstoken".expire-after-write=119S
-## JWT Placeholders/defaults
-smallrye.jwt.new-token.lifespan=120
-smallrye.jwt.new-token.issuer=262450
-
## Webhook configs
eclipse.webhooks.github.context=eclipsefdn/eca
eclipse.webhooks.github.server-target=https://api.eclipse.org
+eclipse.webhooks.github.app-id=262450
+
+## JWT Placeholders/defaults
+smallrye.jwt.new-token.lifespan=120
+smallrye.jwt.new-token.issuer=${eclipse.webhooks.github.app-id}
## Git-eca mail configs
eclipse.git-eca.mail-validation.allow-list=noreply@github.com,49699333+dependabot[bot]@users.noreply.github.com,bot@stepsecurity.io
diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties
index 4b362d93..5599303b 100644
--- a/src/test/resources/application.properties
+++ b/src/test/resources/application.properties
@@ -35,6 +35,7 @@ eclipse.git-eca.mail.allow-list=noreply@github.com
## Reports configs
eclipse.git-eca.reports.access-key=samplekey
eclipse.git-eca.tasks.gh-revalidation.enabled=false
+eclipse.git-eca.tasks.gh-installation.enabled=false
## Misc
eclipse.gitlab.access-token=token_val
--
GitLab