Skip to content
Snippets Groups Projects
Commit ceb411b2 authored by Martin Lowe's avatar Martin Lowe :flag_ca:
Browse files

Merge branch 'zacharysabourin/master/82' into 'master'

feat: Implement scheduled task to reconcile private project data

Closes #82

See merge request !110
parents afb992eb 66e6b279
No related branches found
No related tags found
1 merge request!110feat: Implement scheduled task to reconcile private project data
Pipeline #12724 passed
Showing
with 285 additions and 17 deletions
......@@ -5,7 +5,7 @@
<artifactId>git-eca</artifactId>
<version>1.1.0</version>
<properties>
<eclipse-api-version>0.6.7</eclipse-api-version>
<eclipse-api-version>0.6.8</eclipse-api-version>
<compiler-plugin.version>3.8.1</compiler-plugin.version>
<maven.compiler.parameters>true</maven.compiler.parameters>
<maven.compiler.source>11</maven.compiler.source>
......@@ -72,6 +72,11 @@
<groupId>io.quarkus</groupId>
<artifactId>quarkus-resteasy-qute</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-scheduler</artifactId>
</dependency>
<!-- Annotation preprocessors - reduce all of the boiler plate -->
<dependency>
<groupId>com.google.auto.value</groupId>
......
......@@ -22,7 +22,7 @@ import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import org.eclipse.microprofile.rest.client.inject.RegisterRestClient;
import org.eclipsefoundation.git.eca.model.EclipseUser;
import org.eclipsefoundation.git.eca.api.models.EclipseUser;
/**
* Binding interface for the Eclipse Foundation user account API. Runtime
......
......@@ -12,19 +12,24 @@
package org.eclipsefoundation.git.eca.api;
import javax.enterprise.context.ApplicationScoped;
import javax.ws.rs.BeanParam;
import javax.ws.rs.GET;
import javax.ws.rs.HeaderParam;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Response;
import org.eclipse.microprofile.rest.client.inject.RegisterRestClient;
import org.eclipsefoundation.git.eca.model.GitlabProjectResponse;
import org.eclipsefoundation.core.service.APIMiddleware.BaseAPIParameters;
import org.eclipsefoundation.git.eca.api.models.GitlabProjectResponse;
/**
* Interface for interacting with the GitLab API. Used to fetch project data.
*/
@ApplicationScoped
@RegisterRestClient
@Path("/projects")
public interface GitlabAPI {
/**
......@@ -36,6 +41,21 @@ public interface GitlabAPI {
* @return A GitlabProjectResponse object
*/
@GET
@Path("/projects/{id}")
GitlabProjectResponse getProjectInfo(@HeaderParam("PRIVATE-TOKEN") String privateToken, @PathParam("id") int projectId);
@Path("/{id}")
GitlabProjectResponse getProjectInfo(@HeaderParam("PRIVATE-TOKEN") String privateToken,
@PathParam("id") int projectId);
/**
* Fetches data for private projects. A token of adequate permissions is
* required. Visibility should be set to "private" and per_page should be set to
* 100 to minimize API calls.
*
* @param privateToken the header token
* @param visibility the project visibility
* @param perPage the number of results per page
* @return A Response containing a private project list
*/
@GET
Response getPrivateProjects(@BeanParam BaseAPIParameters baseParams, @HeaderParam("PRIVATE-TOKEN") String privateToken,
@QueryParam("visibility") String visibility, @QueryParam("per_page") Integer perPage);
}
......@@ -9,10 +9,12 @@
*
* SPDX-License-Identifier: EPL-2.0
**********************************************************************/
package org.eclipsefoundation.git.eca.model;
package org.eclipsefoundation.git.eca.api.models;
import javax.annotation.Nullable;
import org.eclipsefoundation.git.eca.model.GitUser;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
......
......@@ -9,7 +9,9 @@
*
* SPDX-License-Identifier: EPL-2.0
**********************************************************************/
package org.eclipsefoundation.git.eca.model;
package org.eclipsefoundation.git.eca.api.models;
import java.time.ZonedDateTime;
import javax.annotation.Nullable;
......@@ -24,8 +26,12 @@ public abstract class GitlabProjectResponse {
public abstract Integer getId();
public abstract String getPathWithNamespace();
public abstract Integer getCreatorId();
public abstract ZonedDateTime getCreatedAt();
@Nullable
public abstract ForkedProject getForkedFromProject();
......@@ -38,9 +44,15 @@ public abstract class GitlabProjectResponse {
public abstract static class Builder {
public abstract Builder setId(Integer id);
@JsonProperty("path_with_namespace")
public abstract Builder setPathWithNamespace(String path);
@JsonProperty("creator_id")
public abstract Builder setCreatorId(Integer creatorId);
@JsonProperty("created_at")
public abstract Builder setCreatedAt(ZonedDateTime created);
@JsonProperty("forked_from_project")
public abstract Builder setForkedFromProject(@Nullable ForkedProject project);
......
......@@ -13,6 +13,8 @@ package org.eclipsefoundation.git.eca.dto;
import java.io.Serializable;
import java.time.LocalDateTime;
import java.util.List;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
......@@ -164,6 +166,25 @@ public class PrivateProjectEvent extends BareNode {
new Object[] { Integer.valueOf(projectId) }));
}
List<String> projectIds = params.get(GitEcaParameterNames.PROJECT_IDS.getName());
if (projectIds != null) {
statement.addClause(
new ParameterizedSQLStatement.Clause(TABLE.getAlias() + ".compositeId.projectId IN ?",
new Object[] {
projectIds.stream().map(Integer::valueOf).collect(Collectors.toList()) }));
}
List<String> notInProjectIds = params.get(GitEcaParameterNames.NOT_IN_PROJECT_IDS.getName());
if (notInProjectIds != null) {
statement.addClause(
new ParameterizedSQLStatement.Clause(TABLE.getAlias() + ".compositeId.projectId NOT IN ?",
new Object[] {
notInProjectIds.stream().map(Integer::valueOf).collect(Collectors.toList()) }));
statement.addClause(new ParameterizedSQLStatement.Clause(TABLE.getAlias() + ".deletionDate IS NULL",
new Object[] {}));
}
String projectPath = params.getFirst(GitEcaParameterNames.PROJECT_PATH.getName());
if (StringUtils.isNotBlank(projectPath)) {
statement.addClause(
......
......@@ -24,6 +24,8 @@ public final class GitEcaParameterNames implements UrlParameterNamespace {
public static final String SHA_RAW = "sha";
public static final String SHAS_RAW = "shas";
public static final String PROJECT_ID_RAW = "project_id";
public static final String PROJECT_IDS_RAW = "project_ids";
public static final String NOT_IN_PROJECT_IDS_RAW = "not_in_project_ids";
public static final String REPO_URL_RAW = "repo_url";
public static final String FINGERPRINT_RAW = "fingerprint";
public static final String USER_ID_RAW = "user_id";
......@@ -33,6 +35,8 @@ public final class GitEcaParameterNames implements UrlParameterNamespace {
public static final UrlParameter SHA = new UrlParameter(SHA_RAW);
public static final UrlParameter SHAS = new UrlParameter(SHAS_RAW);
public static final UrlParameter PROJECT_ID = new UrlParameter(PROJECT_ID_RAW);
public static final UrlParameter PROJECT_IDS = new UrlParameter(PROJECT_IDS_RAW);
public static final UrlParameter NOT_IN_PROJECT_IDS = new UrlParameter(NOT_IN_PROJECT_IDS_RAW);
public static final UrlParameter REPO_URL = new UrlParameter(REPO_URL_RAW);
public static final UrlParameter FINGERPRINT = new UrlParameter(FINGERPRINT_RAW);
public static final UrlParameter USER_ID = new UrlParameter(USER_ID_RAW);
......@@ -41,7 +45,8 @@ public final class GitEcaParameterNames implements UrlParameterNamespace {
@Override
public List<UrlParameter> getParameters() {
return Arrays.asList(COMMIT_ID, SHA, SHAS, PROJECT_ID, REPO_URL, USER_ID, PROJECT_PATH, PARENT_PROJECT);
return Arrays.asList(COMMIT_ID, SHA, SHAS, PROJECT_ID, PROJECT_IDS, NOT_IN_PROJECT_IDS, REPO_URL,
FINGERPRINT, USER_ID, PROJECT_PATH, PARENT_PROJECT);
}
}
......@@ -35,10 +35,10 @@ import javax.ws.rs.core.Response;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.eclipsefoundation.core.model.RequestWrapper;
import org.eclipsefoundation.core.service.CachingService;
import org.eclipsefoundation.git.eca.api.models.EclipseUser;
import org.eclipsefoundation.git.eca.dto.CommitValidationStatus;
import org.eclipsefoundation.git.eca.helper.CommitHelper;
import org.eclipsefoundation.git.eca.model.Commit;
import org.eclipsefoundation.git.eca.model.EclipseUser;
import org.eclipsefoundation.git.eca.model.GitUser;
import org.eclipsefoundation.git.eca.model.Project;
import org.eclipsefoundation.git.eca.model.ValidationRequest;
......
......@@ -13,7 +13,7 @@ package org.eclipsefoundation.git.eca.service;
import java.util.List;
import org.eclipsefoundation.git.eca.model.EclipseUser;
import org.eclipsefoundation.git.eca.api.models.EclipseUser;
import org.eclipsefoundation.git.eca.model.Project;
public interface UserService {
......
......@@ -29,7 +29,7 @@ import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.core.service.CachingService;
import org.eclipsefoundation.git.eca.api.AccountsAPI;
import org.eclipsefoundation.git.eca.api.BotsAPI;
import org.eclipsefoundation.git.eca.model.EclipseUser;
import org.eclipsefoundation.git.eca.api.models.EclipseUser;
import org.eclipsefoundation.git.eca.model.Project;
import org.eclipsefoundation.git.eca.service.OAuthService;
import org.eclipsefoundation.git.eca.service.UserService;
......
......@@ -27,8 +27,8 @@ import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.core.model.RequestWrapper;
import org.eclipsefoundation.core.service.CachingService;
import org.eclipsefoundation.git.eca.api.GitlabAPI;
import org.eclipsefoundation.git.eca.api.models.GitlabProjectResponse;
import org.eclipsefoundation.git.eca.dto.PrivateProjectEvent;
import org.eclipsefoundation.git.eca.model.GitlabProjectResponse;
import org.eclipsefoundation.git.eca.model.SystemHook;
import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
import org.eclipsefoundation.git.eca.service.SystemHookService;
......
/*********************************************************************
* Copyright (c) 2022 Eclipse Foundation.
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* Author: Zachary Sabourin <zachary.sabourin@eclipse-foundation.org>
*
* SPDX-License-Identifier: EPL-2.0
**********************************************************************/
package org.eclipsefoundation.git.eca.tasks;
import java.net.URI;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.enterprise.context.ApplicationScoped;
import javax.enterprise.inject.Instance;
import javax.inject.Inject;
import javax.ws.rs.core.MultivaluedMap;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.core.model.FlatRequestWrapper;
import org.eclipsefoundation.core.model.RequestWrapper;
import org.eclipsefoundation.core.service.APIMiddleware;
import org.eclipsefoundation.core.service.CachingService;
import org.eclipsefoundation.git.eca.api.GitlabAPI;
import org.eclipsefoundation.git.eca.api.models.GitlabProjectResponse;
import org.eclipsefoundation.git.eca.dto.PrivateProjectEvent;
import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
import org.eclipsefoundation.persistence.dao.impl.DefaultHibernateDao;
import org.eclipsefoundation.persistence.model.RDBMSQuery;
import org.eclipsefoundation.persistence.service.FilterService;
import org.jboss.resteasy.specimpl.MultivaluedMapImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.quarkus.arc.Arc;
import io.quarkus.arc.InstanceHandle;
import io.quarkus.scheduler.Scheduled;
@ApplicationScoped
public class ScheduledPrivateProjectScanTask {
public static final Logger LOGGER = LoggerFactory.getLogger(ScheduledPrivateProjectScanTask.class);
@ConfigProperty(name = "eclipse.scheduled.private-project.enabled", defaultValue = "true")
Instance<Boolean> enabled;
@ConfigProperty(name = "eclipse.gitlab.access-token")
String apiToken;
@Inject
@RestClient
GitlabAPI api;
@Inject
CachingService cache;
@Scheduled(every = "P7D")
void reconcilePrivateProjects() {
if (Boolean.FALSE.equals(enabled.get())) {
LOGGER.warn("Private project scan task did not run. It has been disabled through configuration.");
} else {
InstanceHandle<APIMiddleware> middlewareHandle = Arc.container().instance(APIMiddleware.class);
APIMiddleware middleware = middlewareHandle.get();
// Fetch all private projects from GL, while paginating results
Optional<List<GitlabProjectResponse>> response = cache.get("all", new MultivaluedMapImpl<>(),
GitlabProjectResponse.class,
() -> middleware.getAll(p -> api.getPrivateProjects(p, apiToken, "private", 100),
GitlabProjectResponse.class));
response.ifPresent(this::processGitlabResponseList);
}
}
/**
* Processes the list of gitlab project response objects and updates the missed
* deletion, creation, and rename events in the DB
*
* @param projectList the list of private projects to process
*/
private void processGitlabResponseList(List<GitlabProjectResponse> projectList) {
InstanceHandle<DefaultHibernateDao> daoHandle = Arc.container().instance(DefaultHibernateDao.class);
DefaultHibernateDao dao = daoHandle.get();
InstanceHandle<FilterService> filtersHandle = Arc.container().instance(FilterService.class);
FilterService filters = filtersHandle.get();
updateMissedDeletionEvents(projectList, dao, filters);
updateMissedCreationAndRenameEvents(projectList, dao, filters);
}
/**
* Updates the DB records that aren't included in the Gitlab project list.
*
* @param projectList The list of private projects to process
* @param dao The dao service
* @param filters The filters service
*/
private void updateMissedDeletionEvents(List<GitlabProjectResponse> projectList, DefaultHibernateDao dao,
FilterService filters) {
RequestWrapper wrapper = new FlatRequestWrapper(URI.create("https://api.eclipse.org"));
// Ad ids to be reverse searched against
MultivaluedMap<String, String> excludingParams = new MultivaluedMapImpl<>();
excludingParams.put(GitEcaParameterNames.NOT_IN_PROJECT_IDS.getName(),
projectList.stream().map(p -> Integer.toString(p.getId())).collect(Collectors.toList()));
// Get all excluding the ids found on GL
List<PrivateProjectEvent> deletedResults = dao
.get(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class), excludingParams));
if (deletedResults != null) {
List<PrivateProjectEvent> dtos = deletedResults.stream()
.map(this::createDtoWithDeletionDate).collect(Collectors.toList());
dao.add(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class)), dtos);
}
}
/**
* Creates new DB records for each private project that was missed when created
* or renamed.
*
* @param projectList The list of private projects to process
* @param dao The dao service
* @param filters The filters service
*/
private void updateMissedCreationAndRenameEvents(List<GitlabProjectResponse> projectList, DefaultHibernateDao dao,
FilterService filters) {
RequestWrapper wrapper = new FlatRequestWrapper(URI.create("https://api.eclipse.org"));
MultivaluedMap<String, String> includingParams = new MultivaluedMapImpl<>();
includingParams.put(GitEcaParameterNames.PROJECT_IDS.getName(),
projectList.stream().map(p -> Integer.toString(p.getId())).collect(Collectors.toList()));
// Get by id since project ids never change
List<PrivateProjectEvent> results = dao
.get(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class), includingParams));
if (results != null) {
// Includes only projects with namespaces that don't exist or events not in DB
List<PrivateProjectEvent> dtos = projectList.stream()
.filter(p -> results.stream().noneMatch(r -> r.getCompositeId().getProjectPath()
.equalsIgnoreCase(p.getPathWithNamespace())))
.map(this::createDto)
.collect(Collectors.toList());
dao.add(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class)), dtos);
}
}
/**
* Takes a PrivateProjectEvent, copies it, updates the deletion date, and
* returns the copy.
*
* @param event The event to update
* @return a PrivateProjectEvent with an updated deletion date
*/
private PrivateProjectEvent createDtoWithDeletionDate(PrivateProjectEvent event) {
PrivateProjectEvent out = new PrivateProjectEvent(event.getCompositeId().getUserId(),
event.getCompositeId().getProjectId(), event.getCompositeId().getProjectPath());
out.setCreationDate(event.getCreationDate());
out.setDeletionDate(LocalDateTime.now());
out.setParentProject(event.getParentProject());
return out;
}
/**
* Takes the received GitlabProjectResponse object and maps it to a
* PrivateProjectEvent DTO.
*
* @param project received project data
* @return A PrivateProjectEvent object
*/
private PrivateProjectEvent createDto(GitlabProjectResponse project) {
PrivateProjectEvent dto = new PrivateProjectEvent(project.getCreatorId(), project.getId(),
project.getPathWithNamespace());
dto.setCreationDate(project.getCreatedAt().toLocalDateTime());
if (project.getForkedFromProject() != null) {
dto.setParentProject(project.getForkedFromProject().getId());
}
return dto;
}
}
......@@ -18,7 +18,7 @@ import java.util.Optional;
import javax.inject.Inject;
import org.eclipsefoundation.git.eca.model.EclipseUser;
import org.eclipsefoundation.git.eca.api.models.EclipseUser;
import org.eclipsefoundation.git.eca.model.Project;
import org.eclipsefoundation.git.eca.service.ProjectsService;
import org.eclipsefoundation.git.eca.service.UserService;
......
......@@ -20,8 +20,8 @@ import javax.enterprise.context.ApplicationScoped;
import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.git.eca.api.AccountsAPI;
import org.eclipsefoundation.git.eca.model.EclipseUser;
import org.eclipsefoundation.git.eca.model.EclipseUser.ECA;
import org.eclipsefoundation.git.eca.api.models.EclipseUser;
import org.eclipsefoundation.git.eca.api.models.EclipseUser.ECA;
import io.quarkus.test.Mock;
......
......@@ -13,14 +13,17 @@ package org.eclipsefoundation.git.eca.test.api;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import javax.enterprise.context.ApplicationScoped;
import javax.ws.rs.core.Response;
import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.core.service.APIMiddleware.BaseAPIParameters;
import org.eclipsefoundation.git.eca.api.GitlabAPI;
import org.eclipsefoundation.git.eca.model.GitlabProjectResponse;
import org.eclipsefoundation.git.eca.model.GitlabProjectResponse.ForkedProject;
import org.eclipsefoundation.git.eca.api.models.GitlabProjectResponse;
import org.eclipsefoundation.git.eca.api.models.GitlabProjectResponse.ForkedProject;
import io.quarkus.test.Mock;
......@@ -54,4 +57,9 @@ public class MockGitlabAPI implements GitlabAPI {
public GitlabProjectResponse getProjectInfo(String privateToken, int projectId) {
return projects.stream().filter(p -> p.getId() == projectId).findFirst().orElseGet(null);
}
@Override
public Response getPrivateProjects(BaseAPIParameters baseParams,String privateToken, String visibility, Integer perPage) {
return Response.ok().build();
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment