Skip to content
Snippets Groups Projects
Commit c4bcf571 authored by Martin Lowe's avatar Martin Lowe :flag_ca:
Browse files

Merge branch 'zacharysabourin/master/105' into 'master'

feat: Add new column to PrivateProjectEvent table for EF username

Closes #105

See merge request !119
parents 6e683cd0 4d940d07
No related branches found
No related tags found
1 merge request!119feat: Add new column to PrivateProjectEvent table for EF username
Pipeline #13631 passed
Showing
with 205 additions and 43 deletions
......@@ -30,6 +30,7 @@ CREATE TABLE IF NOT EXISTS `PrivateProjectEvent` (
`userId` int(10) NOT NULL,
`projectId` int(10) NOT NULL,
`projectPath` varchar(255) NOT NULL,
`ef_username` varchar(100) DEFAULT NULL,
`parentProject` int(10) DEFAULT NULL,
`creationDate` datetime NOT NULL,
`deletionDate` datetime DEFAULT NULL,
......
......@@ -395,6 +395,9 @@ components:
project_path:
type: string
description: the project's path with namespace
ef_username:
type: string
description: the user's EF account name
parent_project:
type:
- integer
......
......@@ -23,13 +23,13 @@ import javax.ws.rs.core.Response;
import org.eclipse.microprofile.rest.client.inject.RegisterRestClient;
import org.eclipsefoundation.core.service.APIMiddleware.BaseAPIParameters;
import org.eclipsefoundation.git.eca.api.models.GitlabProjectResponse;
import org.eclipsefoundation.git.eca.api.models.GitlabUserResponse;
/**
* Interface for interacting with the GitLab API. Used to fetch project data.
*/
@ApplicationScoped
@RegisterRestClient
@Path("/projects")
public interface GitlabAPI {
/**
......@@ -41,7 +41,7 @@ public interface GitlabAPI {
* @return A GitlabProjectResponse object
*/
@GET
@Path("/{id}")
@Path("/projects/{id}")
GitlabProjectResponse getProjectInfo(@HeaderParam("PRIVATE-TOKEN") String privateToken,
@PathParam("id") int projectId);
......@@ -56,6 +56,20 @@ public interface GitlabAPI {
* @return A Response containing a private project list
*/
@GET
Response getPrivateProjects(@BeanParam BaseAPIParameters baseParams, @HeaderParam("PRIVATE-TOKEN") String privateToken,
@QueryParam("visibility") String visibility, @QueryParam("per_page") Integer perPage);
@Path("/projects")
Response getPrivateProjects(@BeanParam BaseAPIParameters baseParams,
@HeaderParam("PRIVATE-TOKEN") String privateToken, @QueryParam("visibility") String visibility,
@QueryParam("per_page") Integer perPage);
/**
* Fetches data for a user using the userId. The id and a token of
* adequate permissions is required.
*
* @param privateToken the header token
* @param userId the project id
* @returnA A GitlabUserResponse object
*/
@GET
@Path("/users/{id}")
GitlabUserResponse getUserInfo(@HeaderParam("PRIVATE-TOKEN") String privateToken, @PathParam("id") int userId);
}
/*********************************************************************
* Copyright (c) 2022 Eclipse Foundation.
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* Author: Zachary Sabourin <zachary.sabourin@eclipse-foundation.org>
*
* SPDX-License-Identifier: EPL-2.0
**********************************************************************/
package org.eclipsefoundation.git.eca.api.models;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
import com.google.auto.value.AutoValue;
@AutoValue
@JsonDeserialize(builder = AutoValue_GitlabUserResponse.Builder.class)
public abstract class GitlabUserResponse {
public abstract Integer getId();
public abstract String getName();
public static Builder builder() {
return new AutoValue_GitlabUserResponse.Builder();
}
@AutoValue.Builder
@JsonPOJOBuilder(withPrefix = "set")
public abstract static class Builder {
public abstract Builder setId(Integer id);
public abstract Builder setName(String name);
public abstract GitlabUserResponse build();
}
}
......@@ -20,6 +20,7 @@ import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.persistence.Column;
import javax.persistence.Embeddable;
import javax.persistence.EmbeddedId;
import javax.persistence.Entity;
......@@ -41,6 +42,8 @@ public class PrivateProjectEvent extends BareNode {
@EmbeddedId
private EventCompositeId compositeId;
@Column(name = "ef_username")
private String efUsername;
private Integer parentProject;
private LocalDateTime creationDate;
private LocalDateTime deletionDate;
......@@ -69,6 +72,14 @@ public class PrivateProjectEvent extends BareNode {
this.compositeId = compositeId;
}
public String getEFUsername() {
return this.efUsername;
}
public void setEFUsername(String efUsername) {
this.efUsername = efUsername;
}
public Integer getParentProject() {
return this.parentProject;
}
......@@ -135,6 +146,8 @@ public class PrivateProjectEvent extends BareNode {
builder.append(getCompositeId().getProjectId());
builder.append(", projectPath=");
builder.append(getCompositeId().getProjectPath());
builder.append(", ef_username=");
builder.append(getEFUsername());
builder.append(", parentProject=");
builder.append(getParentProject());
builder.append(", creationDate=");
......
......@@ -15,6 +15,7 @@ import java.time.LocalDateTime;
import javax.annotation.Nullable;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
import com.google.auto.value.AutoValue;
......@@ -29,6 +30,9 @@ public abstract class PrivateProjectData {
public abstract String getProjectPath();
@JsonProperty("ef_username")
public abstract String getEFUsername();
@Nullable
public abstract Integer getParentProject();
......@@ -51,6 +55,8 @@ public abstract class PrivateProjectData {
public abstract Builder setProjectPath(String path);
public abstract Builder setEFUsername(String username);
public abstract Builder setParentProject(@Nullable Integer id);
public abstract Builder setCreationDate(LocalDateTime time);
......
......@@ -15,19 +15,25 @@ import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.enterprise.context.ApplicationScoped;
import javax.enterprise.context.control.ActivateRequestContext;
import javax.inject.Inject;
import javax.transaction.Transactional;
import javax.ws.rs.core.MultivaluedMap;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.eclipse.microprofile.context.ManagedExecutor;
import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.eclipsefoundation.core.model.RequestWrapper;
import org.eclipsefoundation.core.service.CachingService;
import org.eclipsefoundation.git.eca.api.GitlabAPI;
import org.eclipsefoundation.git.eca.api.models.GitlabProjectResponse;
import org.eclipsefoundation.git.eca.api.models.GitlabUserResponse;
import org.eclipsefoundation.git.eca.dto.PrivateProjectEvent;
import org.eclipsefoundation.git.eca.model.SystemHook;
import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
......@@ -40,12 +46,16 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ApplicationScoped
@ActivateRequestContext
public class DefaultSystemHookService implements SystemHookService {
private static final Logger LOGGER = LoggerFactory.getLogger(DefaultSystemHookService.class);
@ConfigProperty(name = "eclipse.gitlab.access-token")
String apiToken;
@ConfigProperty(name = "eclipse.system-hook.pool-size")
Integer poolSize;
@Inject
@RestClient
GitlabAPI api;
......@@ -59,140 +69,176 @@ public class DefaultSystemHookService implements SystemHookService {
@Inject
FilterService filters;
@Inject
ManagedExecutor managedExecutor;
ScheduledExecutorService ses;
@PostConstruct
public void init() {
ses = Executors.newScheduledThreadPool(poolSize);
}
@PreDestroy
public void destroy() {
ses.shutdownNow();
}
@Override
public void processProjectCreateHook(RequestWrapper wrapper, SystemHook hook) {
if (hook != null) {
managedExecutor.execute(() -> trackPrivateProjectCreation(wrapper, hook));
ses.schedule(() -> trackPrivateProjectCreation(wrapper, hook), 300, TimeUnit.MILLISECONDS);
}
}
@Override
public void processProjectDeleteHook(RequestWrapper wrapper, SystemHook hook) {
if (hook != null) {
managedExecutor.execute(() -> trackPrivateProjectDeletion(wrapper, hook));
ses.execute(() -> trackPrivateProjectDeletion(wrapper, hook));
}
}
@Override
public void processProjectRenameHook(RequestWrapper wrapper, SystemHook hook) {
if (hook != null) {
managedExecutor.execute(() -> trackPrivateProjectRenaming(wrapper, hook));
ses.schedule(() -> trackPrivateProjectRenaming(wrapper, hook), 300, TimeUnit.MILLISECONDS);
}
}
/**
* Gathers all relevant data related to the created project and persists the information into a database
* Gathers all relevant data related to the created project and persists the
* information into a database
*
* @param wrapper the request wrapper containing all uri, ip, and query params
* @param hook the incoming system hook
* @param hook the incoming system hook
*/
@Transactional
void trackPrivateProjectCreation(RequestWrapper wrapper, SystemHook hook) {
try {
LOGGER.debug("Tracking creation of project: [id: {}, path: {}]", hook.getProjectId(), hook.getPathWithNamespace());
LOGGER.debug("Tracking creation of project: [id: {}, path: {}]", hook.getProjectId(),
hook.getPathWithNamespace());
Optional<GitlabProjectResponse> response = cache
.get(Integer.toString(hook.getProjectId()), new MultivaluedMapImpl<>(), GitlabProjectResponse.class,
() -> api.getProjectInfo(apiToken, hook.getProjectId()));
Optional<GitlabProjectResponse> response = cache.get(Integer.toString(hook.getProjectId()),
new MultivaluedMapImpl<>(), GitlabProjectResponse.class,
() -> api.getProjectInfo(apiToken, hook.getProjectId()));
if (response.isPresent()) {
PrivateProjectEvent dto = mapToDto(hook, response.get());
dao.add(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class)), Arrays.asList(dto));
} else {
LOGGER.error("No info for project: [id: {}, path: {}]", hook.getProjectId(), hook.getPathWithNamespace());
LOGGER.error("No info for project: [id: {}, path: {}]", hook.getProjectId(),
hook.getPathWithNamespace());
}
} catch (Exception e) {
LOGGER.error("Error fetching data relevant project data from GL for project: {}", hook.getProjectId());
LOGGER.error("Error fetching data relevant project data from GL for project: {}", hook.getProjectId(), e);
}
}
/**
* Retrieves the event record for the project that has been deleted. Adds the deletionDate field and updates the DB
* Retrieves the event record for the project that has been deleted. Adds the
* deletionDate field and updates the DB
* record.
*
* @param wrapper the request wrapper containing all uri, ip, and query params
* @param hook the incoming system hook
* @param hook the incoming system hook
*/
@Transactional
void trackPrivateProjectDeletion(RequestWrapper wrapper, SystemHook hook) {
try {
LOGGER.debug("Tracking deletion of project: [id: {}, path: {}]", hook.getProjectId(), hook.getPathWithNamespace());
LOGGER.debug("Tracking deletion of project: [id: {}, path: {}]", hook.getProjectId(),
hook.getPathWithNamespace());
MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
params.add(GitEcaParameterNames.PROJECT_ID.getName(), Integer.toString(hook.getProjectId()));
params.add(GitEcaParameterNames.PROJECT_PATH.getName(), hook.getPathWithNamespace());
List<PrivateProjectEvent> results = dao.get(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class), params));
List<PrivateProjectEvent> results = dao
.get(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class), params));
if (results.isEmpty()) {
LOGGER.error("No results for project event: [id: {}, path: {}]", hook.getProjectId(), hook.getPathWithNamespace());
LOGGER.error("No results for project event: [id: {}, path: {}]", hook.getProjectId(),
hook.getPathWithNamespace());
} else {
results.get(0).setDeletionDate(hook.getUpdatedAt().toLocalDateTime());
dao.add(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class)), results);
}
} catch (Exception e) {
LOGGER.error(String.format("Error updating project: [id: %s, path: %s]", hook.getProjectId(), hook.getPathWithNamespace()), e);
LOGGER.error(String.format("Error updating project: [id: %s, path: %s]", hook.getProjectId(),
hook.getPathWithNamespace()), e);
}
}
/**
* Retrieves the event record for the project that has been renamed. Updates the projectPath field and creates a new DB
* Retrieves the event record for the project that has been renamed. Updates the
* projectPath field and creates a new DB
* record.
*
* @param wrapper the request wrapper containing all uri, ip, and query params
* @param hook the incoming system hook
* @param hook the incoming system hook
*/
@Transactional
void trackPrivateProjectRenaming(RequestWrapper wrapper, SystemHook hook) {
try {
LOGGER.debug("Tracking renaming of project: [id: {}, path: {}]", hook.getProjectId(), hook.getOldPathWithNamespace());
LOGGER.debug("Tracking renaming of project: [id: {}, path: {}]", hook.getProjectId(),
hook.getOldPathWithNamespace());
MultivaluedMap<String, String> params = new MultivaluedMapImpl<>();
params.add(GitEcaParameterNames.PROJECT_ID.getName(), Integer.toString(hook.getProjectId()));
params.add(GitEcaParameterNames.PROJECT_PATH.getName(), hook.getOldPathWithNamespace());
List<PrivateProjectEvent> results = dao.get(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class), params));
List<PrivateProjectEvent> results = dao
.get(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class), params));
if (results.isEmpty()) {
LOGGER.error("No results for project event: [id: {}, path: {}]", hook.getProjectId(), hook.getOldPathWithNamespace());
LOGGER.error("No results for project event: [id: {}, path: {}]", hook.getProjectId(),
hook.getOldPathWithNamespace());
} else {
// Create a new event and track in the DB
PrivateProjectEvent event = new PrivateProjectEvent(results.get(0).getCompositeId().getUserId(),
results.get(0).getCompositeId().getProjectId(), hook.getPathWithNamespace());
event.setEFUsername(fetchUserName(event.getCompositeId().getUserId()));
event.setCreationDate(LocalDateTime.now());
event.setParentProject(results.get(0).getParentProject());
dao.add(new RDBMSQuery<>(wrapper, filters.get(PrivateProjectEvent.class)), Arrays.asList(event));
}
} catch (Exception e) {
LOGGER
.error(String.format("Error updating project: [id: %s, path: %s]", hook.getProjectId(), hook.getOldPathWithNamespace()),
e);
LOGGER.error(String.format("Error updating project: [id: %s, path: %s]", hook.getProjectId(),
hook.getOldPathWithNamespace()), e);
}
}
/**
* Takes the received hook and Gitlab api response body and maps the values to a PrivateProjectEvent dto.
* Takes the received hook and Gitlab api response body and maps the values to a
* PrivateProjectEvent dto.
*
* @param hookthe received system hook
* @param gitlabInfo The gitlab project data
* @param hookthe received system hook
* @param projectInfo The gitlab project data
* @return A PrivateProjectEvent object
*/
private PrivateProjectEvent mapToDto(SystemHook hook, GitlabProjectResponse gitlabInfo) {
private PrivateProjectEvent mapToDto(SystemHook hook, GitlabProjectResponse projectInfo) {
PrivateProjectEvent eventDto = new PrivateProjectEvent(gitlabInfo.getCreatorId(), hook.getProjectId(), hook.getPathWithNamespace());
PrivateProjectEvent eventDto = new PrivateProjectEvent(projectInfo.getCreatorId(), hook.getProjectId(),
hook.getPathWithNamespace());
eventDto.setEFUsername(fetchUserName(projectInfo.getCreatorId()));
eventDto.setCreationDate(hook.getCreatedAt().toLocalDateTime());
if (gitlabInfo.getForkedFromProject() != null) {
eventDto.setParentProject(gitlabInfo.getForkedFromProject().getId());
if (projectInfo.getForkedFromProject() != null) {
eventDto.setParentProject(projectInfo.getForkedFromProject().getId());
}
return eventDto;
}
/**
* fetches a user's name using the given user ID.
*
* @param userId the desried user's id
* @return the username or null
*/
private String fetchUserName(Integer userId) {
Optional<GitlabUserResponse> response = cache.get(Integer.toString(userId), new MultivaluedMapImpl<>(),
GitlabUserResponse.class, () -> api.getUserInfo(apiToken, userId));
return response.isPresent() ? response.get().getName().replace("\\s+", "").toLowerCase() : null;
}
}
......@@ -30,6 +30,7 @@ import org.eclipsefoundation.core.service.APIMiddleware;
import org.eclipsefoundation.core.service.CachingService;
import org.eclipsefoundation.git.eca.api.GitlabAPI;
import org.eclipsefoundation.git.eca.api.models.GitlabProjectResponse;
import org.eclipsefoundation.git.eca.api.models.GitlabUserResponse;
import org.eclipsefoundation.git.eca.dto.PrivateProjectEvent;
import org.eclipsefoundation.git.eca.namespace.GitEcaParameterNames;
import org.eclipsefoundation.persistence.dao.impl.DefaultHibernateDao;
......@@ -167,6 +168,7 @@ public class ScheduledPrivateProjectScanTask {
PrivateProjectEvent out = new PrivateProjectEvent(event.getCompositeId().getUserId(),
event.getCompositeId().getProjectId(), event.getCompositeId().getProjectPath());
out.setEFUsername(fetchUserName(event.getCompositeId().getUserId()));
out.setCreationDate(event.getCreationDate());
out.setDeletionDate(LocalDateTime.now());
out.setParentProject(event.getParentProject());
......@@ -184,6 +186,7 @@ public class ScheduledPrivateProjectScanTask {
PrivateProjectEvent dto = new PrivateProjectEvent(project.getCreatorId(), project.getId(),
project.getPathWithNamespace());
dto.setEFUsername(fetchUserName(project.getCreatorId()));
dto.setCreationDate(project.getCreatedAt().toLocalDateTime());
if (project.getForkedFromProject() != null) {
......@@ -192,4 +195,16 @@ public class ScheduledPrivateProjectScanTask {
return dto;
}
/**
* fetches a user's name using the given user ID.
*
* @param userId the desried user's id
* @return the username or null
*/
private String fetchUserName(Integer userId) {
Optional<GitlabUserResponse> response = cache.get(Integer.toString(userId), new MultivaluedMapImpl<>(),
GitlabUserResponse.class, () -> api.getUserInfo(apiToken, userId));
return response.isPresent() ? response.get().getName().replace("\\s+", "").toLowerCase() : null;
}
}
......@@ -41,5 +41,7 @@ eclipse.mail.allowlist=noreply@github.com,49699333+dependabot[bot]@users.noreply
%dev.eclipse.optional-resources.enabled=true
%dev.eclipse.scheduled.private-project.enabled=false
eclipse.system-hook.pool-size=5
#%dev.quarkus.log.level=DEBUG
#quarkus.log.category."org.eclipsefoundation".level=DEBUG
\ No newline at end of file
......@@ -13,7 +13,6 @@ package org.eclipsefoundation.git.eca.test.api;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import javax.enterprise.context.ApplicationScoped;
......@@ -24,6 +23,7 @@ import org.eclipsefoundation.core.service.APIMiddleware.BaseAPIParameters;
import org.eclipsefoundation.git.eca.api.GitlabAPI;
import org.eclipsefoundation.git.eca.api.models.GitlabProjectResponse;
import org.eclipsefoundation.git.eca.api.models.GitlabProjectResponse.ForkedProject;
import org.eclipsefoundation.git.eca.api.models.GitlabUserResponse;
import io.quarkus.test.Mock;
......@@ -33,6 +33,7 @@ import io.quarkus.test.Mock;
public class MockGitlabAPI implements GitlabAPI {
private List<GitlabProjectResponse> projects;
private List<GitlabUserResponse> users;
public MockGitlabAPI() {
projects = new ArrayList<>();
......@@ -51,6 +52,21 @@ public class MockGitlabAPI implements GitlabAPI {
.setCreatorId(33)
.setForkedFromProject(ForkedProject.builder().setId(69).build())
.build()));
users = new ArrayList<>();
users.addAll(Arrays.asList(
GitlabUserResponse.builder()
.setId(1)
.setName("admin")
.build(),
GitlabUserResponse.builder()
.setId(55)
.setName("testaccount")
.build(),
GitlabUserResponse.builder()
.setId(33)
.setName("fakeuser")
.build()));
}
@Override
......@@ -59,7 +75,13 @@ public class MockGitlabAPI implements GitlabAPI {
}
@Override
public Response getPrivateProjects(BaseAPIParameters baseParams,String privateToken, String visibility, Integer perPage) {
public Response getPrivateProjects(BaseAPIParameters baseParams, String privateToken, String visibility,
Integer perPage) {
return Response.ok().build();
}
@Override
public GitlabUserResponse getUserInfo(String privateToken, int userId) {
return users.stream().filter(p -> p.getId() == userId).findFirst().orElseGet(null);
}
}
......@@ -37,10 +37,11 @@ CREATE TABLE PrivateProjectEvent (
userId int NOT NULL,
projectId int NOT NULL,
projectPath varchar(255) NOT NULL,
ef_username varchar(100) DEFAULT NULL,
parentProject int DEFAULT NULL,
creationDate datetime NOT NULL,
deletionDate datetime DEFAULT NULL,
PRIMARY KEY (userId, projectId, projectPath)
);
INSERT INTO PrivateProjectEvent(userId, projectId, projectPath, creationDate) VALUES(1, 133, 'eclipse/test/project', '2022-11-11T12:00:00.000');
INSERT INTO PrivateProjectEvent(userId, projectId, projectPath, parentProject, creationDate, deletionDate) VALUES(1, 150, 'eclipse/test/project/fork', 133, '2022-11-15T12:00:00.000', '2022-11-20T12:00:00.000');
\ No newline at end of file
INSERT INTO PrivateProjectEvent(userId, projectId, projectPath, ef_username, creationDate) VALUES(1, 133, 'eclipse/test/project', 'testtesterson', '2022-11-11T12:00:00.000');
INSERT INTO PrivateProjectEvent(userId, projectId, projectPath, ef_username, parentProject, creationDate, deletionDate) VALUES(1, 150, 'eclipse/test/project/fork', 'tyronetesty', 133, '2022-11-15T12:00:00.000', '2022-11-20T12:00:00.000');
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment