Merge branch 'zacharysabourin/master/7' into 'master'

feat: Use Link Headers to paginate

See merge request !97

.gitignore

@@ -44,4 +44,6 @@ secret.properties
 
 # Additional build resources
 src/test/resources/schemas
-.env
\ No newline at end of file
+.env
+/.apt_generated/
+/.apt_generated_tests/

Makefile

@@ -5,8 +5,8 @@ pre-setup:;
 setup:;
 	@echo "Generating secret files from templates using environment file + variables"
 	@source .env && rm -f ./config/application/secret.properties && envsubst < config/application/secret.properties.sample > config/application/secret.properties
-dev-start:;
-	source .env && mvn compile quarkus:dev
+dev-start: clean;
+	source .env && mvn compile -e quarkus:dev -Dconfig.secret.properties=$$PWD/config/application/secret.properties
 clean:;
 	mvn clean
 compile-java: generate-spec;
@@ -21,8 +21,8 @@ generate-spec: install-yarn validate-spec;
 	yarn run generate-json-schema
 validate-spec: install-yarn;
 compile-start: compile-quick;
-	docker-compose down
-	docker-compose build
-	docker-compose up -d
+	docker compose down
+	docker compose build
+	docker compose up -d
 start-spec: validate-spec;
 	yarn run start
\ No newline at end of file

config/.env.sample

-export GIT_ECA_MARIADB_USER=$MARIADB_USER
+export GIT_ECA_MARIADB_USER=$MARIADB_USERNAME
 export GIT_ECA_MARIADB_PASSWORD=$MARIADB_PASSWORD
 export GIT_ECA_MARIADB_HOST=$MARIADB_HOST
 export GIT_ECA_MARIADB_PORT=$MARIADB_PORT

pom.xml

@@ -5,7 +5,7 @@
   <artifactId>git-eca</artifactId>
   <version>0.0.1</version>
   <properties>
-		<eclipse-api-version>0.6.3-SNAPSHOT</eclipse-api-version>
+		<eclipse-api-version>0.6.5-SNAPSHOT</eclipse-api-version>
     <compiler-plugin.version>3.8.1</compiler-plugin.version>
     <maven.compiler.parameters>true</maven.compiler.parameters>
 		<maven.compiler.source>11</maven.compiler.source>

src/main/java/org/eclipsefoundation/git/eca/api/ProjectsAPI.java

@@ -9,17 +9,15 @@
  ******************************************************************************/
 package org.eclipsefoundation.git.eca.api;
 
-import java.util.List;
-
 import javax.enterprise.context.ApplicationScoped;
+import javax.ws.rs.BeanParam;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Response;
 
 import org.eclipse.microprofile.rest.client.inject.RegisterRestClient;
-import org.eclipsefoundation.git.eca.model.Project;
-
+import org.eclipsefoundation.core.service.APIMiddleware.BaseAPIParameters;
+import org.jboss.resteasy.annotations.GZIP;
 /**
  * Interface for interacting with the PMI Projects API. Used to link Git
  * repos/projects with an Eclipse project to validate committer access.
@@ -30,6 +28,7 @@ import org.eclipsefoundation.git.eca.model.Project;
 @ApplicationScoped
 @Path("/api/projects")
 @RegisterRestClient
+@GZIP
 public interface ProjectsAPI {
 
 	/**
@@ -39,6 +38,5 @@ public interface ProjectsAPI {
 	 * @return a list of Eclipse Foundation projects.
 	 */
 	@GET
-	@Produces("application/json")
-	List<Project> getProject(@QueryParam("page") int page, @QueryParam("pagesize") int pageSize);
+	Response getProjects(@BeanParam BaseAPIParameters baseParams);
 }

src/main/java/org/eclipsefoundation/git/eca/service/impl/PaginationProjectsService.java

+/**
+ * Copyright (C) 2020 Eclipse Foundation
+ *
+ * This program and the accompanying materials are made available under the terms of the Eclipse
+ * Public License 2.0 which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.git.eca.service.impl;
+
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import javax.annotation.PostConstruct;
+import javax.enterprise.context.ApplicationScoped;
+import javax.inject.Inject;
+
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+import org.eclipse.microprofile.context.ManagedExecutor;
+import org.eclipse.microprofile.rest.client.inject.RestClient;
+import org.eclipsefoundation.core.service.APIMiddleware;
+import org.eclipsefoundation.git.eca.api.ProjectsAPI;
+import org.eclipsefoundation.git.eca.model.Project;
+import org.eclipsefoundation.git.eca.service.ProjectsService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListenableFutureTask;
+
+import io.quarkus.runtime.Startup;
+
+/**
+ * Projects service implementation that handles pagination of data manually, as
+ * well as makes use of
+ * a loading cache to have data be always available with as little latency to
+ * the user as possible.
+ *
+ * @author Martin Lowe
+ */
+@Startup
+@ApplicationScoped
+public class PaginationProjectsService implements ProjectsService {
+    private static final Logger LOGGER = LoggerFactory.getLogger(PaginationProjectsService.class);
+
+    @Inject
+    ManagedExecutor exec;
+
+    @ConfigProperty(name = "cache.pagination.refresh-frequency-seconds", defaultValue = "3600")
+    long refreshAfterWrite;
+
+    @Inject
+    APIMiddleware middleware;
+
+    @Inject
+    @RestClient
+    ProjectsAPI projects;
+    // this class has a separate cache as this data is long to load and should be
+    // always available.
+    LoadingCache<String, List<Project>> internalCache;
+
+    /**
+     * Initializes the internal loader cache and pre-populates the data with the one
+     * available key. If
+     * more than one key is used, eviction of previous results will happen and
+     * create degraded
+     * performance.
+     */
+    @PostConstruct
+    public void init() {
+        // set up the internal cache
+        this.internalCache = CacheBuilder.newBuilder()
+                .maximumSize(1)
+                .refreshAfterWrite(refreshAfterWrite, TimeUnit.SECONDS)
+                .build(
+                        new CacheLoader<String, List<Project>>() {
+                            @Override
+                            public List<Project> load(String key) throws Exception {
+                                return getProjectsInternal();
+                            }
+
+                            /**
+                             * Implementation required for refreshAfterRewrite to be async rather than sync
+                             * and blocking while awaiting for expensive reload to complete.
+                             */
+                            @Override
+                            public ListenableFuture<List<Project>> reload(String key, List<Project> oldValue)
+                                    throws Exception {
+                                ListenableFutureTask<List<Project>> task = ListenableFutureTask.create(
+                                        () -> {
+                                            LOGGER.debug("Retrieving new project data async");
+                                            List<Project> newProjects = oldValue;
+                                            try {
+                                                newProjects = getProjectsInternal();
+                                            } catch (Exception e) {
+                                                LOGGER.error(
+                                                        "Error while reloading internal projects data, data will be stale for current cycle.",
+                                                        e);
+                                            }
+                                            LOGGER.debug("Done refreshing project values");
+                                            return newProjects;
+                                        });
+                                // run the task using the Quarkus managed executor
+                                exec.execute(task);
+                                return task;
+                            }
+                        });
+
+        // pre-cache the projects to reduce load time for other users
+        LOGGER.debug("Starting pre-cache of projects");
+        if (getProjects() == null) {
+            LOGGER.warn(
+                    "Unable to populate pre-cache for Eclipse projects. Calls may experience degraded performance.");
+        }
+        LOGGER.debug("Completed pre-cache of projects assets");
+    }
+
+    @Override
+    public List<Project> getProjects() {
+        try {
+            return internalCache.get("projects");
+        } catch (ExecutionException e) {
+            throw new RuntimeException("Could not load Eclipse projects", e);
+        }
+    }
+
+    /**
+     * Logic for retrieving projects from API.
+     *
+     * @return list of projects for the cache
+     */
+    private List<Project> getProjectsInternal() {
+
+        return middleware.getAll(params -> projects.getProjects(params), Project.class)
+                .stream()
+                .map(proj -> {
+                    proj.getGerritRepos()
+                            .forEach(repo -> {
+                                if (repo.getUrl().endsWith(".git")) {
+                                    repo.setUrl(repo.getUrl().substring(0, repo.getUrl().length() - 4));
+                                }
+                            });
+                    return proj;
+                })
+                .collect(Collectors.toList());
+    }
+}

src/main/java/org/eclipsefoundation/git/eca/service/impl/PagintationProjectsService.java

-/**
- * Copyright (C) 2020 Eclipse Foundation
- *
- * This program and the accompanying materials are made available under the terms of the Eclipse
- * Public License 2.0 which is available at https://www.eclipse.org/legal/epl-2.0/
- *
- * SPDX-License-Identifier: EPL-2.0
- */
-package org.eclipsefoundation.git.eca.service.impl;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import javax.annotation.PostConstruct;
-import javax.enterprise.context.ApplicationScoped;
-import javax.inject.Inject;
-
-import org.eclipse.microprofile.config.inject.ConfigProperty;
-import org.eclipse.microprofile.context.ManagedExecutor;
-import org.eclipse.microprofile.rest.client.inject.RestClient;
-import org.eclipsefoundation.git.eca.api.ProjectsAPI;
-import org.eclipsefoundation.git.eca.model.Project;
-import org.eclipsefoundation.git.eca.service.ProjectsService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListenableFutureTask;
-
-import io.quarkus.runtime.Startup;
-
-/**
- * Projects service implementation that handles pagination of data manually, as well as makes use of
- * a loading cache to have data be always available with as little latency to the user as possible.
- *
- * @author Martin Lowe
- */
-@Startup
-@ApplicationScoped
-public class PagintationProjectsService implements ProjectsService {
-  private static final Logger LOGGER = LoggerFactory.getLogger(PagintationProjectsService.class);
-
-  @Inject ManagedExecutor exec;
-
-  @ConfigProperty(name = "cache.pagination.refresh-frequency-seconds", defaultValue = "3600")
-  long refreshAfterWrite;
-
-  @Inject @RestClient ProjectsAPI projects;
-  // this class has a separate cache as this data is long to load and should be
-  // always available.
-  LoadingCache<String, List<Project>> internalCache;
-
-  /**
-   * Initializes the internal loader cache and pre-populates the data with the one available key. If
-   * more than one key is used, eviction of previous results will happen and create degraded
-   * performance.
-   */
-  @PostConstruct
-  public void init() {
-    // set up the internal cache
-    this.internalCache =
-        CacheBuilder.newBuilder()
-            .maximumSize(1)
-            .refreshAfterWrite(refreshAfterWrite, TimeUnit.SECONDS)
-            .build(
-                new CacheLoader<String, List<Project>>() {
-                  @Override
-                  public List<Project> load(String key) throws Exception {
-                    return getProjectsInternal();
-                  }
-
-                  /**
-                   * Implementation required for refreshAfterRewrite to be async rather than sync
-                   * and blocking while awaiting for expensive reload to complete.
-                   */
-                  @Override
-                  public ListenableFuture<List<Project>> reload(String key, List<Project> oldValue)
-                      throws Exception {
-                    ListenableFutureTask<List<Project>> task =
-                        ListenableFutureTask.create(
-                            () -> {
-                              LOGGER.debug("Retrieving new project data async");
-                              List<Project> newProjects = oldValue;
-                              try {
-                                newProjects = getProjectsInternal();
-                              } catch (Exception e) {
-                                LOGGER.error(
-                                    "Error while reloading internal projects data, data will be stale for current cycle.",
-                                    e);
-                              }
-                              LOGGER.debug("Done refreshing project values");
-                              return newProjects;
-                            });
-                    // run the task using the Quarkus managed executor
-                    exec.execute(task);
-                    return task;
-                  }
-                });
-
-    // pre-cache the projects to reduce load time for other users
-    LOGGER.debug("Starting pre-cache of projects");
-    if (getProjects() == null) {
-      LOGGER.warn(
-          "Unable to populate pre-cache for Eclipse projects. Calls may experience degraded performance.");
-    }
-    LOGGER.debug("Completed pre-cache of projects assets");
-  }
-
-  @Override
-  public List<Project> getProjects() {
-    try {
-      return internalCache.get("projects");
-    } catch (ExecutionException e) {
-      throw new RuntimeException("Could not load Eclipse projects", e);
-    }
-  }
-
-  /**
-   * Logic for retrieving projects from API. Will loop until there are no more projects to be found
-   *
-   * @return list of projects for the
-   */
-  private List<Project> getProjectsInternal() {
-    int page = 0;
-    int pageSize = 100;
-    List<Project> out = new LinkedList<>();
-    List<Project> in;
-    do {
-      page++;
-      in = projects.getProject(page, pageSize);
-      in.forEach(
-          proj ->
-              proj.getGerritRepos()
-                  .forEach(
-                      repo -> {
-                        if (repo.getUrl().endsWith(".git")) {
-                          repo.setUrl(repo.getUrl().substring(0, repo.getUrl().length() - 4));
-                        }
-                      }));
-      out.addAll(in);
-    } while (in != null && !in.isEmpty());
-    return out;
-  }
-}

src/main/resources/application.properties

@@ -18,6 +18,8 @@ quarkus.datasource.jdbc.max-size = 15
 quarkus.hibernate-orm.packages=org.eclipsefoundation.git.eca.dto
 quarkus.hibernate-orm.datasource=<default>
 
+quarkus.http.enable-compression=true
+
 quarkus.http.port=8080
 
 ## OAUTH CONFIG

src/test/java/org/eclipsefoundation/git/eca/test/api/MockProjectsAPI.java

@@ -18,8 +18,10 @@ import java.util.Map;
 
 import javax.annotation.PostConstruct;
 import javax.enterprise.context.ApplicationScoped;
+import javax.ws.rs.core.Response;
 
 import org.eclipse.microprofile.rest.client.inject.RestClient;
+import org.eclipsefoundation.core.service.APIMiddleware.BaseAPIParameters;
 import org.eclipsefoundation.git.eca.api.ProjectsAPI;
 import org.eclipsefoundation.git.eca.model.Project;
 import org.eclipsefoundation.git.eca.model.Project.GitlabProject;
@@ -86,7 +88,7 @@ public class MockProjectsAPI implements ProjectsAPI {
     }
 
     @Override
-    public List<Project> getProject(int page, int pageSize) {
-        return page == 1 ? new ArrayList<>(src) : Collections.emptyList();
+    public Response getProjects(BaseAPIParameters baseParams) {
+        return Response.ok(baseParams.getPage() == 1 ? new ArrayList<>(src) : Collections.emptyList()).build();
     }
 }