Merge branch 'malowe/main/spam-report' into 'main'

Add additional metrics to the spam report to identify bad users

See merge request !3

src/main/java/org/eclipsefoundation/reports/api/AccountsAPI.java

+/**
+ * Copyright (c) 2023 Eclipse Foundation
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * Author: Martin Lowe <martin.lowe@eclipse-foundation.org>
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.reports.api;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+
+import org.eclipse.microprofile.rest.client.inject.RegisterRestClient;
+import org.eclipsefoundation.reports.api.models.EclipseForumResponse;
+import org.eclipsefoundation.reports.api.models.EclipseMailingListsResponse;
+
+/**
+ * Connects to the Eclipse Foundation metadata links for users to lookup whether users have interacted with common
+ * services, such as the forum and mailing list subscriptions.
+ */
+@RegisterRestClient(baseUri = "https://api.eclipse.org/account/profile")
+public interface AccountsAPI {
+
+    /**
+     * Lookup forum post information for the given user.
+     * 
+     * @param user username of individual to lookup
+     * @return list of forum posts for the passed user if they exist
+     */
+    @GET
+    @Path("{user}/forum")
+    EclipseForumResponse getForumPostsForUser(@PathParam("user") String user);
+
+
+    /**
+     * Lookup mailing list subscription information for the given user.
+     * 
+     * @param user username of individual to lookup
+     * @return list of mailing list subscriptions for the passed user if they exist
+     */
+    @GET
+    @Path("{user}/mailing-list")
+    EclipseMailingListsResponse getMailingListForUser(@PathParam("user") String user);
+}

src/main/java/org/eclipsefoundation/reports/api/MediaWikiAPI.java

+/**
+ * Copyright (c) 2023 Eclipse Foundation
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * Author: Martin Lowe <martin.lowe@eclipse-foundation.org>
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.reports.api;
+
+import javax.ws.rs.BeanParam;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
+
+import org.eclipse.microprofile.rest.client.inject.RegisterRestClient;
+import org.eclipsefoundation.reports.api.models.MediaWikiQueryResponse;
+
+/**
+ * Client for interacting with the Eclipse mediawiki to look up whether users have mad contributions in the wiki as an
+ * active user metric.
+ */
+@RegisterRestClient(baseUri = "https://wiki.eclipse.org")
+public interface MediaWikiAPI {
+
+    /**
+     * Query the mediawiki instance to look up user contributions as defined in the param object.
+     * 
+     * @param param wrapped parameters used to query user contributions
+     * @return the response from mediawiki for the query.
+     */
+    @GET
+    @Path("api.php")
+    MediaWikiQueryResponse query(@BeanParam MWQuery param);
+
+    /**
+     * Parameter wrapper for mediawiki queries, allowing us to look up specific user stats for requested actions.
+     */
+    public class MWQuery {
+        @QueryParam("ucuser")
+        public final String user;
+        @QueryParam("action")
+        public final String action;
+        @QueryParam("list")
+        public final String list;
+        @QueryParam("format")
+        public final String format;
+
+        public MWQuery(String user) {
+            this.user = user;
+            this.action = "query";
+            this.list = "usercontribs";
+            this.format = "json";
+        }
+    }
+}

src/main/java/org/eclipsefoundation/reports/api/models/EclipseForumResponse.java

+/**
+ * Copyright (c) 2023 Eclipse Foundation
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * Author: Martin Lowe <martin.lowe@eclipse-foundation.org>
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.reports.api.models;
+
+import java.util.List;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
+import com.google.auto.value.AutoValue;
+
+@AutoValue
+@JsonDeserialize(builder = AutoValue_EclipseForumResponse.Builder.class)
+public abstract class EclipseForumResponse {
+
+    public abstract String getPostedMsgCount();
+
+    public abstract List<ForumPost> getPosts();
+
+    public abstract Builder toBuilder();
+
+    public static Builder builder() {
+        return new AutoValue_EclipseForumResponse.Builder();
+    }
+
+    @AutoValue.Builder
+    @JsonPOJOBuilder(withPrefix = "set")
+    public abstract static class Builder {
+
+        public abstract Builder setPostedMsgCount(String query);
+
+        public abstract Builder setPosts(List<ForumPost> posts);
+
+        public abstract EclipseForumResponse build();
+    }
+
+    @AutoValue
+    @JsonDeserialize(builder = AutoValue_EclipseForumResponse_ForumPost.Builder.class)
+    public abstract static class ForumPost {
+
+        public abstract String getRootMsgSubject();
+
+        public static Builder builder() {
+            return new AutoValue_EclipseForumResponse_ForumPost.Builder();
+        }
+
+        @AutoValue.Builder
+        @JsonPOJOBuilder(withPrefix = "set")
+        public abstract static class Builder {
+
+            public abstract Builder setRootMsgSubject(String rootMsgSubject);
+
+            public abstract ForumPost build();
+        }
+    }
+}
\ No newline at end of file

src/main/java/org/eclipsefoundation/reports/api/models/EclipseMailingListsResponse.java

+/**
+ * Copyright (c) 2023 Eclipse Foundation
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * Author: Martin Lowe <martin.lowe@eclipse-foundation.org>
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipsefoundation.reports.api.models;
+
+import java.util.List;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
+import com.google.auto.value.AutoValue;
+
+@AutoValue
+@JsonDeserialize(builder = AutoValue_EclipseMailingListsResponse.Builder.class)
+public abstract class EclipseMailingListsResponse {
+
+    public abstract List<MailingListSubscription> getMailingListSubscriptions();
+
+    public abstract Builder toBuilder();
+
+    public static Builder builder() {
+        return new AutoValue_EclipseMailingListsResponse.Builder();
+    }
+
+    @AutoValue.Builder
+    @JsonPOJOBuilder(withPrefix = "set")
+    public abstract static class Builder {
+
+        public abstract Builder setMailingListSubscriptions(List<MailingListSubscription> mailingListsSubscriptions);
+
+        public abstract EclipseMailingListsResponse build();
+    }
+
+    @AutoValue
+    @JsonDeserialize(builder = AutoValue_EclipseMailingListsResponse_MailingListSubscription.Builder.class)
+    public abstract static class MailingListSubscription {
+
+        public abstract String getListName();
+
+        public static Builder builder() {
+            return new AutoValue_EclipseMailingListsResponse_MailingListSubscription.Builder();
+        }
+
+        @AutoValue.Builder
+        @JsonPOJOBuilder(withPrefix = "set")
+        public abstract static class Builder {
+
+            public abstract Builder setListName(String listName);
+
+            public abstract MailingListSubscription build();
+        }
+    }
+}
\ No newline at end of file

src/main/java/org/eclipsefoundation/reports/api/models/MediaWikiQueryResponse.java

+/*********************************************************************
+* Copyright (c) 2023 Eclipse Foundation.
+*
+* This program and the accompanying materials are made
+* available under the terms of the Eclipse Public License 2.0
+* which is available at https://www.eclipse.org/legal/epl-2.0/
+*
+ * Author: Martin Lowe <martin.lowe@eclipse-foundation.org>
+*
+* SPDX-License-Identifier: EPL-2.0
+**********************************************************************/
+package org.eclipsefoundation.reports.api.models;
+
+import java.time.ZonedDateTime;
+import java.util.List;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
+import com.google.auto.value.AutoValue;
+
+/**
+ * Entity representing a Working Group committee
+ */
+@AutoValue
+@JsonDeserialize(builder = AutoValue_MediaWikiQueryResponse.Builder.class)
+public abstract class MediaWikiQueryResponse {
+
+    public abstract QueryOutput getQuery();
+
+    public abstract Builder toBuilder();
+
+    public static Builder builder() {
+        return new AutoValue_MediaWikiQueryResponse.Builder();
+    }
+
+    @AutoValue.Builder
+    @JsonPOJOBuilder(withPrefix = "set")
+    public abstract static class Builder {
+
+        public abstract Builder setQuery(QueryOutput query);
+
+        public abstract MediaWikiQueryResponse build();
+    }
+
+    @AutoValue
+    @JsonDeserialize(builder = AutoValue_MediaWikiQueryResponse_QueryOutput.Builder.class)
+    public abstract static class QueryOutput {
+
+        public abstract List<Usercontrib> getUsercontribs();
+
+        public static Builder builder() {
+            return new AutoValue_MediaWikiQueryResponse_QueryOutput.Builder();
+        }
+
+        @AutoValue.Builder
+        @JsonPOJOBuilder(withPrefix = "set")
+        public abstract static class Builder {
+
+            public abstract Builder setUsercontribs(List<Usercontrib> usercontribs);
+
+            public abstract QueryOutput build();
+        }
+    }
+
+    @AutoValue
+    @JsonDeserialize(builder = AutoValue_MediaWikiQueryResponse_Usercontrib.Builder.class)
+    public abstract static class Usercontrib {
+
+        public abstract String getUser();
+
+        public abstract ZonedDateTime getTimestamp();
+
+        public static Builder builder() {
+            return new AutoValue_MediaWikiQueryResponse_Usercontrib.Builder();
+        }
+
+        @AutoValue.Builder
+        @JsonPOJOBuilder(withPrefix = "set")
+        public abstract static class Builder {
+
+            public abstract Builder setTimestamp(ZonedDateTime timestamp);
+
+            public abstract Builder setUser(String user);
+
+            public abstract Usercontrib build();
+        }
+    }
+}

src/main/java/org/eclipsefoundation/reports/cli/SpamLDAPReport.java

@@ -33,7 +33,10 @@ import javax.inject.Inject;
 
 import org.apache.commons.lang3.StringUtils;
 import org.eclipse.microprofile.rest.client.inject.RestClient;
+import org.eclipsefoundation.reports.api.AccountsAPI;
 import org.eclipsefoundation.reports.api.DisposableDomainGithubProjectRaw;
+import org.eclipsefoundation.reports.api.MediaWikiAPI;
+import org.eclipsefoundation.reports.api.MediaWikiAPI.MWQuery;
 import org.eclipsefoundation.reports.config.LdapCLIOptions;
 import org.eclipsefoundation.reports.dtos.EvtLog;
 import org.eclipsefoundation.reports.helper.LDAPHelper;
@@ -66,8 +69,7 @@ import picocli.CommandLine;
 public class SpamLDAPReport implements Runnable {
     public static final Logger LOGGER = LoggerFactory.getLogger(SpamLDAPReport.class);
 
-    private static final DateTimeFormatter CREATE_TIMESTAMP_FORMATTER = DateTimeFormatter
-            .ofPattern("yyyyMMddHHmmss'Z'");
+    private static final DateTimeFormatter CREATE_TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMddHHmmss'Z'");
 
     // add common options to the command
     @CommandLine.Mixin
@@ -75,6 +77,10 @@ public class SpamLDAPReport implements Runnable {
 
     @RestClient
     DisposableDomainGithubProjectRaw gh;
+    @RestClient
+    MediaWikiAPI mediaWiki;
+    @RestClient
+    AccountsAPI accounts;
     @Inject
     ObjectMapper om;
 
@@ -105,9 +111,7 @@ public class SpamLDAPReport implements Runnable {
             results = IntStream.range(0, entries.size()).mapToObj(e -> {
                 // give an indicator every 5% roughly
                 if (e % fivePercentCount == 0) {
-                    LOGGER
-                            .info("{}% of report post-processing complete",
-                                    (int) Math.ceil(((float) e) / entries.size() * 100));
+                    LOGGER.info("{}% of report post-processing complete", (int) Math.ceil(((float) e) / entries.size() * 100));
                 }
                 // convert LDAP entry to the report entry and return it
                 return convertLdapRecord(entries.get(e));
@@ -128,8 +132,7 @@ public class SpamLDAPReport implements Runnable {
     }
 
     /**
-     * Using the common LDAP helper, retrieve the disposable domain and do a search for any account that uses that
-     * domain.
+     * Using the common LDAP helper, retrieve the disposable domain and do a search for any account that uses that domain.
      * 
      * @param idx the current index of the domain, used for lookup and logging
      * @param value the list of known disposable domains
@@ -146,8 +149,7 @@ public class SpamLDAPReport implements Runnable {
         try {
             // do an LDAP query with the spam domain, retrieving additional properties used in generating the report
             return ldap
-                    .search(Map.of("mail", "*" + d), Optional.empty(),
-                            Optional.of(Arrays.asList("uid", "mail", "createTimestamp")))
+                    .search(Map.of("mail", "*" + d), Optional.empty(), Optional.of(Arrays.asList("uid", "mail", "createTimestamp")))
                     .stream();
         } catch (LDAPException e) {
             LOGGER.error("Error while retrieving disposable accounts for domain {}", d);
@@ -157,8 +159,7 @@ public class SpamLDAPReport implements Runnable {
 
     /**
      * Using a basic fetch call, retrieves raw stringified domain list from a Github project that maintains a list of
-     * disposable email domains. This list is then serialized into a list of domains to be used in downstream LDAP
-     * queries.
+     * disposable email domains. This list is then serialized into a list of domains to be used in downstream LDAP queries.
      * 
      * @return list of known disposable email domains
      */
@@ -173,8 +174,10 @@ public class SpamLDAPReport implements Runnable {
     }
 
     /**
-     * Does conversion from raw LDAP entry to a report entry, using a DB binding to check for login events for the user
-     * for last login time, and splitting the DN to get the groups the user is a part of.
+     * Does conversion from raw LDAP entry to a report entry, using a DB binding to check for login events for the user for
+     * last login time, and splitting the DN to get the groups the user is a part of.
+     * 
+     * Will additionally reach out to the Eclipse Mediawiki to check for contributions, as well as
      * 
      * @param e the LDAP result to convert to a report result entry
      * @return the report entry for the LDAP entry
@@ -184,6 +187,7 @@ public class SpamLDAPReport implements Runnable {
         String uid = e.getAttributeValue("uid");
         // lookup the user in the DB to find last session time
         EvtLog lastSession = EvtLog.findLastLoginSession(uid);
+        String rawPostedMsgCount = accounts.getForumPostsForUser(uid).getPostedMsgCount();
         return ReportEntry
                 .builder()
                 .setEmail(e.getAttributeValue("mail"))
@@ -198,6 +202,9 @@ public class SpamLDAPReport implements Runnable {
                 .setCreationDate(StringUtils.isBlank(creationValue) ? null
                         : LocalDateTime.parse(creationValue, CREATE_TIMESTAMP_FORMATTER).atZone(ZoneId.of("UTC")))
                 .setLastActive(lastSession == null ? null : lastSession.getEvtDateTime().atZone(ZoneId.of("UTC")))
+                .setHasWikiContributions(!mediaWiki.query(new MWQuery(uid)).getQuery().getUsercontribs().isEmpty())
+                .setForumPostCount(StringUtils.isNumeric(rawPostedMsgCount) ? Integer.valueOf(rawPostedMsgCount) : 0)
+                .setMailingListSubscriptionCount(accounts.getMailingListForUser(uid).getMailingListSubscriptions().size())
                 .build();
     }
 
@@ -208,7 +215,8 @@ public class SpamLDAPReport implements Runnable {
      *
      */
     @AutoValue
-    @JsonPropertyOrder({ "id", "email", "groups", "creationDate", "lastActive" })
+    @JsonPropertyOrder({ "id", "email", "groups", "creationDate", "lastActive", "hasWikiContributions", "forumPostCount",
+            "mailingListSubscriptionCount" })
     @JsonDeserialize(builder = AutoValue_SpamLDAPReport_ReportEntry.Builder.class)
     public abstract static class ReportEntry {
         public abstract String getId();
@@ -223,6 +231,12 @@ public class SpamLDAPReport implements Runnable {
         @Nullable
         public abstract ZonedDateTime getLastActive();
 
+        public abstract boolean getHasWikiContributions();
+
+        public abstract int getForumPostCount();
+
+        public abstract int getMailingListSubscriptionCount();
+
         public static Builder builder() {
             return new AutoValue_SpamLDAPReport_ReportEntry.Builder();
         }
@@ -240,6 +254,12 @@ public class SpamLDAPReport implements Runnable {
 
             public abstract Builder setLastActive(@Nullable ZonedDateTime lastActive);
 
+            public abstract Builder setHasWikiContributions(boolean hasWikiContributions);
+
+            public abstract Builder setForumPostCount(int hasForumPosts);
+
+            public abstract Builder setMailingListSubscriptionCount(int hasMailingListSubscriptions);
+
             public abstract ReportEntry build();
         }
     }

src/main/java/org/eclipsefoundation/reports/config/CustomJacksonConfiguration.java

+/*********************************************************************
+* Copyright (c) 2023 Eclipse Foundation.
+*
+* This program and the accompanying materials are made
+* available under the terms of the Eclipse Public License 2.0
+* which is available at https://www.eclipse.org/legal/epl-2.0/
+*
+* Author: Martin Lowe <martin.lowe@eclipse-foundation.org>
+*
+* SPDX-License-Identifier: EPL-2.0
+**********************************************************************/
+package org.eclipsefoundation.reports.config;
+
+import java.text.SimpleDateFormat;
+
+import javax.inject.Singleton;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.PropertyNamingStrategies;
+
+import io.quarkus.jackson.ObjectMapperCustomizer;
+
+/**
+ * Sets up Jackson with default API formatting standards. Copied from the common core as it did everything we needed it
+ * to do.
+ * 
+ * @author Martin Lowe
+ *
+ */
+@Singleton
+public class CustomJacksonConfiguration implements ObjectMapperCustomizer {
+
+    @Override
+    public void customize(ObjectMapper objectMapper) {
+        objectMapper
+                .setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE)
+                .setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX"));
+    }
+}