Rebuild GL sync in TS, add arb. group nesting

src/eclipse/EclipseAPI.ts

+import axios, { AxiosRequestConfig } from 'axios';
+import parse from 'parse-link-header';
+import { AccessToken, ClientCredentials } from 'simple-oauth2';
+import { Logger } from 'winston';
+import { BotDefinition, EclipseProject, EclipseUser } from '../interfaces/EclipseApi';
+import { getLogger } from '../helpers/logger';
+
+const HOUR_IN_SECONDS = 3600;
+const EXIT_ERROR_STATE = 1;
+
+/**
+ * Root config used for interacting with the Eclipse API. OAuth is
+ */
+export interface EclipseApiConfig {
+  oauth?: EclipseApiOAuthConfig;
+  webroots?: EclipseApiWebRoots;
+  staging?: boolean;
+  verbose?: boolean;
+  testMode?: boolean;
+}
+
+export interface EclipseApiWebRoots {
+  projects?: string;
+  api?: string;
+}
+
+/**
+ * Configuration interface for oauth/security binding.
+ */
+export interface EclipseApiOAuthConfig {
+  client_secret: string;
+  client_id: string;
+  endpoint: string;
+  redirect: string;
+  scope: string;
+  timeout?: number;
+}
+
+export class EclipseAPI {
+  config: EclipseApiConfig;
+  client?: ClientCredentials;
+  accessToken: AccessToken | null = null;
+  logger: Logger;
+
+  constructor(config: EclipseApiConfig) {
+    this.config = Object.assign(this.generateDefaultConfigs(config.staging), config);
+    // generate creds if auth is set
+    if (this.config.oauth !== null && this.config.oauth !== undefined) {
+      this.client = new ClientCredentials({
+        client: {
+          id: this.config.oauth.client_id,
+          secret: this.config.oauth.client_secret,
+        },
+        auth: {
+          tokenHost: this.config.oauth.endpoint,
+          tokenPath: '/oauth2/token',
+          authorizePath: '/oauth2/authorize',
+        },
+      });
+    }
+    this.logger = getLogger('info', 'EclipseAPI');
+  }
+
+  /**
+   * Retrieves Eclipse projects with the given query string parameters, with an option to paginate and return all results.
+   *
+   * @param queryStringParams optional query string to use when querying projects
+   * @param paginate Optional, false if only 1 page should be queried, true otherwise.
+   * @returns promise to return either a page or all pages of eclipse projects given a query string.
+   */
+  async eclipseAPI(queryStringParams = '', paginate = true): Promise<EclipseProject[]> {
+    if (this.config.verbose) {
+      this.logger.debug(`EclipseAPI:eclipseAPI(queryStringParams = ${queryStringParams}, paginate = ${paginate})`);
+    }
+    // if test mode is enabled, return data that doesn't impact production
+    if (this.config.testMode) {
+      return testProjects;
+    }
+
+    let hasMore = true;
+    let result = [];
+    let data = [];
+    // add timestamp to url to avoid browser caching
+    let url = 'https://projects.eclipse.org/api/projects' + queryStringParams;
+    // loop through all available users, and add them to a list to be returned
+    do {
+      this.logger.silly('Loading next page...');
+      // get the current page of results, incrementing page count after call
+      result = await axios
+        .get(url)
+        .then(r => {
+          // return the data to the user
+          const links = parse(r.headers.link);
+          // check if we should continue processing
+          if (links === null || links!.self.url === links!.last.url) {
+            hasMore = false;
+          } else {
+            url = links!.next.url;
+          }
+          return r.data;
+        })
+        .catch(err => {
+          this.logger.error(`Error while retrieving results from Eclipse Projects API (${url}): ${err}`);
+        });
+
+      // collect the results
+      if (result !== null && result.length > 0) {
+        data = [...data, ...result];
+      }
+    } while (hasMore && paginate);
+    return data;
+  }
+
+  /**
+   * Retrieves an eclipse foundation user using the given username, returning null if the user cannot be found.
+   *
+   * @param username the username to retrieve data for
+   * @returns the Eclipse Foundation user account data, enhanced with sensitive information if oauth is configured. Returns
+   * null if user cannot be found.
+   */
+  async eclipseUser(username: string): Promise<EclipseUser | null> {
+    if (this.config.verbose) {
+      this.logger.debug(`EclipseAPI:eclipseUser(username = ${username})`);
+    }
+
+    return await axios
+      .get('https://api.eclipse.org/account/profile/' + username, await this.getAuthenticationHeaders())
+      .then(result => result.data)
+      .catch(err => {
+        this.logger.error(`${err}`);
+        return null;
+      });
+  }
+
+  async eclipseBots(): Promise<BotDefinition[]> {
+    if (this.config.verbose) {
+      this.logger.debug('EclipseAPI:eclipseBots()');
+    }
+    const botsRaw = await axios
+      .get('https://api.eclipse.org/bots')
+      .then(result => result.data)
+      .catch(err => this.logger.error(`${err}`));
+    if (botsRaw === undefined || botsRaw.length <= 0) {
+      this.logger.error('Could not retrieve bots from API');
+      process.exit(EXIT_ERROR_STATE);
+    }
+    return botsRaw;
+  }
+
+  /**
+   * Maps bot users by listing bots per project.
+   *
+   * @param botsRaw the raw bot definition list to convert to a project bot mapping.
+   * @param site the site targeted for bots to limit results
+   * @returns a mapping of project to configured bot usernames.
+   */
+  processBots(botsRaw: BotDefinition[], site = 'github.com'): Record<string, string[]> {
+    if (this.config.verbose) {
+      this.logger.debug(`EclipseAPI:processBots(botsRaw = ${JSON.stringify(botsRaw)}, site = ${site})`);
+    }
+    const rgx = new RegExp(`^${site}.*`);
+    const botMap: Record<string, string[]> = {};
+    botsRaw.forEach(bot => {
+      // get the list of bots for project if already created
+      let projBots = botMap[bot.projectId];
+      if (projBots === undefined) {
+        projBots = [];
+      }
+      // get usernames for site + sub resource bots
+      Object.keys(bot).forEach(key => {
+        if (key.match(rgx)) {
+          projBots.push(bot[key].username);
+        }
+      });
+      // dont add empty arrays to output
+      if (projBots.length !== 0) {
+        botMap[bot.projectId] = projBots;
+      }
+    });
+    return botMap;
+  }
+
+  /**
+   * If OAuth has been configured, then retrieves access token and sets into request config to use for calls to Eclipse API endpoints.
+   * This call is needed to be able to retrieve sensitive user profile information.
+   *
+   * @returns request configs including authentication headers if auth is configured, or empty config otherwise.
+   */
+  async getAuthenticationHeaders(): Promise<AxiosRequestConfig> {
+    const token = await this._getAccessToken();
+    if (token === null) {
+      this.logger.info('Authentication token cannot be retrieved, information returned mey be limited');
+      return {};
+    } else {
+      return {
+        headers: {
+          Authorization: `Bearer ${token}`,
+        },
+      };
+    }
+  }
+
+  /**
+   * Retrieves an oauth token and caches it internally to authenticate requests to the Eclipse API. This by default caches
+   * for an hour to reduce turn around on the authentication API.
+   *
+   * @returns the access token if found, otherwise null.
+   */
+  async _getAccessToken(): Promise<string | null> {
+    // check that we have auth configs and that current token is expired before attempting retrieval
+    if (
+      this.config.oauth !== null &&
+      this.client !== null &&
+      (this.accessToken === null || this.accessToken!.expired(this.config.oauth.timeout ?? HOUR_IN_SECONDS))
+    ) {
+      // wrap retrieval in try-catch to give more information to the logs on error.
+      try {
+        this.accessToken = await this.client!.getToken({
+          scope: this.config.oauth.scope,
+        });
+      } catch (error) {
+        this.logger.error(`${error}`);
+        process.exit(EXIT_ERROR_STATE);
+      }
+      return this.accessToken.token.access_token;
+    }
+    return null;
+  }
+
+  generateDefaultConfigs(useStaging = false): EclipseApiConfig {
+    if (useStaging) {
+      return {
+        webroots: {
+          projects: 'https://projects-staging.eclipse.org',
+          api: 'https://api-staging.eclipse.org',
+        },
+        testMode: false,
+        verbose: false,
+      };
+    }
+    return {
+      webroots: {
+        projects: 'https://projects.eclipse.org',
+        api: 'https://api.eclipse.org',
+      },
+      testMode: false,
+      verbose: false,
+    };
+  }
+}
+
+const testProjects: EclipseProject[] = [
+  {
+    project_id: 'technology.spider.pig',
+    url: '',
+    website_repo: [],
+    website_url: '',
+    short_project_id: 'spider.pig',
+    name: 'Spider pig does what a spider pig does',
+    summary: 'Can he fly? No, hes a pig. Look out, here comes the spider pig',
+    logo: '',
+    tags: ['simpsons', 'doh', 'spider pig'],
+    top_level_project: 'technology',
+    gitlab: {
+      ignored_sub_groups: ['eclipse/spider.pig/excludes'],
+      project_group: 'eclipse/spider.pig',
+    },
+    gerrit_repos: [
+      {
+        url: 'https://github.com/eclipsefdn-webdev/spider-pig',
+      },
+    ],
+    github_repos: [
+      {
+        url: 'https://github.com/eclipsefdn-webdev/spider-pig',
+      },
+    ],
+    gitlab_repos: [
+      {
+        url: 'http://localhost/eclipse/spider.pig',
+      },
+    ],
+    contributors: [],
+    committers: [
+      {
+        username: 'malowe',
+        url: 'https://api.eclipse.org/account/profile/malowe',
+      },
+      {
+        username: 'epoirier',
+        url: 'https://api.eclipse.org/account/profile/epoirier',
+      },
+    ],
+    project_leads: [
+      {
+        username: 'malowe',
+        url: 'https://api.eclipse.org/account/profile/malowe',
+      },
+      {
+        username: 'cguindon',
+        url: 'https://api.eclipse.org/account/profile/cguindon',
+      },
+    ],
+    working_groups: [
+      {
+        name: 'Cloud Development Tools',
+        id: 'cloud-development-tools',
+      },
+    ],
+    spec_project_working_group: [],
+    state: 'Regular',
+    releases: [],
+  },
+];