GitlabSyncRunner.ts 32.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
/** ***************************************************************
 Copyright (C) 2022 Eclipse Foundation, Inc.

 This program and the accompanying materials are made
 available under the terms of the Eclipse Public License 2.0
 which is available at https://www.eclipse.org/legal/epl-2.0/

  Contributors:
    Martin Lowe <martin.lowe@eclipse-foundation.org>

 SPDX-License-Identifier: EPL-2.0
******************************************************************/

14
import { Logger } from 'winston';
15
import { Resources } from '@gitbeaker/core/dist/types';
16
17
import { AccessLevel, GroupSchema, MemberSchema, ProjectSchema, UserSchema } from '@gitbeaker/core/dist/types/types';
import { v4 } from 'uuid';
18
19
20
import { EclipseAPI, EclipseApiConfig } from '../../eclipse/EclipseAPI';
import { getLogger } from '../../helpers/logger';
import { SecretReader, getBaseConfig } from '../../helpers/SecretReader';
21
import { EclipseProject } from '../../interfaces/EclipseApi';
22
23

// used to make use of default requested based on Got rather than recreating our own
24
25
import { Gitlab } from '@gitbeaker/core';
import { requesterFn } from './AxiosRequester';
26

27
28
29
const adminPermissionsLevel = 50;
const maintainerPermissionsLevel = 40;
const allowlistedUsers: string[] = ['webmaster', 'root'];
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
/**
 * Represents the nested group cache that can represent the relationships between groups and to simplify child lookups.
 */
interface GroupCache {
  _self: GroupSchema | null;
  projectTargets: string[];
  children: Record<string, GroupCache>;
}

interface EclipseUserAccess {
  url: string;
  accessLevel: AccessLevel;
}

interface GitlabSyncRunnerConfig {
  host: string;
  provider: string;
  secretLocation?: string;
  project?: string;
  verbose: boolean;
  devMode: boolean;
  dryRun: boolean;
52
  rootGroup?: string;
53
  staging?: boolean;
54
55
56
57
}

export class GitlabSyncRunner {
  // internal state
58
59
  accessToken = '';
  eclipseToken = '';
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
  config: GitlabSyncRunnerConfig;
  logger: Logger;

  // api access
  api: Resources.Gitlab;
  eApi: EclipseAPI;
  bots: Record<string, string[]> = {};

  // caches to optimize calling
  namedUsers: Record<string, UserSchema> = {};
  groupCache: GroupCache = {
    _self: null,
    children: {},
    projectTargets: [],
  };
75
  projectsCache: ProjectSchema[] = [];
76
77
78
  eclipseProjectCache: Record<string, EclipseProject> = {};
  gMems: Record<number, MemberSchema[]> = {};

79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
  /**
   * Sets the internal config with a few default values and creates the bindings for the APIs that are
   * accessed during the run of this script.
   *
   * @param config the initial script configuration object.
   */
  constructor(config: GitlabSyncRunnerConfig) {
    this.config = Object.assign(
      {
        host: 'http://gitlab.eclipse.org/',
        provider: 'oauth2_generic',
        verbose: false,
        devMode: false,
        dryRun: false,
        rootGroup: 'eclipse',
      },
95
      config,
96
97
    );

98
99
100
101
102
103
104
    this.logger = getLogger(this.config.verbose ? 'debug' : 'info', 'main');
    this._prepareSecret();

    // create API instances
    this.api = new Gitlab({
      host: this.config.host,
      token: this.accessToken,
105
      requesterFn: requesterFn,
106
    });
107
    const eclipseAPIConfig: EclipseApiConfig = JSON.parse(this.eclipseToken);
108
109
110
111
112
    eclipseAPIConfig.testMode = this.config.devMode;
    eclipseAPIConfig.verbose = this.config.verbose;
    this.eApi = new EclipseAPI(eclipseAPIConfig);
  }

113
114
115
116
117
  /**
   * Prepares the secrets required for the script to run. Specifically the eclipseToken used for Eclipse
   * API access and the accessToken which is used for sudo+api access on the Gitlab instance targeted by
   * this script.
   */
118
119
  _prepareSecret() {
    // retrieve the secret API file root if set
120
    const settings = getBaseConfig();
121
122
123
    if (this.config.secretLocation !== undefined) {
      settings.root = this.config.secretLocation;
    }
124
125
    const reader = new SecretReader(settings);
    let data = reader.readSecret('access-token');
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
    if (data !== null) {
      this.accessToken = data.trim();
      // retrieve the Eclipse API token (needed for emails)
      data = reader.readSecret('eclipse-oauth-config');
      if (data !== null) {
        this.eclipseToken = data.trim();
      } else {
        this.logger.error('Could not find the Eclipse OAuth config, exiting');
        process.exit(1);
      }
    } else {
      this.logger.error('Could not find the GitLab access token, exiting');
      process.exit(1);
    }
  }

142
143
144
145
146
147
148
149
150
  /**
   * Run the full sync script, syncing the PMI to the Gitlab instance targeted by the script. This script
   * will sync the namespace groups named in projects with the users that are set as members of the project.
   * It will also clear users added outside of this process with the exception of bot users to maintain more
   * strict control of the access permissions.
   *
   * @returns a promise that is completed once the run completes.
   */
  async run(): Promise<void> {
151
152
153
154
    // prepopulate caches to optimally retrieve info used in sync ops
    await this.prepareCaches();
    // fetch org group from results, create if missing
    this.logger.info('Starting sync');
155
    const g = this.getRootGroup();
156
157
    if (g._self === null) {
      this.logger.error(`Unable to start sync of GitLab content. Base group (${this.config.rootGroup}) could not be found`);
158
159
160
      return;
    }

161
162
    for (const projectIdx in this.eclipseProjectCache) {
      const project = this.eclipseProjectCache[projectIdx];
163
164
165
166
      if (this.config.project !== undefined && project.short_project_id !== this.config.project) {
        this.logger.info(`Project target set ('${this.config.project}'). Skipping non-matching project ID ${project.short_project_id}`);
        continue;
      }
167
168

      // fetch group namespace indicated by the project and ensure format
Martin Lowe's avatar
Martin Lowe committed
169
      const actualNamespace = project.gitlab.project_group;
170
      const [projectNamespace, projectNamespaceTLP] = [
171
172
173
174
175
176
177
178
179
180
181
        `${this.config.rootGroup}/${project.short_project_id}`,
        `${this.config.rootGroup}/${project.top_level_project}/${project.short_project_id}`,
      ];
      if (actualNamespace === undefined || actualNamespace.trim() === '') {
        this.logger.info(`Skipping project '${project.project_id}' as it has no Gitlab namespace`);
        continue;
      } else if (
        actualNamespace.localeCompare(projectNamespace, undefined, { sensitivity: 'base' }) !== 0 &&
        actualNamespace.localeCompare(projectNamespaceTLP, undefined, { sensitivity: 'base' }) !== 0
      ) {
        this.logger.info(
182
          `Skipping namespace '${actualNamespace}' for project '${project.short_project_id}', does not match allowed formats`,
183
184
185
        );
        continue;
      }
186
187
      this.logger.info(`Processing '${project.short_project_id}'`);

188
      // check group cache to ensure well formed.
189
      const namespaceGroup = await this.getCachedGroup(actualNamespace);
190
191
192
193
      if (namespaceGroup === null || namespaceGroup._self === null) {
        this.logger.error(`Could not find group with namespace ${actualNamespace}`);
        continue;
      }
194
      // get the list of users to be added for current project
195
      const userList = this.getUserList(project);
196
      // for each user, get their gitlab user and add to the project group
197
      const usernames = Object.keys(userList);
198
      // update the group to add the users for the current project
199
200
201
      for (const usernameIdx in usernames) {
        const uname = usernames[usernameIdx];
        const user = await this.getUser(uname, userList[uname].url);
202
203
        if (user === null) {
          this.logger.verbose(`Could not retrieve user for UID '${uname}', skipping`);
204
205
206
          continue;
        }

207
208
        await this.addUserToGroup(user, namespaceGroup._self!, userList[uname].accessLevel);
        // if not tracked, track current project for group for post-sync cleanup
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
        if (namespaceGroup.projectTargets.indexOf(project.project_id) === -1) {
          namespaceGroup.projectTargets.push(project.project_id);
        }
      }

      // retrieve bots for current project and add them to the groups
      for (const botIdx in this.bots[project.project_id]) {
        const bot = this.bots[project.project_id][botIdx];
        this.logger.verbose(`Found ${bot} for ${project.project_id}`);
        // get the bot user if it exists already
        const botUser = await this.getUser(bot, bot);
        if (botUser == null) {
          this.logger.info(`Could not retrieve user for bot user ${bot} for project ${project.project_id}, `
            + 'not attempting to add to group');
          continue;
224
        }
225
226
227
        // add bot user to the group
        this.logger.verbose(`Adding bot ${bot} to group ${namespaceGroup._self!.path}`);
        await this.addUserToGroup(botUser, namespaceGroup._self!, maintainerPermissionsLevel);
228
229
      }
    }
230
    // perform cleanup operations to clean out extra users
231
    this.cleanupGroups();
232
    this.cleanupProjects();
233
234
235
236
237
238
239
  }

  /**
   * Generates the caches needed for running the Gitlab sync process.
   */
  async prepareCaches() {
    // get raw project data and post process to add additional context
240
    try {
241
      const data = await this.eApi.eclipseAPI();
242

243
      // get the bots for the projects
244
      const rawBots = await this.eApi.eclipseBots();
245
      this.bots = this.eApi.processBots(rawBots, 'gitlab.eclipse.org');
246

247
248
      // get all current groups for the instance
      this.projectsCache = await this.api.Projects.all();
249
250
      const groups = await this.api.Groups.all();
      const users = await this.api.Users.all();
251

252
253
      // generates the nested cache
      this.generateGroupsCache(groups);
254
255
      this.namedUsers = users.reduce((acc, item) => ({ ...acc, [item.username]: item }), {} as Record<string, UserSchema>);
      this.eclipseProjectCache = data.reduce(
256
        (acc, item) => ({ ...acc, [item.project_id]: item }),
257
258
        {} as Record<string, EclipseProject>,
      );
259
260
261
    } catch (e) {
      this.logger.error(`Cannot fetch resources associated with sync operations, exiting: ${e}`);
      process.exit(1);
262
263
264
265
    }
  }

  /**
266
267
   * Iterate through each group, checking self and ancestor project users and comparing against the current groups users to ensure that
   * there are no additional users added with permissions.
268
   */
269
  async cleanupGroups(currentLevel: GroupCache = this.getRootGroup(), collectedProjects: string[] = []) {
270
    this.logger.debug(`cleanupGroups(currentLevel = ${currentLevel._self?.full_path}, collectedProjects = ${collectedProjects})`);
271
    const self = currentLevel._self;
272
273
274
275
276
    if (self === null) {
      this.logger.error('Error encountered during group cleanup process, ending early');
      return;
    }
    // collect and deduplicate project IDs
277
    const projects = [...Array.from(new Set([...currentLevel.projectTargets, ...collectedProjects]))];
278
    // build the user mapping to pass to cleanup
279
    let projectUsers: Record<string, EclipseUserAccess> = {};
280
    for (const pidx in projects) {
281
      // check if any of the matched projects marks this as an external/skipped namespace
282
      const project = this.eclipseProjectCache[projects[pidx]];
Martin Lowe's avatar
Martin Lowe committed
283
      if (project.gitlab.ignored_sub_groups.some(v => v.localeCompare(self.full_path, undefined, { sensitivity: 'base' }) === 0)) {
284
285
286
287
        this.logger.info(`Group '${self.full_path}' was marked as an external/protected namespace by project '${project.project_id}'`);
        return;
      }
      projectUsers = Object.assign(projectUsers, this.getUserList(project));
288
    }
289
290
    // clean up additional users
    await this.removeAdditionalUsers(projectUsers, self, ...projects);
291
    // for each of the children, pass the collected projects forward and process
292
    for (const cidx in currentLevel.children) {
293
294
295
296
      this.cleanupGroups(currentLevel.children[cidx], projects);
    }
  }

297
298
299
300
301
302
303
304
305
  /**
   * Removes users not tracked in the expectedUsers map from the passed group. Project IDs are used to look up bot user
   * account names as they are exempt from being removed as they are used for CI ops.
   *
   * @param expectedUsers map of usernames to their EclipseUser entry
   * @param group the Gitlab group that is being cleaned of extra users.
   * @param projectIDs list of project IDs that impact the given group
   * @returns a promise that completes once all additional users are removed or the check finishes
   */
306
307
308
309
310
  async removeAdditionalUsers(
    expectedUsers: Record<string, EclipseUserAccess>,
    group: GroupSchema,
    ...projectIDs: string[]
  ): Promise<void> {
311
    if (this.config.verbose) {
312
313
314
      this.logger.debug(
        `GitlabSync:removeAdditionalUsers(expectedUsers = ${JSON.stringify(expectedUsers)}, group = ${
          group.full_path
315
        }, projectIDs = ${projectIDs})`,
316
      );
317
318
    }
    // get the current list of users for the group
319
    const members = await this.getGroupMembers(group);
320
321
    if (members === null || !(members instanceof Array)) {
      this.logger.warn(`Could not find any group members for group '${group.full_path}'. Skipping user removal check`);
322
323
324
      return;
    }
    // check that each of the users in the group match whats expected
325
    const expectedUsernames = Object.keys(expectedUsers);
326
    members?.forEach(async member => {
327
328
      // check access and ensure user isn't an owner
      this.logger.verbose(`Checking user '${member.username}' access to group '${group.name}'`);
329
      if (this.shouldRemoveUser(member, expectedUsers, projectIDs, expectedUsernames)) {
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
        if (this.config.dryRun) {
          this.logger.info(`Dryrun flag active, would have removed user '${member.username}' from group '${group.name}'`);
          return;
        }
        this.logger.info(`Removing user '${member.username}' from group '${group.name}'`);
        try {
          await this.api.GroupMembers.remove(group.id, member.id);
        } catch (err) {
          if (this.config.verbose) {
            this.logger.error(`${err}`);
          }
          this.logger.warn(`Error while removing user '${member.username}' from group '${group.name}'`);
        }
      }
    });
  }

347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
  /**
   * Checks for the following states:
   *
   * - User is outside the allowlisted users
   * - User is outside the expected user list
   * - The user has the wrong permissions set and isn't a project lead
   * - the user isn't a bot
   *
   * @param member the current group member being checked
   * @param expectedUsers the user access mapping for the current group
   * @param projectIDs projects associated with the current group
   * @param expectedUsernames the usernames from the users mapping, passed to save processing time
   * @returns true if all conditions in method description are met, otherwise false
   */
  shouldRemoveUser(
    member: MemberSchema,
    expectedUsers: Record<string, EclipseUserAccess>,
    projectIDs: string[],
365
    expectedUsernames: string[],
366
367
  ): boolean {
    return (
368
      allowlistedUsers.indexOf(member.username) === -1 &&
369
      (expectedUsernames.indexOf(member.username) === -1 ||
370
371
        (member.access_level !== expectedUsers[member.username]!.accessLevel &&
          expectedUsers[member.username]!.accessLevel !== maintainerPermissionsLevel)) &&
372
373
374
375
      !this.isBot(member.username, projectIDs)
    );
  }

376
377
378
379
380
  /**
   * Iterates over the projects cache and cleans out the users and keeps bots for build operations. Skips over projects
   * outside the scope of the designated root group to avoid over processing groups.
   */
  async cleanupProjects() {
381
    this.projectsCache.forEach(async p => {
382
383
384
385
386
387
      // don't process projects outside target namespace
      if (!this.withinNamespace(p.namespace.full_path)) {
        return;
      }

      // get the group of the project and clean it up
388
      const group = await this.getCachedGroup(p.namespace.full_path);
389
      if (group !== null) {
390
        this.cleanUpProjectUsers(p, ...group!.projectTargets);
391
      } else {
392
        this.logger.info(`Skipping processing of project '${p.name}'`);
393
394
395
396
397
398
399
400
401
402
403
404
      }
    });
  }

  /**
   * Removes any non-owner user that isn't a bot from projects. Membership is managed at the group level, not the direct
   * project level.
   *
   * @param project the Gitlab project to sanitize
   * @param projectIDs the Eclipse projects that impact the Gitlab project.
   */
  async cleanUpProjectUsers(project: ProjectSchema, ...projectIDs: string[]) {
405
406
407
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:cleanUpProjectUsers(project = ${project.id})`);
    }
408
409
410
    const projectMembers = await this.api.ProjectMembers.all(project.id, { includeInherited: false });
    for (const idx in projectMembers) {
      const member = projectMembers[idx];
411
      this.logger.verbose(`Checking '${member.username}' for removal on project '${project.namespace.full_path}'(${member.access_level})`);
412
      // skip bot user or admin users
413
      if (this.isBot(member.username, projectIDs) || member.access_level === adminPermissionsLevel) {
414
415
416
417
418
419
        continue;
      }
      if (this.config.dryRun) {
        this.logger.debug(`Dryrun flag active, would have removed user '${member.username}' from project '${project.name}'(${project.id})`);
        continue;
      }
420
421
      this.logger.info(`Removing user '${member.username}' with permissions '${member.access_level}' from project `
        + `'${project.name}'(${project.id})`);
422
423
424
425
426
427
428
429
430
431
432
      try {
        await this.api.ProjectMembers.remove(project.id, member.id);
      } catch (err) {
        if (this.config.verbose) {
          this.logger.error(`${err}`);
        }
        this.logger.error(`Error while removing user '${member.username}' from project '${project.name}'(${project.id})`);
      }
    }
  }

433
434
435
436
437
438
439
440
441
  /**
   * Ensures that the user exists within the group with the given access level (no more or less). If a user has too high
   * permissions, the membership is modified to have the given access instead.
   *
   * @param user the user that is being given permissions
   * @param group group that the user should be added to
   * @param perms the permission set to give the user
   * @returns the membership information for the user wrt to this Gitlab group.
   */
442
443
  async addUserToGroup(user: UserSchema, group: GroupSchema, perms: AccessLevel): Promise<MemberSchema | null> {
    if (this.config.verbose) {
444
      this.logger.debug(`GitlabSync:addUserToGroup(user = ${user?.username}, group = ${group?.full_path}, perms = ${perms})`);
445
446
    }
    // get the members for the current group
447
    const members = await this.getGroupMembers(group);
448
449
450
451
452
453
    if (members === null) {
      this.logger.warn(`Could not find any references to group with ID ${group.id}`);
      return null;
    }

    // check if user is already present
454
    for (let i = 0; i < members.length; i++) {
455
      const member = members[i];
456
457
458
459
460
461
462
463
464
465
466
467
      if (member.username === user.username) {
        this.logger.verbose(`User '${user.username}' is already a member of ${group.name}`);
        if (member.access_level !== perms) {
          // skip if dryrun
          if (this.config.dryRun) {
            this.logger.info(`Dryrun flag active, would have updated user '${member.username}' in group '${group.name}'`);
            return null;
          }

          // modify user, catching errors
          this.logger.info(`Fixing permission level for user '${user.username}' in group '${group.name}'`);
          try {
468
            const updatedMember = await this.api.GroupMembers.edit(group.id, user.id, perms);
469
            // update inner array
470
            members![i] = updatedMember;
471
472
473
474
475
476
477
478
479
480
            this.gMems[group.id] = members!;
          } catch (err) {
            if (this.config.verbose) {
              this.logger.error(`${err}`);
            }
            this.logger.warn(`Error while fixing permission level for user '${user.username}' in group '${group.name}'`);
            return null;
          }
        }
        // return a copy of the updated user
481
        return members![i];
482
      }
483
    }
484
485
486
    // check if dry run before updating
    if (this.config.dryRun) {
      this.logger.info(
487
        `Dryrun flag active, would have added user '${user.username}' to group '${group.name}' with access level '${perms}'`,
488
489
490
491
492
493
494
      );
      return null;
    }

    this.logger.info(`Adding '${user.username}' to '${group.name}' group`);
    try {
      // add member to group, track, and return a copy
495
      const newMember = await this.api.GroupMembers.add(group.id, user.id, perms);
496
497
498
499
500
501
502
503
504
505
506
507
508
509
      members.push(newMember);
      this.gMems[group.id] = members;

      // return a copy
      return newMember;
    } catch (err) {
      if (this.config.verbose) {
        this.logger.error(`${err}`);
      }
      this.logger.warn(`Error while adding '${user.username}' to '${group.name}' group`);
    }
    return null;
  }

510
511
512
513
514
515
516
517
  /**
   * Retrieves a Gitlab user object for the given Eclipse user given their username and access URL. If the
   * user does not yet exist, a new user is created, cached, and returned for use.
   *
   * @param uname the Eclipse username of user to retrieve from Gitlab
   * @param url the Eclipse user access URL
   * @returns the gitlab user, or null if it can't be found or created.
   */
518
519
520
521
522
523
524
525
526
  async getUser(uname: string, url: string): Promise<UserSchema | null> {
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:getUser(uname = ${uname}, url = ${url})`);
    }
    if (url === undefined || url === '') {
      this.logger.error(`Cannot fetch user information for user '${uname}' with no set URL`);
      return null;
    }

527
    let u = this.namedUsers[uname];
528
529
530
531
532
533
534
    if (u === undefined) {
      if (this.config.dryRun) {
        this.logger.info(`Dryrun is enabled. Would have created user ${uname} but was skipped`);
        return null;
      }

      // retrieve user data
535
      const data = await this.eApi.eclipseUser(uname);
536
537
538
539
540
      if (data === null) {
        this.logger.error(`Cannot create linked user account for '${uname}', no external data found`);
        return null;
      }
      this.logger.verbose(`Creating new user with name '${uname}'`);
541
      const opts = {
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
        username: uname,
        password: v4(),
        force_random_password: true,
        name: `${data!.first_name} ${data!.last_name}`,
        email: data!.mail,
        extern_uid: data!.uid,
        provider: this.config.provider,
        skip_confirmation: true,
      };
      // check if dry run before creating new user
      if (this.config.dryRun) {
        this.logger.info(`Dryrun flag active, would have created new user '${uname}' with options ${JSON.stringify(opts)}`);
        return null;
      }

      // if verbose, display information being used to generate user
      if (this.config.verbose) {
        // copy the object and redact the password for security
560
        const optLog = JSON.parse(JSON.stringify(opts));
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
        optLog.password = 'redacted';
        this.logger.debug(`Creating user with options: ${JSON.stringify(optLog)}`);
      }
      try {
        u = await this.api.Users.create(opts);
      } catch (err) {
        if (this.config.verbose) {
          this.logger.error(`${err}`);
        }
      }
      if (u === null) {
        this.logger.warn(`Error while creating user '${uname}'`);
        return null;
      }
      // set it back
      this.namedUsers[uname] = u;
    }
    return u;
  }

581
582
583
584
585
586
587
588
589
590
591
592
593
  /**
   * Used to create missing groups in the Gitlab instance. Does not insert into the nest cache as this should only be
   * called from said cache. This method does not support creating root level groups.
   *
   * @param name the name of the group to create
   * @param parent the group that this group belongs to
   * @returns the new group schema once the call finishes
   */
  async createMissingGroup(name: string, parent: GroupSchema): Promise<GroupSchema | null> {
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:createMissingGroup(name = ${name}, parent = ${parent.id})`);
    }
    // default options for creating new group
594
    const opts = {
595
596
597
598
599
600
601
602
603
604
605
606
607
608
      project_creation_level: 'maintainer',
      visibility: 'public',
      request_access_enabled: false,
      parent_id: parent.id,
    };
    this.logger.info(`Creating missing group '${name}' in namespace '${parent.full_path} (${parent.id})'`);
    try {
      return await this.api.Groups.create(name, name, opts);
    } catch (err) {
      this.logger.error(`${err}`);
      return null;
    }
  }

609
610
611
612
613
614
  /**
   * Retrieves the list of direct members for a given group, ignoring inherited users.
   *
   * @param group the Gitlab group to retrieve members for
   * @returns a list of Gitlab group members for the given group, or null if there is an error while fetching.
   */
615
616
  async getGroupMembers(group: GroupSchema): Promise<MemberSchema[] | null> {
    if (this.config.verbose) {
617
      this.logger.debug(`GitlabSync:getGroupMembers(group = ${group?.full_path})`);
618
    }
619
    let members = this.gMems[group.id];
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
    if (members === undefined) {
      try {
        members = await this.api.GroupMembers.all(group.id, { includeInherited: false });
      } catch (err) {
        if (this.config.verbose) {
          this.logger.error(`${err}`);
        }
      }
      if (members === null) {
        this.logger.warn(`Unable to find group members for group with ID '${group.id}'`);
        return null;
      }
      this.gMems[group.id] = members;
    }
    return members;
  }

  /** HELPERS */

639
640
641
642
  /**
   * Generate the nested group cache using the raw Gitlab group definitions.
   * @param rawGroups
   */
643
644
645
646
647
648
649
650
651
652
653
654
655
  generateGroupsCache(rawGroups: GroupSchema[]): void {
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:generateGroupsCache(projects = count->${rawGroups.length})`);
    }
    // create initial cache container
    this.groupCache = {
      _self: null,
      projectTargets: [],
      children: {},
    };

    // iterate through groups and insert into the nested cache
    for (let i = 0; i < rawGroups.length; i++) {
656
      this.addGroup(rawGroups[i]);
657
658
659
    }
  }

660
661
662
  /**
   * @returns the root group cache for the current sync operation if it exists. If missing, the script ends processing.
   */
663
  getRootGroup(): GroupCache {
664
    const rootGroupCache = this.groupCache.children[this.config.rootGroup];
665
666
667
668
669
670
671
    if (rootGroupCache === undefined) {
      this.logger.error(`Could not find root group '${this.config.rootGroup}' for group caching, exiting`);
      process.exit(1);
    }
    return rootGroupCache;
  }

672
673
674
675
676
677
678
679
  /**
   * Retrieves the group for the given namespace path. This namespace path should be formatted such that each group path is separated
   * by a slash, eg. eclipse/sample/group. This will be split and used to iterate through the nested cache, returning the group once
   * each namespace path part is used.
   *
   * @param namespace the full path of the group namespace to retrieve.
   * @returns the group cache for the group indicated by the namespace string, or null if there is no matching group.
   */
680
  async getCachedGroup(namespace: string): Promise<GroupCache | null> {
681
682
683
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:getCachedGroup(${namespace})`);
    }
684
    if (!this.withinNamespace(namespace)) {
685
686
687
      this.logger.info(`Returning null for ${namespace} as it is outside of the root group ${this.config.rootGroup}`);
      return null;
    }
688
689
690
    return this.tunnelAndRetrieve(namespace.split('/'), this.groupCache);
  }

691
692
693
694
695
696
697
698
  /**
   * Adds a group to the nested group cache, using the groups full_path property to discover how to insert the
   * entry into the nested cache. Any cache nodes that do not exist yet will be created as the group is inserted.
   *
   * @param g the Gitlab group that is being inserted into the group cache.
   * @returns the group cache entry for the cached gitlab group
   */
  addGroup(g: GroupSchema): GroupCache {
699
700
701
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:addGroup(g = ${g.id})`);
    }
702
    const namespace = g.full_path;
703
    // split into group namespace paths (eclipse/sample/group.path into ['eclipse','sample','group.path'])
704
    const namespaceParts = namespace.split('/');
705
706
707
    return this.tunnelAndInsert(namespaceParts, g, this.groupCache);
  }

708
709
710
711
712
713
714
715
716
717
  /**
   * Recursive function for inserting groups into the nested group cache. Will tunnel through cache, creating
   * entries as necessary before inserting the group at the nesting level representing the final group in the
   * full path of the group.
   *
   * @param namespaceParts the parts of the full_path left to process for group nesting
   * @param g  the group to be inserted into the group cache.
   * @param parent the parent level for the current level of insertion
   * @returns the group cache entry for the group once inserted.
   */
718
  tunnelAndInsert(namespaceParts: string[], g: GroupSchema, parent: GroupCache): GroupCache {
719
720
721
722
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:tunnelAndInsert(namespaceParts = '${namespaceParts}', g = ${g.id})`);
    }

723
    // get the next level cache if it exists, creating it if it doesn't
724
725
726
727
728
729
730
731
732
733
734
    let child = parent.children[namespaceParts[0]];
    if (child === undefined) {
      child = {
        _self: null,
        projectTargets: [],
        children: {},
      };
      parent.children[namespaceParts[0]] = child;
    }
    // check if we should continue tunneling or insert and finish processing
    if (namespaceParts.length > 1) {
735
      return this.tunnelAndInsert(namespaceParts.slice(1, namespaceParts.length), g, child);
736
    }
737
738
    child._self = g;
    return child;
739
740
  }

741
742
743
744
745
746
747
748
  /**
   * Recursive access to the nested group cache. Retrieves the group described by the namespace parts and returns
   * it, returning null if it can't be found.
   *
   * @param namespaceParts the full path for a group namespace split into parts
   * @param parent the parent to search through for the next part of the recursive call.
   * @returns The group cache for the designated group, or null if it can't be found.
   */
749
  async tunnelAndRetrieve(namespaceParts: string[], parent: GroupCache): Promise<GroupCache | null> {
750
751
752
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:tunnelAndRetrieve(namespaceParts = '${namespaceParts}')`);
    }
753
754
    let child = parent.children[namespaceParts[0]];
    if (child === undefined) {
755
      // attempt to create the new group
756
      const newGroup = await this.createMissingGroup(namespaceParts[0], parent._self);
757
758
759
760
761
762
      if (newGroup === null) {
        this.logger.warn(`Could not create missing group with name '${namespaceParts[0]}' in group with path '${parent._self.full_path}'`);
        return null;
      }
      // insert the new child group into the cache and continue
      child = this.tunnelAndInsert(newGroup.full_path.split('/'), newGroup, this.getRootGroup());
763
764
765
    }
    // check if we should continue tunneling or insert and finish processing
    if (namespaceParts.length > 1) {
766
      return this.tunnelAndRetrieve(namespaceParts.slice(1, namespaceParts.length), child);
767
    }
768
    return child;
769
770
  }

771
772
773
774
775
776
  /**
   * Gets list of users with access permissions for the given Eclipse project.
   *
   * @param project the Eclipse project to parse user entries for
   * @returns the mapping of users to access permissions and entity access URL.
   */
777
778
  getUserList(project: EclipseProject): Record<string, EclipseUserAccess> {
    if (this.config.verbose) {
779
      this.logger.debug(`GitlabSync:getUserList(project = ${project.project_id})`);
780
    }
781
    const l: Record<string, EclipseUserAccess> = {};
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
    // add the contributors with reporter access
    project.contributors.forEach(v => {
      l[v.username] = {
        url: v.url,
        accessLevel: 20,
      };
    });
    // add the committers with developer access
    project.committers.forEach(v => {
      l[v.username] = {
        url: v.url,
        accessLevel: 30,
      };
    });
    // add the project leads not yet tracked with reporter access
    project.project_leads.forEach(v => {
      l[v.username] = {
        url: v.url,
        accessLevel: 40,
      };
    });
    // add the bots with developer access
804
    const botList = this.bots[project.project_id];
805
806
807
808
809
810
811
812
813
814
815
    if (botList !== undefined && botList.length === 0) {
      botList.forEach(v => {
        l[v] = {
          url: '',
          accessLevel: 30,
        };
      });
    }
    return l;
  }

816
817
818
819
820
821
  /**
   * Sanitizes and normalizes strings for use in creating/accessing groups.
   *
   * @param pid the project ID to normalize
   * @returns normalized group name for value.
   */
822
823
824
825
826
827
828
829
830
831
  sanitizeGroupName(pid: string): string {
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:sanitizeGroupName(pid = ${pid})`);
    }
    if (pid !== undefined) {
      return pid.toLowerCase().replace(/[^\s\da-zA-Z-.]/g, '-');
    }
    return '';
  }

832
833
834
835
836
837
838
  /**
   * Checks whether a user is a bot for the given projects.
   *
   * @param uname potential bot username
   * @param projectIDs the projects that the user could be a bot for.
   * @returns true if the user is a designated bot for the projects, otherwise false.
   */
839
  isBot(uname: string, projectIDs: string[]): boolean {
840
    return projectIDs.some(v => this.bots[v] !== undefined && this.bots[v]!.indexOf(uname) !== -1);
841
  }
842
843
844
845
846
847
848
849
850
851

  /**
   * Check to ensure that a given namespace is within the target group namespace.
   *
   * @param namespace the namespace to check
   * @returns true if the namespace is under the configured root group, false otherwise.
   */
  withinNamespace(namespace: string):boolean {
    return namespace.startsWith(this.config.rootGroup + '/');
  }
852
}