GitlabSyncRunner.ts 28.9 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
/** ***************************************************************
 Copyright (C) 2022 Eclipse Foundation, Inc.

 This program and the accompanying materials are made
 available under the terms of the Eclipse Public License 2.0
 which is available at https://www.eclipse.org/legal/epl-2.0/

  Contributors:
    Martin Lowe <martin.lowe@eclipse-foundation.org>

 SPDX-License-Identifier: EPL-2.0
******************************************************************/

14
import { Logger } from 'winston';
15
import { Resources } from '@gitbeaker/core/dist/types';
16
17
import { AccessLevel, GroupSchema, MemberSchema, ProjectSchema, UserSchema } from '@gitbeaker/core/dist/types/types';
import { v4 } from 'uuid';
18
19
20
21
import { EclipseAPI, EclipseApiConfig } from '../../eclipse/EclipseAPI';
import { getLogger } from '../../helpers/logger';
import { SecretReader, getBaseConfig } from '../../helpers/SecretReader';
import { EclipseProject, EclipseUser } from '../../interfaces/EclipseApi';
22
23

// used to make use of default requested based on Got rather than recreating our own
24
25
import { Gitlab } from '@gitbeaker/core';
import { requesterFn } from './AxiosRequester';
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

const ADMIN_PERMISSIONS_LEVEL = 50;

/**
 * Represents the nested group cache that can represent the relationships between groups and to simplify child lookups.
 */
interface GroupCache {
  _self: GroupSchema | null;
  projectTargets: string[];
  children: Record<string, GroupCache>;
}

interface EclipseUserAccess {
  url: string;
  accessLevel: AccessLevel;
}

interface GitlabSyncRunnerConfig {
  host: string;
  provider: string;
  secretLocation?: string;
  project?: string;
  verbose: boolean;
  devMode: boolean;
  dryRun: boolean;
51
  rootGroup?: string;
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
}

export class GitlabSyncRunner {
  // internal state
  accessToken: string = '';
  eclipseToken: string = '';
  config: GitlabSyncRunnerConfig;
  logger: Logger;

  // api access
  api: Resources.Gitlab;
  eApi: EclipseAPI;
  bots: Record<string, string[]> = {};

  // caches to optimize calling
  namedUsers: Record<string, UserSchema> = {};
  groupCache: GroupCache = {
    _self: null,
    children: {},
    projectTargets: [],
  };
73
  projectsCache: ProjectSchema[] = [];
74
75
76
  eclipseProjectCache: Record<string, EclipseProject> = {};
  gMems: Record<number, MemberSchema[]> = {};

77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
  /**
   * Sets the internal config with a few default values and creates the bindings for the APIs that are
   * accessed during the run of this script.
   *
   * @param config the initial script configuration object.
   */
  constructor(config: GitlabSyncRunnerConfig) {
    this.config = Object.assign(
      {
        host: 'http://gitlab.eclipse.org/',
        provider: 'oauth2_generic',
        verbose: false,
        devMode: false,
        dryRun: false,
        rootGroup: 'eclipse',
      },
      config
    );

96
97
98
99
100
101
102
    this.logger = getLogger(this.config.verbose ? 'debug' : 'info', 'main');
    this._prepareSecret();

    // create API instances
    this.api = new Gitlab({
      host: this.config.host,
      token: this.accessToken,
103
      requesterFn: requesterFn,
104
105
106
107
108
109
110
    });
    let eclipseAPIConfig: EclipseApiConfig = JSON.parse(this.eclipseToken);
    eclipseAPIConfig.testMode = this.config.devMode;
    eclipseAPIConfig.verbose = this.config.verbose;
    this.eApi = new EclipseAPI(eclipseAPIConfig);
  }

111
112
113
114
115
  /**
   * Prepares the secrets required for the script to run. Specifically the eclipseToken used for Eclipse
   * API access and the accessToken which is used for sudo+api access on the Gitlab instance targeted by
   * this script.
   */
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
  _prepareSecret() {
    // retrieve the secret API file root if set
    var settings = getBaseConfig();
    if (this.config.secretLocation !== undefined) {
      settings.root = this.config.secretLocation;
    }
    var reader = new SecretReader(settings);
    var data = reader.readSecret('access-token');
    if (data !== null) {
      this.accessToken = data.trim();
      // retrieve the Eclipse API token (needed for emails)
      data = reader.readSecret('eclipse-oauth-config');
      if (data !== null) {
        this.eclipseToken = data.trim();
      } else {
        this.logger.error('Could not find the Eclipse OAuth config, exiting');
        process.exit(1);
      }
    } else {
      this.logger.error('Could not find the GitLab access token, exiting');
      process.exit(1);
    }
  }

140
141
142
143
144
145
146
147
148
  /**
   * Run the full sync script, syncing the PMI to the Gitlab instance targeted by the script. This script
   * will sync the namespace groups named in projects with the users that are set as members of the project.
   * It will also clear users added outside of this process with the exception of bot users to maintain more
   * strict control of the access permissions.
   *
   * @returns a promise that is completed once the run completes.
   */
  async run(): Promise<void> {
149
150
151
152
153
    // prepopulate caches to optimally retrieve info used in sync ops
    await this.prepareCaches();

    // fetch org group from results, create if missing
    this.logger.info('Starting sync');
154
155
156
    var g = this.getRootGroup();
    if (g._self === null) {
      this.logger.error(`Unable to start sync of GitLab content. Base group (${this.config.rootGroup}) could not be found`);
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
      return;
    }

    for (let projectIdx in this.eclipseProjectCache) {
      var project = this.eclipseProjectCache[projectIdx];
      if (this.config.project !== undefined && project.short_project_id !== this.config.project) {
        this.logger.info(`Project target set ('${this.config.project}'). Skipping non-matching project ID ${project.short_project_id}`);
        continue;
      }
      this.logger.info(`Processing '${project.short_project_id}'`);

      // get the list of users to be added for current project
      var userList = this.getUserList(project);
      // for each user, get their gitlab user and add to the project group
      var usernames = Object.keys(userList);

      // fetch group namespaces indicated by the project
      for (let idx in project.gitlab_repos) {
        let [host, namespace] = this.splitNamespaceUrl(project.gitlab_repos[idx].url);
        // make sure namespace URL is valid
        if (host === null || namespace === null) {
178
          this.logger.error(`Could not generate namespace/host from namespace URL: ${project.gitlab_repos[idx].url}`);
179
180
        }
        // check if hosts are the same ignoring case, skipping if they are different
181
182
        if (host.localeCompare(new URL(this.config.host).hostname, undefined, { sensitivity: 'base' }) !== 0) {
          this.logger.error(`Found host '${host}' when processing for '${new URL(this.config.host).hostname}', skipping`);
183
184
185
186
187
188
          continue;
        }

        // check group cache to ensure well formed.
        let namespaceGroup = this.getCachedGroup(namespace);
        if (namespaceGroup === null || namespaceGroup._self === null) {
189
          this.logger.error(`Could not find group with namespace ${namespace}`);
190
191
192
193
194
195
196
197
198
199
200
201
202
          continue;
        }
        // update the group to add the users for the current project
        for (var usernameIdx in usernames) {
          var uname = usernames[usernameIdx];
          var user = await this.getUser(uname, userList[uname].url);
          if (user === null) {
            this.logger.verbose(`Could not retrieve user for UID '${uname}', skipping`);
            continue;
          }

          await this.addUserToGroup(user, namespaceGroup._self!, userList[uname].accessLevel);
          // if not tracked, track current project for group for post-sync cleanup
203
          if (namespaceGroup.projectTargets.indexOf(project.short_project_id) === -1) {
204
205
206
207
208
            namespaceGroup.projectTargets.push(project.short_project_id);
          }
        }
      }
    }
209
    // perform cleanup operations to clean out extra users
210
    this.cleanupGroups();
211
    this.cleanupProjects();
212
213
214
215
216
217
218
  }

  /**
   * Generates the caches needed for running the Gitlab sync process.
   */
  async prepareCaches() {
    // get raw project data and post process to add additional context
219
220
    try {
      let data = await this.eApi.eclipseAPI();
221

222
223
224
      // get the bots for the projects
      let rawBots = await this.eApi.eclipseBots();
      this.bots = this.eApi.processBots(rawBots, 'gitlab.eclipse.org');
225

226
227
228
229
      // get all current groups for the instance
      this.projectsCache = await this.api.Projects.all();
      var groups = await this.api.Groups.all();
      var users = await this.api.Users.all();
230

231
232
233
234
235
236
237
238
239
240
241
242
243
      // generates the nested cache
      this.generateGroupsCache(groups);
      // map the users to their usernames for easy lookups
      for (var userIdx in users) {
        this.namedUsers[users[userIdx].username] = users[userIdx];
      }
      for (let projectIdx in data) {
        let p = data[projectIdx];
        this.eclipseProjectCache[p.short_project_id] = p;
      }
    } catch (e) {
      this.logger.error(`Cannot fetch resources associated with sync operations, exiting: ${e}`);
      process.exit(1);
244
245
246
247
248
249
250
    }
  }

  /**
   * Iterate through each group, checking self and ancestor project users and comparing against the current groups users to ensure that there are no
   * additional users added with permissions.
   */
251
252
  cleanupGroups(currentLevel: GroupCache = this.getRootGroup(), collectedProjects: string[] = []) {
    this.logger.debug(`cleanupGroups(currentLevel = ${currentLevel._self?.full_path}, collectedProjects = ${collectedProjects})`);
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
    let self = currentLevel._self;
    if (self === null) {
      this.logger.error('Error encountered during group cleanup process, ending early');
      return;
    }
    // collect and deduplicate project IDs
    let projects = [...Array.from(new Set([...currentLevel.projectTargets, ...collectedProjects]))];
    // build the user mapping to pass to cleanup
    let projectUsers: Record<string, EclipseUser> = {};
    for (let pidx in projects) {
      projectUsers = Object.assign(projectUsers, this.getUserList(this.eclipseProjectCache[projects[pidx]]));
    }
    // TODO we need to be able to pass multiple projects
    this.removeAdditionalUsers(projectUsers, self, ...projects);
    // for each of the children, pass the collected projects forward and process
    for (let cidx in currentLevel.children) {
      this.cleanupGroups(currentLevel.children[cidx], projects);
    }
  }

273
274
275
276
277
278
279
280
281
282
  /**
   * Removes users not tracked in the expectedUsers map from the passed group. Project IDs are used to look up bot user
   * account names as they are exempt from being removed as they are used for CI ops.
   *
   * @param expectedUsers map of usernames to their EclipseUser entry
   * @param group the Gitlab group that is being cleaned of extra users.
   * @param projectIDs list of project IDs that impact the given group
   * @returns a promise that completes once all additional users are removed or the check finishes
   */
  async removeAdditionalUsers(expectedUsers: Record<string, EclipseUser>, group: GroupSchema, ...projectIDs: string[]): Promise<void> {
283
    if (this.config.verbose) {
284
285
286
287
288
      this.logger.debug(
        `GitlabSync:removeAdditionalUsers(expectedUsers = ${JSON.stringify(expectedUsers)}, group = ${
          group.full_path
        }, projectIDs = ${projectIDs})`
      );
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
    }
    // get the current list of users for the group
    var members = await this.getGroupMembers(group);
    if (members === null) {
      this.logger.warn(`Could not find any group members for ID ${group.id}'. Skipping user removal check`);
      return;
    }

    // check that each of the users in the group match whats expected
    var expectedUsernames = Object.keys(expectedUsers);
    members!.forEach(async member => {
      // check access and ensure user isn't an owner
      this.logger.verbose(`Checking user '${member.username}' access to group '${group.name}'`);
      if (
        member.access_level !== ADMIN_PERMISSIONS_LEVEL &&
        expectedUsernames.indexOf(member.username) === -1 &&
        !this.isBot(member.username, projectIDs)
      ) {
        if (this.config.dryRun) {
          this.logger.info(`Dryrun flag active, would have removed user '${member.username}' from group '${group.name}'`);
          return;
        }
        this.logger.info(`Removing user '${member.username}' from group '${group.name}'`);
        try {
          await this.api.GroupMembers.remove(group.id, member.id);
        } catch (err) {
          if (this.config.verbose) {
            this.logger.error(`${err}`);
          }
          this.logger.warn(`Error while removing user '${member.username}' from group '${group.name}'`);
        }
      }
    });
  }

324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
  /**
   * Iterates over the projects cache and cleans out the users and keeps bots for build operations. Skips over projects
   * outside the scope of the designated root group to avoid over processing groups.
   */
  async cleanupProjects() {
    this.projectsCache.forEach(p => {
      let group = this.getCachedGroup(p.namespace.full_path);
      if (group !== null) {
        this.cleanUpProjectUsers(p, ...group!.projectTargets);
      } else {
        this.logger.info(`Skipping processing of project '${p.name}'`);
      }
    });
  }

  /**
   * Removes any non-owner user that isn't a bot from projects. Membership is managed at the group level, not the direct
   * project level.
   *
   * @param project the Gitlab project to sanitize
   * @param projectIDs the Eclipse projects that impact the Gitlab project.
   */
  async cleanUpProjectUsers(project: ProjectSchema, ...projectIDs: string[]) {
347
348
349
350
351
352
353
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:cleanUpProjectUsers(project = ${project.id})`);
    }
    var projectMembers = await this.api.ProjectMembers.all(project.id, { includeInherited: false });
    for (var idx in projectMembers) {
      let member = projectMembers[idx];
      // skip bot user or admin users
354
      if (this.isBot(member.username, projectIDs) || member.access_level === ADMIN_PERMISSIONS_LEVEL) {
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
        continue;
      }
      if (this.config.dryRun) {
        this.logger.debug(`Dryrun flag active, would have removed user '${member.username}' from project '${project.name}'(${project.id})`);
        continue;
      }
      this.logger.info(`Removing user '${member.username}' from project '${project.name}'(${project.id})`);
      try {
        await this.api.ProjectMembers.remove(project.id, member.id);
      } catch (err) {
        if (this.config.verbose) {
          this.logger.error(`${err}`);
        }
        this.logger.error(`Error while removing user '${member.username}' from project '${project.name}'(${project.id})`);
      }
    }
  }

373
374
375
376
377
378
379
380
381
  /**
   * Ensures that the user exists within the group with the given access level (no more or less). If a user has too high
   * permissions, the membership is modified to have the given access instead.
   *
   * @param user the user that is being given permissions
   * @param group group that the user should be added to
   * @param perms the permission set to give the user
   * @returns the membership information for the user wrt to this Gitlab group.
   */
382
383
  async addUserToGroup(user: UserSchema, group: GroupSchema, perms: AccessLevel): Promise<MemberSchema | null> {
    if (this.config.verbose) {
384
      this.logger.debug(`GitlabSync:addUserToGroup(user = ${user?.username}, group = ${group?.full_path}, perms = ${perms})`);
385
386
387
388
389
390
391
392
393
    }
    // get the members for the current group
    var members = await this.getGroupMembers(group);
    if (members === null) {
      this.logger.warn(`Could not find any references to group with ID ${group.id}`);
      return null;
    }

    // check if user is already present
394
395
    for (let i = 0; i < members.length; i++) {
      let member = members[i];
396
397
398
399
400
401
402
403
404
405
406
407
408
409
      if (member.username === user.username) {
        this.logger.verbose(`User '${user.username}' is already a member of ${group.name}`);
        if (member.access_level !== perms) {
          // skip if dryrun
          if (this.config.dryRun) {
            this.logger.info(`Dryrun flag active, would have updated user '${member.username}' in group '${group.name}'`);
            return null;
          }

          // modify user, catching errors
          this.logger.info(`Fixing permission level for user '${user.username}' in group '${group.name}'`);
          try {
            var updatedMember = await this.api.GroupMembers.edit(group.id, user.id, perms);
            // update inner array
410
            members![i] = updatedMember;
411
412
413
414
415
416
417
418
419
420
            this.gMems[group.id] = members!;
          } catch (err) {
            if (this.config.verbose) {
              this.logger.error(`${err}`);
            }
            this.logger.warn(`Error while fixing permission level for user '${user.username}' in group '${group.name}'`);
            return null;
          }
        }
        // return a copy of the updated user
421
        return members![i];
422
      }
423
    }
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
    // check if dry run before updating
    if (this.config.dryRun) {
      this.logger.info(
        `Dryrun flag active, would have added user '${user.username}' to group '${group.name}' with access level '${perms}'`
      );
      return null;
    }

    this.logger.info(`Adding '${user.username}' to '${group.name}' group`);
    try {
      // add member to group, track, and return a copy
      var newMember = await this.api.GroupMembers.add(group.id, user.id, perms);
      members.push(newMember);
      this.gMems[group.id] = members;

      // return a copy
      return newMember;
    } catch (err) {
      if (this.config.verbose) {
        this.logger.error(`${err}`);
      }
      this.logger.warn(`Error while adding '${user.username}' to '${group.name}' group`);
    }
    return null;
  }

450
451
452
453
454
455
456
457
  /**
   * Retrieves a Gitlab user object for the given Eclipse user given their username and access URL. If the
   * user does not yet exist, a new user is created, cached, and returned for use.
   *
   * @param uname the Eclipse username of user to retrieve from Gitlab
   * @param url the Eclipse user access URL
   * @returns the gitlab user, or null if it can't be found or created.
   */
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
  async getUser(uname: string, url: string): Promise<UserSchema | null> {
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:getUser(uname = ${uname}, url = ${url})`);
    }
    if (url === undefined || url === '') {
      this.logger.error(`Cannot fetch user information for user '${uname}' with no set URL`);
      return null;
    }

    var u = this.namedUsers[uname];
    if (u === undefined) {
      if (this.config.dryRun) {
        this.logger.info(`Dryrun is enabled. Would have created user ${uname} but was skipped`);
        return null;
      }

      // retrieve user data
      var data = await this.eApi.eclipseUser(uname);
      if (data === null) {
        this.logger.error(`Cannot create linked user account for '${uname}', no external data found`);
        return null;
      }
      this.logger.verbose(`Creating new user with name '${uname}'`);
      var opts = {
        username: uname,
        password: v4(),
        force_random_password: true,
        name: `${data!.first_name} ${data!.last_name}`,
        email: data!.mail,
        extern_uid: data!.uid,
        provider: this.config.provider,
        skip_confirmation: true,
      };
      // check if dry run before creating new user
      if (this.config.dryRun) {
        this.logger.info(`Dryrun flag active, would have created new user '${uname}' with options ${JSON.stringify(opts)}`);
        return null;
      }

      // if verbose, display information being used to generate user
      if (this.config.verbose) {
        // copy the object and redact the password for security
        var optLog = JSON.parse(JSON.stringify(opts));
        optLog.password = 'redacted';
        this.logger.debug(`Creating user with options: ${JSON.stringify(optLog)}`);
      }
      try {
        u = await this.api.Users.create(opts);
      } catch (err) {
        if (this.config.verbose) {
          this.logger.error(`${err}`);
        }
      }
      if (u === null) {
        this.logger.warn(`Error while creating user '${uname}'`);
        return null;
      }
      // set it back
      this.namedUsers[uname] = u;
    }
    return u;
  }

521
522
523
524
525
526
  /**
   * Retrieves the list of direct members for a given group, ignoring inherited users.
   *
   * @param group the Gitlab group to retrieve members for
   * @returns a list of Gitlab group members for the given group, or null if there is an error while fetching.
   */
527
528
  async getGroupMembers(group: GroupSchema): Promise<MemberSchema[] | null> {
    if (this.config.verbose) {
529
      this.logger.debug(`GitlabSync:getGroupMembers(group = ${group?.full_path})`);
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
    }
    var members = this.gMems[group.id];
    if (members === undefined) {
      try {
        members = await this.api.GroupMembers.all(group.id, { includeInherited: false });
      } catch (err) {
        if (this.config.verbose) {
          this.logger.error(`${err}`);
        }
      }
      if (members === null) {
        this.logger.warn(`Unable to find group members for group with ID '${group.id}'`);
        return null;
      }
      this.gMems[group.id] = members;
    }
    return members;
  }

  /** HELPERS */

551
552
553
554
  /**
   * Generate the nested group cache using the raw Gitlab group definitions.
   * @param rawGroups
   */
555
556
557
558
559
560
561
562
563
564
565
566
567
  generateGroupsCache(rawGroups: GroupSchema[]): void {
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:generateGroupsCache(projects = count->${rawGroups.length})`);
    }
    // create initial cache container
    this.groupCache = {
      _self: null,
      projectTargets: [],
      children: {},
    };

    // iterate through groups and insert into the nested cache
    for (let i = 0; i < rawGroups.length; i++) {
568
      this.addGroup(rawGroups[i]);
569
570
571
    }
  }

572
573
574
  /**
   * @returns the root group cache for the current sync operation if it exists. If missing, the script ends processing.
   */
575
576
577
578
579
580
581
582
583
  getRootGroup(): GroupCache {
    let rootGroupCache = this.groupCache.children[this.config.rootGroup];
    if (rootGroupCache === undefined) {
      this.logger.error(`Could not find root group '${this.config.rootGroup}' for group caching, exiting`);
      process.exit(1);
    }
    return rootGroupCache;
  }

584
585
586
587
588
589
590
591
  /**
   * Retrieves the group for the given namespace path. This namespace path should be formatted such that each group path is separated
   * by a slash, eg. eclipse/sample/group. This will be split and used to iterate through the nested cache, returning the group once
   * each namespace path part is used.
   *
   * @param namespace the full path of the group namespace to retrieve.
   * @returns the group cache for the group indicated by the namespace string, or null if there is no matching group.
   */
592
593
594
595
  getCachedGroup(namespace: string): GroupCache | null {
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:getCachedGroup(${namespace})`);
    }
596
597
598
599
    if (!namespace.startsWith(this.config.rootGroup)) {
      this.logger.info(`Returning null for ${namespace} as it is outside of the root group ${this.config.rootGroup}`);
      return null;
    }
600
601
602
    return this.tunnelAndRetrieve(namespace.split('/'), this.groupCache);
  }

603
604
605
606
607
608
609
610
  /**
   * Adds a group to the nested group cache, using the groups full_path property to discover how to insert the
   * entry into the nested cache. Any cache nodes that do not exist yet will be created as the group is inserted.
   *
   * @param g the Gitlab group that is being inserted into the group cache.
   * @returns the group cache entry for the cached gitlab group
   */
  addGroup(g: GroupSchema): GroupCache {
611
612
613
614
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:addGroup(g = ${g.id})`);
    }
    let namespace = g.full_path;
615
616
617
618
619
    // split into group namespace paths (eclipse/sample/group.path into ['eclipse','sample','group.path'])
    let namespaceParts = namespace.split('/');
    return this.tunnelAndInsert(namespaceParts, g, this.groupCache);
  }

620
621
622
623
624
625
626
627
628
629
  /**
   * Recursive function for inserting groups into the nested group cache. Will tunnel through cache, creating
   * entries as necessary before inserting the group at the nesting level representing the final group in the
   * full path of the group.
   *
   * @param namespaceParts the parts of the full_path left to process for group nesting
   * @param g  the group to be inserted into the group cache.
   * @param parent the parent level for the current level of insertion
   * @returns the group cache entry for the group once inserted.
   */
630
  tunnelAndInsert(namespaceParts: string[], g: GroupSchema, parent: GroupCache): GroupCache {
631
632
633
634
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:tunnelAndInsert(namespaceParts = '${namespaceParts}', g = ${g.id})`);
    }

635
    // get the next level cache if it exists, creating it if it doesn't
636
637
638
639
640
641
642
643
644
645
646
    let child = parent.children[namespaceParts[0]];
    if (child === undefined) {
      child = {
        _self: null,
        projectTargets: [],
        children: {},
      };
      parent.children[namespaceParts[0]] = child;
    }
    // check if we should continue tunneling or insert and finish processing
    if (namespaceParts.length > 1) {
647
      return this.tunnelAndInsert(namespaceParts.slice(1, namespaceParts.length), g, child);
648
649
650
651
652
653
    } else {
      child._self = g;
      return child;
    }
  }

654
655
656
657
658
659
660
661
  /**
   * Recursive access to the nested group cache. Retrieves the group described by the namespace parts and returns
   * it, returning null if it can't be found.
   *
   * @param namespaceParts the full path for a group namespace split into parts
   * @param parent the parent to search through for the next part of the recursive call.
   * @returns The group cache for the designated group, or null if it can't be found.
   */
662
  tunnelAndRetrieve(namespaceParts: string[], parent: GroupCache): GroupCache | null {
663
664
665
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:tunnelAndRetrieve(namespaceParts = '${namespaceParts}')`);
    }
666
667
668
669
670
671
    let child = parent.children[namespaceParts[0]];
    if (child === undefined) {
      return null;
    }
    // check if we should continue tunneling or insert and finish processing
    if (namespaceParts.length > 1) {
672
      return this.tunnelAndRetrieve(namespaceParts.slice(1, namespaceParts.length), child);
673
674
675
676
677
    } else {
      return child;
    }
  }

678
679
680
681
682
683
  /**
   * Gets list of users with access permissions for the given Eclipse project.
   *
   * @param project the Eclipse project to parse user entries for
   * @returns the mapping of users to access permissions and entity access URL.
   */
684
685
  getUserList(project: EclipseProject): Record<string, EclipseUserAccess> {
    if (this.config.verbose) {
686
      this.logger.debug(`GitlabSync:getUserList(project = ${project.short_project_id})`);
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
    }
    var l: Record<string, EclipseUserAccess> = {};
    // add the contributors with reporter access
    project.contributors.forEach(v => {
      l[v.username] = {
        url: v.url,
        accessLevel: 20,
      };
    });
    // add the committers with developer access
    project.committers.forEach(v => {
      l[v.username] = {
        url: v.url,
        accessLevel: 30,
      };
    });
    // add the project leads not yet tracked with reporter access
    project.project_leads.forEach(v => {
      l[v.username] = {
        url: v.url,
        accessLevel: 40,
      };
    });
    // add the bots with developer access
    var botList = this.bots[project.project_id];
    if (botList !== undefined && botList.length === 0) {
      botList.forEach(v => {
        l[v] = {
          url: '',
          accessLevel: 30,
        };
      });
    }
    return l;
  }

723
724
725
726
727
728
  /**
   * Sanitizes and normalizes strings for use in creating/accessing groups.
   *
   * @param pid the project ID to normalize
   * @returns normalized group name for value.
   */
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
  sanitizeGroupName(pid: string): string {
    if (this.config.verbose) {
      this.logger.debug(`GitlabSync:sanitizeGroupName(pid = ${pid})`);
    }
    if (pid !== undefined) {
      return pid.toLowerCase().replace(/[^\s\da-zA-Z-.]/g, '-');
    }
    return '';
  }

  /**
   * Uses TS URL type to ingest a namespace URL and return the host and namespace for use downstream
   *
   * @param rawUrl the raw namespace URL to check and split.
   * @returns the host first, and the path of the url second which should be the namespace.
   */
  splitNamespaceUrl(rawUrl: string): [string | null, string | null] {
    try {
      let url = new URL(rawUrl);
748
      return [url.host, url.pathname.substring(1, url.pathname.length)];
749
750
751
752
753
754
755
756
757
758
759
760
    } catch (e) {
      // cast and message with error
      let message = '';
      if (typeof e === 'string') {
        message = e.toUpperCase();
      } else if (e instanceof Error) {
        message = e.message;
      }
      this.logger.error(`Could not convert URL (${rawUrl}) to namespace: ${message}`);
    }
    return [null, null];
  }
761
762
763
764
765
766
767
768

  /**
   * Checks whether a user is a bot for the given projects.
   *
   * @param uname potential bot username
   * @param projectIDs the projects that the user could be a bot for.
   * @returns true if the user is a designated bot for the projects, otherwise false.
   */
769
770
771
772
773
774
775
776
777
778
779
  isBot(uname: string, projectIDs: string[]): boolean {
    for (let pidx in projectIDs) {
      var botList = this.bots[projectIDs[pidx]];
      // check if the current user is in the current key-values list
      if (botList !== undefined && botList.indexOf(uname) !== -1) {
        return true;
      }
    }
    return false;
  }
}