Commit 0dcc99af authored by Martin Lowe's avatar Martin Lowe 🇨🇦
Browse files

Iss #216 - Add test environment setup script

parent d3a0b02e
......@@ -12,6 +12,7 @@
"pretest": "eslint --ignore-path .gitignore .",
"test": "mocha --timeout 60000 && mocha --config=./test/ts/.mocharc.json --timeout 60000",
"lab-sync": "ts-node -v && ts-node src/scripts/gl/GitlabSync.ts",
"lab-env": "ts-node -v && ts-node src/scripts/gl/TestEnvironment.ts",
"import-backup": "node src/auto_backup/Import.js"
},
"license": "EPL-2.0",
......
......@@ -36,12 +36,12 @@ interface GroupCache {
children: Record<string, GroupCache>;
}
interface EclipseUserAccess {
export interface EclipseUserAccess {
url: string;
accessLevel: AccessLevel;
}
interface GitlabSyncRunnerConfig {
export interface GitlabSyncRunnerConfig {
host: string;
provider: string;
secretLocation?: string;
......@@ -238,15 +238,20 @@ export class GitlabSyncRunner {
async prepareCaches() {
// get raw project data and post process to add additional context
try {
this.logger.info('Populating projects cache');
const data = await this.eApi.eclipseAPI();
// get the bots for the projects
this.logger.info('Populating bots cache');
const rawBots = await this.eApi.eclipseBots();
this.bots = this.eApi.processBots(rawBots, 'gitlab.eclipse.org');
// get all current groups for the instance
this.logger.info('Populating Gitlab projects cache');
this.projectsCache = await this.api.Projects.all();
this.logger.info('Populating Gitlab groups cache');
const groups = await this.api.Groups.all();
this.logger.info('Populating Gitlab users cache');
const users = await this.api.Users.all();
// generates the nested cache
......@@ -625,7 +630,7 @@ export class GitlabSyncRunner {
}
this.gMems[group.id] = members;
}
return members;
return [...members];
}
/** HELPERS */
......@@ -654,11 +659,14 @@ export class GitlabSyncRunner {
/**
* @returns the root group cache for the current sync operation if it exists. If missing, the script ends processing.
*/
getRootGroup(): GroupCache {
getRootGroup(endProcessing = true): GroupCache {
const rootGroupCache = this.groupCache.children[this.config.rootGroup];
if (rootGroupCache === undefined) {
this.logger.error(`Could not find root group '${this.config.rootGroup}' for group caching, exiting`);
process.exit(1);
if (endProcessing) {
this.logger.error(`Could not find root group '${this.config.rootGroup}' for group caching, exiting`);
process.exit(1);
}
throw new Error(`Could not find root group '${this.config.rootGroup}' for root group fetch`);
}
return rootGroupCache;
}
......
/** ***************************************************************
Copyright (C) 2022 Eclipse Foundation, Inc.
This program and the accompanying materials are made
available under the terms of the Eclipse Public License 2.0
which is available at https://www.eclipse.org/legal/epl-2.0/
Contributors:
Martin Lowe <martin.lowe@eclipse-foundation.org>
SPDX-License-Identifier: EPL-2.0
******************************************************************/
import yargs from 'yargs';
const args = yargs(process.argv)
.usage('Usage: $0 [options]')
.example('$0', '')
.option('H', {
alias: 'host',
description: 'GitLab host base URL',
default: 'http://gitlab.eclipse.org/',
string: true,
})
.option('s', {
alias: 'secretLocation',
description: 'The location of the access-token file containing an API access token',
string: true,
}).argv;
import { AccessLevel } from '@gitbeaker/core/dist/types/types';
import { UserSchema } from '@gitbeaker/core/dist/types/types';
import { Logger } from 'winston';
import { getLogger } from '../../helpers/logger';
import { GitlabSyncRunner } from './GitlabSyncRunner';
// used to split project name from group namespace
const PROJECT_SPLIT_REGEX = /^(.*)\/([^/]+)$/gm;
// These values are pulled from the AccessLevel type, as it isn't typesafe otherwise
// eslint-disable-next-line no-magic-numbers
const accessLevelComparable: ReadonlyArray<number> = [0, 5, 10, 20, 30, 40, 50] as const;
const isAccessLevel = (x: unknown): x is AccessLevel => {
if (!x || typeof x !== 'number') {
return false;
}
return accessLevelComparable.includes(x as number);
};
// simple sample project/group structure to create
enum SampleType {
project = 'project',
group = 'group',
}
interface EclipseUserAccess {
url: string;
accessLevel: number;
}
interface SampleStructure {
namespace: string;
users: Record<string, EclipseUserAccess>;
type: SampleType;
}
import * as projectsRaw from './test_environment.json';
const sampleStructures = <SampleStructure[]>projectsRaw;
export class TestEnvironment {
syncRunner: GitlabSyncRunner;
logger: Logger;
constructor({ secretLocation, host }: Record<string, string>) {
this.syncRunner = new GitlabSyncRunner({
devMode: false,
host: host,
dryRun: false,
provider: 'oauth2_generic',
verbose: true,
project: null,
secretLocation: secretLocation,
staging: false,
});
this.logger = getLogger('debug', 'main');
}
/**
* Run the test environment build script.
*/
async run(): Promise<void> {
await this.syncRunner.prepareCaches();
// ensure the needed root group exists
await this.createRootGroup('eclipse');
for (const sampleStructureIdx in sampleStructures) {
const sampleStructure = sampleStructures[sampleStructureIdx];
switch (sampleStructure.type) {
case SampleType.group:
await this.handleSampleGroup(sampleStructure);
break;
case SampleType.project:
await this.handleSampleProject(sampleStructure);
break;
}
}
}
/**
* Builds the sample group with users set in the sample structure element.
*
* @param sampleStructure the group structure to be created.
* @returns an async process for adding the group.
*/
async handleSampleGroup(sampleStructure: SampleStructure): Promise<void> {
// get the actual group
const projectGroup = await this.syncRunner.getCachedGroup(sampleStructure.namespace);
if (projectGroup === null || projectGroup._self === null) {
this.logger.error(`Could not find group with namespace ${projectGroup}`);
return;
}
// Auto format breaks this line, so added rule ignore
// eslint-disable-next-line space-before-function-paren
await this.userAction(sampleStructure.users, async (user, accessLevel) => {
this.logger.verbose(`Adding user ${user.id} to group ${projectGroup._self!.path}`);
await this.syncRunner.addUserToGroup(user, projectGroup._self!, accessLevel);
});
}
async handleSampleProject(sampleStructure: SampleStructure): Promise<void> {
// get the matching project if it exists
const matches = this.syncRunner.projectsCache.find(v => v.path_with_namespace.localeCompare(sampleStructure.namespace) !== 0);
let projectId: number;
if (!matches) {
this.logger.info(`No match found for ${sampleStructure.namespace}, creating new project`);
try {
// create project, splitting the designated namespace into parts
const results = PROJECT_SPLIT_REGEX.exec(sampleStructure.namespace);
if (!results || !results[1] || !results[2]) {
this.logger.error(`Could not extract a group from project namespace, skipping '${sampleStructure.namespace}'`);
return;
}
const project = await this.syncRunner.api.Projects.create({
namespace_id: (await this.syncRunner.getCachedGroup(results[1]))._self.id,
name: results[2],
});
projectId = project.id;
} catch (e) {
this.logger.error(`Error while creating project '${sampleStructure.namespace}': ${e}`);
return;
}
} else {
projectId = matches.id;
}
if (projectId === undefined) {
this.logger.error('no set project ID, skipping');
return;
}
// Auto format breaks this line, so added rule ignore
// eslint-disable-next-line space-before-function-paren
this.userAction(sampleStructure.users, async (user, accessLevel) => {
this.logger.verbose(`Adding user ${user.id} to project ${projectId}`);
await this.syncRunner.api.ProjectMembers.add(projectId, user.id, accessLevel);
});
}
/**
* Performs a user action for each user present in the users record. Will convert the user records to a UserSchema from Gitlab
* and then perform the callback, along with the users access level.
*
* @param users the users to action on using the given callback
* @param callback the action to perform for each user
*/
async userAction(users: Record<string, EclipseUserAccess>, callback: (user: UserSchema, accessLevel: AccessLevel) => Promise<void>) {
const usernames = Object.keys(users);
for (const usernameIdx in usernames) {
const uname = usernames[usernameIdx];
const user = await this.syncRunner.getUser(uname, users[uname].url);
if (user === null) {
this.logger.verbose(`Could not retrieve user for UID '${uname}', skipping`);
continue;
}
// type check the access level
const accessLevel = users[uname].accessLevel;
if (!isAccessLevel(accessLevel)) {
this.logger.warn(`Invalid access level of '${accessLevel}' passed, skipping adding user to entity`);
continue;
}
// call the user action with the cast access level
try {
await callback(user, accessLevel);
} catch (e) {
this.logger.warn(`Error encountered during user action callback, some users may be out of sync: ${e}`);
}
}
}
/**
* Creates root group (no parent group) in the target Gitlab instance.
*
* @param name the path and name of the Gitlab root level group to create.
* @returns the group if it can be created
*/
async createRootGroup(name: string): Promise<void> {
try {
const rootGroup = this.syncRunner.getRootGroup(false);
if (rootGroup !== null && rootGroup._self !== null) {
this.logger.info(`Root group '${name}' already exists, not creating group`);
return;
}
} catch (e) {
this.logger.info(`Creating root group '${name}'`);
}
try {
await this.syncRunner.api.Groups.create(name, name, {
project_creation_level: 'maintainer',
visibility: 'public',
request_access_enabled: false,
});
} catch (e) {
this.logger.error(`Could not create root group with name '${name}'`);
process.exit(1);
}
}
}
// run the test environment creation script
new TestEnvironment({ secretLocation: args['s'], host: args['h'] }).run();
[
{
"namespace": "eclipse/spider.pig/excludes",
"type": "group",
"users": {
"zacharysabourin": { "accessLevel": 20, "url": "https://api.eclipse.org/account/profile/zacharysabourin" }
}
},
{
"namespace": "eclipse/spider.pig/excludes/sample-excluded-project",
"type": "project",
"users": {
"zacharysabourin": { "accessLevel": 30, "url": "https://api.eclipse.org/account/profile/zacharysabourin" }
}
},
{
"namespace": "eclipse/spider.pig/sample-standard-project",
"type": "project",
"users": {
"zacharysabourin": { "accessLevel": 30, "url": "https://api.eclipse.org/account/profile/zacharysabourin" },
"epoirier": { "accessLevel": 40, "url": "https://api.eclipse.org/account/profile/epoirier" }
}
},
{
"namespace": "eclipse/technology/dash",
"type": "group",
"users": {
"epoirier": { "accessLevel": 30, "url": "https://api.eclipse.org/account/profile/epoirier" },
"malowe": { "accessLevel": 40, "url": "https://api.eclipse.org/account/profile/malowe" }
}
},
{
"namespace": "eclipse/technology/dash/sample-project",
"type": "project",
"users": {
"zacharysabourin": { "accessLevel": 30, "url": "https://api.eclipse.org/account/profile/zacharysabourin" }
}
}
]
{
"compilerOptions": {
"esModuleInterop": true,
"sourceMap": true
"sourceMap": true,
"resolveJsonModule": true
},
"include": [
"src/**/*"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment