Commit 7915ce9e authored by Martin Lowe's avatar Martin Lowe 🇨🇦
Browse files

Update GL sync to use new namespace fields

parent f53ce047
......@@ -13,10 +13,17 @@ const EXIT_ERROR_STATE = 1;
*/
export interface EclipseApiConfig {
oauth?: EclipseApiOAuthConfig;
webroots?: EclipseApiWebRoots;
staging?: boolean;
verbose?: boolean;
testMode?: boolean;
}
export interface EclipseApiWebRoots {
projects?: string;
api?: string;
}
/**
* Configuration interface for oauth/security binding.
*/
......@@ -36,7 +43,7 @@ export class EclipseAPI {
logger: Logger;
constructor(config: EclipseApiConfig) {
this.config = config;
this.config = Object.assign(this.generateDefaultConfigs(config.staging), config);
// generate creds if auth is set
if (this.config.oauth !== null && this.config.oauth !== undefined) {
this.client = new ClientCredentials({
......@@ -143,7 +150,7 @@ export class EclipseAPI {
/**
* Maps bot users by listing bots per project.
*
*
* @param botsRaw the raw bot definition list to convert to a project bot mapping.
* @param site the site targeted for bots to limit results
* @returns a mapping of project to configured bot usernames.
......@@ -226,11 +233,33 @@ export class EclipseAPI {
}
return null;
}
generateDefaultConfigs(useStaging: boolean = false): EclipseApiConfig {
if (useStaging) {
return {
webroots: {
projects: 'https://projects-staging.eclipse.org',
api: 'https://api-staging.eclipse.org',
},
testMode: false,
verbose: false,
};
} else {
return {
webroots: {
projects: 'https://projects.eclipse.org',
api: 'https://api.eclipse.org',
},
testMode: false,
verbose: false,
};
}
}
}
const testProjects: EclipseProject[] = [
{
project_id: 'spider.pig',
project_id: 'technology.spider.pig',
url: '',
website_repo: [],
website_url: '',
......@@ -239,6 +268,9 @@ const testProjects: EclipseProject[] = [
summary: 'Can he fly? No, hes a pig. Look out, here comes the spider pig',
logo: '',
tags: ['simpsons', 'doh', 'spider pig'],
top_level_project: 'technology',
gitlab_excl_namespaces: ['eclipse/spider.pig/excludes'],
gitlab_namespace: 'eclipse/spider.pig',
gerrit_repos: [
{
url: 'https://github.com/eclipsefdn-webdev/spider-pig',
......
......@@ -41,6 +41,8 @@ export interface EclipseProject {
website_repo: string[];
logo: string;
tags: string[];
gitlab_namespace: string;
gitlab_excl_namespaces: string[];
github_repos: Repo[];
gitlab_repos: Repo[];
gerrit_repos: Repo[];
......@@ -50,6 +52,7 @@ export interface EclipseProject {
working_groups: WorkingGroup[];
spec_project_working_group: WorkingGroup | Array<any>;
state: string;
top_level_project: string;
releases: Release[];
}
......
......@@ -62,6 +62,11 @@ let args = yargs(process.argv)
description: 'The location of the access-token file containing an API access token',
string: true,
})
.option('S', {
alias: 'staging',
description: 'Whether the API should target staging for edge features',
boolean: true,
})
.help('h')
.alias('h', 'help')
.version('0.1')
......@@ -82,5 +87,6 @@ async function run() {
verbose: argv.V,
project: argv.P,
secretLocation: argv.s,
staging: argv.S,
}).run();
}
......@@ -23,9 +23,10 @@ import { EclipseProject, EclipseUser } from '../../interfaces/EclipseApi';
// used to make use of default requested based on Got rather than recreating our own
import { Gitlab } from '@gitbeaker/core';
import { requesterFn } from './AxiosRequester';
import yargs from 'yargs';
const ADMIN_PERMISSIONS_LEVEL = 50;
const ALLOWLISTED_USERS: string[] = ['webmaster', 'root'];
/**
* Represents the nested group cache that can represent the relationships between groups and to simplify child lookups.
*/
......@@ -49,6 +50,7 @@ interface GitlabSyncRunnerConfig {
devMode: boolean;
dryRun: boolean;
rootGroup?: string;
staging?: boolean;
}
export class GitlabSyncRunner {
......@@ -163,46 +165,50 @@ export class GitlabSyncRunner {
this.logger.info(`Project target set ('${this.config.project}'). Skipping non-matching project ID ${project.short_project_id}`);
continue;
}
// fetch group namespace indicated by the project and ensure format
let actualNamespace = project.gitlab_namespace;
let [projectNamespace, projectNamespaceTLP] = [
`${this.config.rootGroup}/${project.short_project_id}`,
`${this.config.rootGroup}/${project.top_level_project}/${project.short_project_id}`,
];
if (actualNamespace === undefined || actualNamespace.trim() === '') {
this.logger.info(`Skipping project '${project.project_id}' as it has no Gitlab namespace`);
continue;
} else if (
actualNamespace.localeCompare(projectNamespace, undefined, { sensitivity: 'base' }) !== 0 &&
actualNamespace.localeCompare(projectNamespaceTLP, undefined, { sensitivity: 'base' }) !== 0
) {
this.logger.info(
`Skipping namespace '${actualNamespace}' for project '${project.short_project_id}', does not match allowed formats`
);
continue;
}
this.logger.info(`Processing '${project.short_project_id}'`);
// check group cache to ensure well formed.
let namespaceGroup = await this.getCachedGroup(actualNamespace);
if (namespaceGroup === null || namespaceGroup._self === null) {
this.logger.error(`Could not find group with namespace ${actualNamespace}`);
continue;
}
// get the list of users to be added for current project
var userList = this.getUserList(project);
// for each user, get their gitlab user and add to the project group
var usernames = Object.keys(userList);
// fetch group namespaces indicated by the project
for (let idx in project.gitlab_repos) {
let [host, namespace] = this.splitNamespaceUrl(project.gitlab_repos[idx].url);
// make sure namespace URL is valid
if (host === null || namespace === null) {
this.logger.error(`Could not generate namespace/host from namespace URL: ${project.gitlab_repos[idx].url}`);
}
// check if hosts are the same ignoring case, skipping if they are different
if (host.localeCompare(new URL(this.config.host).hostname, undefined, { sensitivity: 'base' }) !== 0) {
this.logger.error(`Found host '${host}' when processing for '${new URL(this.config.host).hostname}', skipping`);
// update the group to add the users for the current project
for (var usernameIdx in usernames) {
var uname = usernames[usernameIdx];
var user = await this.getUser(uname, userList[uname].url);
if (user === null) {
this.logger.verbose(`Could not retrieve user for UID '${uname}', skipping`);
continue;
}
// check group cache to ensure well formed.
let namespaceGroup = this.getCachedGroup(namespace);
if (namespaceGroup === null || namespaceGroup._self === null) {
this.logger.error(`Could not find group with namespace ${namespace}`);
continue;
}
// update the group to add the users for the current project
for (var usernameIdx in usernames) {
var uname = usernames[usernameIdx];
var user = await this.getUser(uname, userList[uname].url);
if (user === null) {
this.logger.verbose(`Could not retrieve user for UID '${uname}', skipping`);
continue;
}
await this.addUserToGroup(user, namespaceGroup._self!, userList[uname].accessLevel);
// if not tracked, track current project for group for post-sync cleanup
if (namespaceGroup.projectTargets.indexOf(project.short_project_id) === -1) {
namespaceGroup.projectTargets.push(project.short_project_id);
}
await this.addUserToGroup(user, namespaceGroup._self!, userList[uname].accessLevel);
// if not tracked, track current project for group for post-sync cleanup
if (namespaceGroup.projectTargets.indexOf(project.short_project_id) === -1) {
namespaceGroup.projectTargets.push(project.short_project_id);
}
}
}
......@@ -248,7 +254,7 @@ export class GitlabSyncRunner {
* Iterate through each group, checking self and ancestor project users and comparing against the current groups users to ensure that there are no
* additional users added with permissions.
*/
cleanupGroups(currentLevel: GroupCache = this.getRootGroup(), collectedProjects: string[] = []) {
async cleanupGroups(currentLevel: GroupCache = this.getRootGroup(), collectedProjects: string[] = []) {
this.logger.debug(`cleanupGroups(currentLevel = ${currentLevel._self?.full_path}, collectedProjects = ${collectedProjects})`);
let self = currentLevel._self;
if (self === null) {
......@@ -258,12 +264,18 @@ export class GitlabSyncRunner {
// collect and deduplicate project IDs
let projects = [...Array.from(new Set([...currentLevel.projectTargets, ...collectedProjects]))];
// build the user mapping to pass to cleanup
let projectUsers: Record<string, EclipseUser> = {};
let projectUsers: Record<string, EclipseUserAccess> = {};
for (let pidx in projects) {
projectUsers = Object.assign(projectUsers, this.getUserList(this.eclipseProjectCache[projects[pidx]]));
// check if any of the matched projects marks this as an external/skipped namespace
let project = this.eclipseProjectCache[projects[pidx]];
if (project.gitlab_excl_namespaces.some(v => v.localeCompare(self.full_path, undefined, { sensitivity: 'base' }) === 0)) {
this.logger.info(`Group '${self.full_path}' was marked as an external/protected namespace by project '${project.project_id}'`);
return;
}
projectUsers = Object.assign(projectUsers, this.getUserList(project));
}
// TODO we need to be able to pass multiple projects
this.removeAdditionalUsers(projectUsers, self, ...projects);
// clean up additional users
await this.removeAdditionalUsers(projectUsers, self, ...projects);
// for each of the children, pass the collected projects forward and process
for (let cidx in currentLevel.children) {
this.cleanupGroups(currentLevel.children[cidx], projects);
......@@ -279,7 +291,11 @@ export class GitlabSyncRunner {
* @param projectIDs list of project IDs that impact the given group
* @returns a promise that completes once all additional users are removed or the check finishes
*/
async removeAdditionalUsers(expectedUsers: Record<string, EclipseUser>, group: GroupSchema, ...projectIDs: string[]): Promise<void> {
async removeAdditionalUsers(
expectedUsers: Record<string, EclipseUserAccess>,
group: GroupSchema,
...projectIDs: string[]
): Promise<void> {
if (this.config.verbose) {
this.logger.debug(
`GitlabSync:removeAdditionalUsers(expectedUsers = ${JSON.stringify(expectedUsers)}, group = ${
......@@ -289,21 +305,16 @@ export class GitlabSyncRunner {
}
// get the current list of users for the group
var members = await this.getGroupMembers(group);
if (members === null) {
this.logger.warn(`Could not find any group members for ID ${group.id}'. Skipping user removal check`);
if (members === null || !(members instanceof Array)) {
this.logger.warn(`Could not find any group members for group '${group.full_path}'. Skipping user removal check`);
return;
}
// check that each of the users in the group match whats expected
var expectedUsernames = Object.keys(expectedUsers);
members!.forEach(async member => {
members?.forEach(async member => {
// check access and ensure user isn't an owner
this.logger.verbose(`Checking user '${member.username}' access to group '${group.name}'`);
if (
member.access_level !== ADMIN_PERMISSIONS_LEVEL &&
expectedUsernames.indexOf(member.username) === -1 &&
!this.isBot(member.username, projectIDs)
) {
if (this.shouldRemoveUser(member, expectedUsers, projectIDs, expectedUsernames)) {
if (this.config.dryRun) {
this.logger.info(`Dryrun flag active, would have removed user '${member.username}' from group '${group.name}'`);
return;
......@@ -321,17 +332,48 @@ export class GitlabSyncRunner {
});
}
/**
* Checks for the following states:
*
* - User is outside the allowlisted users
* - User is outside the expected user list
* - The user has the wrong permissions set and isn't a project lead
* - the user isn't a bot
*
* @param member the current group member being checked
* @param expectedUsers the user access mapping for the current group
* @param projectIDs projects associated with the current group
* @param expectedUsernames the usernames from the users mapping, passed to save processing time
* @returns true if all conditions in method description are met, otherwise false
*/
shouldRemoveUser(
member: MemberSchema,
expectedUsers: Record<string, EclipseUserAccess>,
projectIDs: string[],
expectedUsernames: string[]
): boolean {
if (member.access_level === ADMIN_PERMISSIONS_LEVEL) {
}
return (
ALLOWLISTED_USERS.indexOf(member.username) === -1 &&
(expectedUsernames.indexOf(member.username) === -1 ||
(member.access_level !== expectedUsers[member.username]!.accessLevel && expectedUsers[member.username]!.accessLevel !== 40)) &&
!this.isBot(member.username, projectIDs)
);
}
/**
* Iterates over the projects cache and cleans out the users and keeps bots for build operations. Skips over projects
* outside the scope of the designated root group to avoid over processing groups.
*/
async cleanupProjects() {
this.projectsCache.forEach(p => {
let group = this.getCachedGroup(p.namespace.full_path);
let t = this;
this.projectsCache.forEach(async function (p) {
let group = await t.getCachedGroup(p.namespace.full_path);
if (group !== null) {
this.cleanUpProjectUsers(p, ...group!.projectTargets);
t.cleanUpProjectUsers(p, ...group!.projectTargets);
} else {
this.logger.info(`Skipping processing of project '${p.name}'`);
t.logger.info(`Skipping processing of project '${p.name}'`);
}
});
}
......@@ -518,6 +560,34 @@ export class GitlabSyncRunner {
return u;
}
/**
* Used to create missing groups in the Gitlab instance. Does not insert into the nest cache as this should only be
* called from said cache. This method does not support creating root level groups.
*
* @param name the name of the group to create
* @param parent the group that this group belongs to
* @returns the new group schema once the call finishes
*/
async createMissingGroup(name: string, parent: GroupSchema): Promise<GroupSchema | null> {
if (this.config.verbose) {
this.logger.debug(`GitlabSync:createMissingGroup(name = ${name}, parent = ${parent.id})`);
}
// default options for creating new group
let opts = {
project_creation_level: 'maintainer',
visibility: 'public',
request_access_enabled: false,
parent_id: parent.id,
};
this.logger.info(`Creating missing group '${name}' in namespace '${parent.full_path} (${parent.id})'`);
try {
return await this.api.Groups.create(name, name, opts);
} catch (err) {
this.logger.error(`${err}`);
return null;
}
}
/**
* Retrieves the list of direct members for a given group, ignoring inherited users.
*
......@@ -589,7 +659,7 @@ export class GitlabSyncRunner {
* @param namespace the full path of the group namespace to retrieve.
* @returns the group cache for the group indicated by the namespace string, or null if there is no matching group.
*/
getCachedGroup(namespace: string): GroupCache | null {
async getCachedGroup(namespace: string): Promise<GroupCache | null> {
if (this.config.verbose) {
this.logger.debug(`GitlabSync:getCachedGroup(${namespace})`);
}
......@@ -659,13 +729,20 @@ export class GitlabSyncRunner {
* @param parent the parent to search through for the next part of the recursive call.
* @returns The group cache for the designated group, or null if it can't be found.
*/
tunnelAndRetrieve(namespaceParts: string[], parent: GroupCache): GroupCache | null {
async tunnelAndRetrieve(namespaceParts: string[], parent: GroupCache): Promise<GroupCache | null> {
if (this.config.verbose) {
this.logger.debug(`GitlabSync:tunnelAndRetrieve(namespaceParts = '${namespaceParts}')`);
}
let child = parent.children[namespaceParts[0]];
if (child === undefined) {
return null;
// attempt to create the new group
let newGroup = await this.createMissingGroup(namespaceParts[0], parent._self);
if (newGroup === null) {
this.logger.warn(`Could not create missing group with name '${namespaceParts[0]}' in group with path '${parent._self.full_path}'`);
return null;
}
// insert the new child group into the cache and continue
child = this.tunnelAndInsert(newGroup.full_path.split('/'), newGroup, this.getRootGroup());
}
// check if we should continue tunneling or insert and finish processing
if (namespaceParts.length > 1) {
......@@ -736,29 +813,6 @@ export class GitlabSyncRunner {
return '';
}
/**
* Uses TS URL type to ingest a namespace URL and return the host and namespace for use downstream
*
* @param rawUrl the raw namespace URL to check and split.
* @returns the host first, and the path of the url second which should be the namespace.
*/
splitNamespaceUrl(rawUrl: string): [string | null, string | null] {
try {
let url = new URL(rawUrl);
return [url.host, url.pathname.substring(1, url.pathname.length)];
} catch (e) {
// cast and message with error
let message = '';
if (typeof e === 'string') {
message = e.toUpperCase();
} else if (e instanceof Error) {
message = e.message;
}
this.logger.error(`Could not convert URL (${rawUrl}) to namespace: ${message}`);
}
return [null, null];
}
/**
* Checks whether a user is a bot for the given projects.
*
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment