Skip to content
Snippets Groups Projects
Commit 00ed4cb0 authored by Martin Lowe's avatar Martin Lowe :flag_ca:
Browse files

Add typescript implementation of GL sync with new group nesting

parent a6f33d72
No related branches found
No related tags found
1 merge request!210Rebuild GL sync in TS, add arb. group nesting
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 1833 additions and 1034 deletions
package-lock.json
+ 462
− 500
View file @ 00ed4cb0
This diff is collapsed.
src/EclipseAPI.js
+ 2
− 2
View file @ 00ed4cb0
......@@ -183,7 +183,7 @@ module.exports = class EclipseAPI {
},
})
.then(result => result.data)
.catch(err => this.#logger.error(err));
.catch(err => this.#logger.error(`${err}`));
}
async eclipseBots() {
......@@ -192,7 +192,7 @@ module.exports = class EclipseAPI {
}
var botsRaw = await axios.get('https://api.eclipse.org/bots')
.then(result => result.data)
.catch(err => this.#logger.error(err));
.catch(err => this.#logger.error(`${err}`));
if (botsRaw === undefined || botsRaw.length <= 0) {
this.#logger.error('Could not retrieve bots from API');
process.exit(EXIT_ERROR_STATE);
......
src/GitWrapper.js
+ 1
− 1
View file @ 00ed4cb0
......@@ -993,5 +993,5 @@ function logError(err, root) {
log.error(`${err.errors[i].message}`);
}
}
log.error(err);
log.error(`${err}`);
}
src/GitlabSync.js
+ 20
− 525
View file @ 00ed4cb0
......@@ -6,16 +6,19 @@ var argv = require('yargs')
alias: 'dryrun',
description: 'Runs script as dry run, not writing any changes to API',
boolean: true,
default: false,
})
.option('D', {
alias: 'devMode',
description: 'Runs script in dev mode, which returns API data that does not impact production organizations/teams.',
boolean: true,
default: false,
})
.option('V', {
alias: 'verbose',
description: 'Sets the script to run in verbose mode',
boolean: true,
default: false,
})
.option('H', {
alias: 'host',
......@@ -39,529 +42,21 @@ var argv = require('yargs')
.alias('h', 'help')
.version('0.1')
.alias('v', 'version')
.epilog('Copyright 2019 Eclipse Foundation inc.')
.argv;
const ADMIN_PERMISSIONS_LEVEL = 50;
const uuid = require('uuid');
const { SecretReader, getBaseConfig } = require('./SecretReader.js');
const { Gitlab } = require('@gitbeaker/node');
const EclipseAPI = require('./EclipseAPI.js');
const { getLogger } = require('./logger.js');
let logger = getLogger(argv.V ? 'debug' : 'info', 'main');
var api;
var eApi;
var bots;
var namedGroups = {};
var namedProjects = {};
var namedUsers = {};
var gMems = {};
_prepareSecret();
/**
* Retrieves secret API token from system, and then starts the script via _init
*
* @returns
*/
function _prepareSecret() {
// retrieve the secret API token
var accessToken, eclipseToken;
// retrieve the secret API file root if set
var settings = getBaseConfig();
if (argv.s !== undefined) {
settings.root = argv.s;
}
var reader = new SecretReader(settings);
var data = reader.readSecret('access-token');
if (data !== null) {
accessToken = data.trim();
// retrieve the Eclipse API token (needed for emails)
data = reader.readSecret('eclipse-oauth-config');
if (data !== null) {
eclipseToken = data.trim();
run(accessToken, eclipseToken);
} else {
logger.error('Could not find the Eclipse OAuth config, exiting');
}
} else {
logger.error('Could not find the GitLab access token, exiting');
}
}
async function run(secret, eclipseToken) {
api = new Gitlab({
host: argv.H,
token: secret,
});
eApi = new EclipseAPI(JSON.parse(eclipseToken));
eApi.testMode = argv.D;
// get raw project data and post process to add additional context
var data = await eApi.eclipseAPI();
data = eApi.postprocessEclipseData(data, 'gitlab_repos');
// get the bots for the projects
var rawBots = await eApi.eclipseBots();
bots = eApi.processBots(rawBots, 'gitlab.eclipse.org');
// get all current groups for the instance
var groups = await api.Groups.all();
var projects = await api.Projects.all();
var users = await api.Users.all();
// map the groups/projects/users to their name
for (var groupIdx in groups) {
namedGroups[sanitizeGroupName(groups[groupIdx].path)] = groups[groupIdx];
}
for (var projectIdx in projects) {
namedProjects[getCompositeProjectKey(projects[projectIdx].name, projects[projectIdx].namespace.id)] = projects[projectIdx];
}
for (var userIdx in users) {
namedUsers[users[userIdx].username] = users[userIdx];
}
// fetch org group from results, create if missing
logger.info('Starting sync');
var g = await getGroup('Eclipse', 'eclipse', undefined);
if (g === undefined) {
if (argv.d) {
logger.error('Unable to start sync of GitLab content. Base Eclipse group could not be found and dryrun is set');
} else {
logger.error('Unable to start sync of GitLab content. Base Eclipse group could not be created');
}
return;
}
for (projectIdx in data) {
var project = data[projectIdx];
if (argv.P !== undefined && project.short_project_id !== argv.P) {
logger.info(`Project target set ('${argv.P}'). Skipping non-matching project ID ${project.short_project_id}`);
continue;
}
logger.info(`Processing '${project.short_project_id}'`);
// fetch project group from results, create if missing
var projGroup = await getGroup(project.name, project.short_project_id, g);
if (projGroup === undefined) {
if (argv.d) {
logger.warn(`Unable to continue processing project with ID '${project.short_project_id}'.`
+ ' Group does not exist and dryrun has been set.');
} else {
logger.error(`Unable to continue processing project with ID '${project.short_project_id}'.`
+ ' Group does not exist and could not be created.');
}
continue;
}
// get the list of users to be added for current project
var userList = getUserList(project);
// for each user, get their gitlab user and add to the project group
var usernames = Object.keys(userList);
for (var usernameIdx in usernames) {
var uname = usernames[usernameIdx];
var user = await getUser(uname, userList[uname].url);
if (user === undefined) {
logger.verbose(`Could not retrieve user for UID '${uname}', skipping`);
continue;
}
await addUserToGroup(user, projGroup, userList[uname].access_level);
}
// remove users that don't match the expected users
await removeAdditionalUsers(userList, projGroup, project.short_project_id);
// for each of the repos in the Eclipse project, ensure there is a GL
// project
for (var repoIdx in project.gitlab_repos) {
var extRepo = project.gitlab_repos[repoIdx];
if (extRepo === undefined || extRepo.repo === undefined || extRepo.org === undefined) {
continue;
}
if (argv.V) {
logger.debug(`Processing repo '${extRepo.url}'`);
}
// retrieving current project
var p = await getProject(extRepo.repo, projGroup);
if (p !== undefined) {
await cleanUpProjectUsers(p, project.short_project_id);
}
}
}
}
async function removeAdditionalUsers(expectedUsers, group, projectID) {
if (argv.V) {
logger.debug(`GitlabSync:removeAdditionalUsers(expectedUsers = ${expectedUsers}, group = ${group}, projectID = ${projectID})`);
}
// get the current list of users for the group
var members = await getGroupMembers(group);
if (members === undefined) {
logger.warn(`Could not find any group members for ID ${group.id}'. Skipping user removal check`);
return;
}
// check that each of the users in the group match whats expected
var expectedUsernames = Object.keys(expectedUsers);
for (var memberIdx in members) {
var member = members[memberIdx];
// check access and ensure user isn't an owner
logger.verbose(`Checking user '${member.username}' access to group '${group.name}'`);
if (member.access_level !== ADMIN_PERMISSIONS_LEVEL && expectedUsernames.indexOf(member.username) === -1
&& !isBot(member.username, projectID)) {
if (argv.d) {
logger.info(`Dryrun flag active, would have removed user '${member.username}' from group '${group.name}'`);
continue;
}
logger.info(`Removing user '${member.username}' from group '${group.name}'`);
try {
await api.GroupMembers.remove(group.id, member.id);
} catch (err) {
if (argv.V) {
logger.error(err);
}
logger.warn(`Error while removing user '${member.username}' from group '${group.name}'`);
}
}
}
}
async function cleanUpProjectUsers(project, projectID) {
if (argv.V) {
logger.debug(`GitlabSync:cleanUpProjectUsers(project = ${project.id})`);
}
var projectMembers = await api.ProjectMembers.all(project.id, { includeInherited: false });
for (var idx in projectMembers) {
let member = projectMembers[idx];
// skip bot user or admin users
if (isBot(member.username, projectID) || member.access_level === ADMIN_PERMISSIONS_LEVEL) {
continue;
}
if (argv.d) {
logger.debug(`Dryrun flag active, would have removed user '${member.username}' from project '${project.name}'(${project.id})`);
continue;
}
logger.info(`Removing user '${member.username}' from project '${project.name}'(${project.id})`);
try {
await api.ProjectMembers.remove(project.id, member.id);
} catch (err) {
if (argv.V) {
logger.error(err);
}
logger.error(`Error while removing user '${member.username}' from project '${project.name}'(${project.id})`);
}
}
}
function isBot(uname, projectID) {
var botList = bots[projectID];
// check if the current user is in the current key-values list
return botList !== undefined && botList.indexOf(uname) !== -1;
}
/** API FUNCTIONS */
async function addUserToGroup(user, group, perms) {
if (argv.V) {
logger.debug(`GitlabSync:addUserToGroup(user = ${user}, group = ${group}, perms = ${perms})`);
}
// get the members for the current group
var members = await getGroupMembers(group);
if (members === undefined) {
logger.warn(`Could not find any references to group with ID ${group.id}`);
return;
}
// check if user is already present
for (var memberIdx in members) {
if (members[memberIdx].username === user.username) {
logger.verbose(`User '${user.username}' is already a member of ${group.name}`);
if (members[memberIdx].access_level !== perms) {
// skip if dryrun
if (argv.d) {
logger.info(`Dryrun flag active, would have updated user '${members[memberIdx].username}' in group '${group.name}'`);
return;
}
// modify user, catching errors
logger.info(`Fixing permission level for user '${user.username}' in group '${group.name}'`);
try {
var updatedMember = await api.GroupMembers.edit(group.id, user.id, perms);
// update inner array
members[memberIdx] = updatedMember;
gMems[group.id] = members;
} catch (err) {
if (argv.V) {
logger.error(err);
}
logger.warn(`Error while fixing permission level for user '${user.username}' in group '${group.name}'`);
return;
}
}
// return a copy of the updated user
return JSON.parse(JSON.stringify(members[memberIdx]));
}
}
// check if dry run before updating
if (argv.d) {
logger.info(`Dryrun flag active, would have added user '${user.username}' to group '${group.name}' with access level '${perms}'`);
return;
}
logger.info(`Adding '${user.username}' to '${group.name}' group`);
try {
// add member to group, track, and return a copy
var newMember = await api.GroupMembers.add(group.id, user.id, perms);
members.push(newMember);
gMems[group.id] = members;
// return a copy
return JSON.parse(JSON.stringify(newMember));
} catch (err) {
if (argv.V) {
logger.error(err);
}
logger.warn(`Error while adding '${user.username}' to '${group.name}' group`);
}
}
async function getProject(name, parent) {
if (argv.V) {
logger.debug(`GitlabSync:getProject(name = ${name}, parent = ${parent})`);
}
if (name.trim() === '.github') {
logger.warn("Skipping project with name '.github'. No current equivalent to default repository in GitLab.");
return;
}
var p = namedProjects[getCompositeProjectKey(name, parent.id)];
if (p === undefined) {
logger.verbose(`Creating new project with name '${name}'`);
// create the request options for the new user
var opts = {
path: name,
visibility: 'public',
};
if (parent !== undefined) {
opts.namespace_id = parent.id;
}
// check if dry run before creating new project
if (argv.d) {
logger.info(`Dryrun flag active, would have created new project '${name}' with options ${JSON.stringify(opts)}`);
return;
}
// create the new project, and track it
if (argv.V) {
logger.debug(`Creating project with options: ${JSON.stringify(opts)}`);
}
try {
p = await api.Projects.create(opts);
} catch (err) {
if (argv.V) {
logger.error(err);
}
}
if (p === null || p instanceof Array) {
logger.warn(`Error while creating project '${name}'`);
return undefined;
}
if (argv.V) {
logger.debug(`Created project: ${JSON.stringify(p)}`);
}
// set it back
namedProjects[getCompositeProjectKey(name, parent.id)] = p;
}
return p;
}
async function getGroup(name, path, parent, visibility = 'public') {
if (argv.V) {
logger.debug(`GitlabSync:getGroup(name = ${name}, path = ${path}, parent = ${parent}, visibility = ${visibility})`);
}
var g = namedGroups[sanitizeGroupName(path)];
if (g === undefined) {
logger.verbose(`Creating new group with name '${name}'`);
var opts = {
project_creation_level: 'maintainer',
visibility: visibility,
request_access_enabled: false,
};
if (parent !== undefined && parent.id !== undefined) {
opts.parent_id = parent.id;
}
// check if dry run before creating group
if (argv.d) {
logger.info(`Dryrun flag active, would have created new group '${name}' with options ${JSON.stringify(opts)}`);
return;
}
// if verbose is set display user opts
if (argv.V) {
logger.debug(`Creating group with options: ${JSON.stringify(opts)}`);
}
try {
g = await api.Groups.create(name, sanitizeGroupName(path), opts);
} catch (err) {
if (argv.V) {
logger.error(err);
}
}
if (g === null || g instanceof Array) {
logger.warn(`Error while creating group '${name}'`);
return undefined;
}
if (argv.V) {
logger.debug(`Created group: ${JSON.stringify(g)}`);
}
// set it back
namedGroups[sanitizeGroupName(path)] = g;
}
return g;
}
async function getUser(uname, url) {
if (argv.V) {
logger.debug(`GitlabSync:getUser(uname = ${uname}, url = ${url})`);
}
if (url === undefined || url === '') {
logger.error(`Cannot fetch user information for user '${uname}' with no set URL`);
return;
}
var u = namedUsers[uname];
if (u === undefined) {
if (argv.d) {
logger.info(`Dryrun is enabled. Would have created user ${uname} but was skipped`);
return;
}
// retrieve user data
var data = await eApi.eclipseUser(uname);
if (data === undefined) {
logger.error(`Cannot create linked user account for '${uname}', no external data found`);
return;
}
logger.verbose(`Creating new user with name '${uname}'`);
var opts = {
username: uname,
password: uuid.v4(),
force_random_password: true,
name: `${data.first_name} ${data.last_name}`,
email: data.mail,
extern_uid: data.uid,
provider: argv.p,
skip_confirmation: true,
};
// check if dry run before creating new user
if (argv.d) {
logger.info(`Dryrun flag active, would have created new user '${uname}' with options ${JSON.stringify(opts)}`);
return;
}
// if verbose, display information being used to generate user
if (argv.V) {
// copy the object and redact the password for security
var optLog = JSON.parse(JSON.stringify(opts));
optLog.password = 'redacted';
logger.debug(`Creating user with options: ${JSON.stringify(optLog)}`);
}
try {
u = await api.Users.create(opts);
} catch (err) {
if (argv.V) {
logger.error(err);
}
}
if (u === null) {
logger.warn(`Error while creating user '${uname}'`);
return undefined;
}
// set it back
namedUsers[uname] = u;
}
return u;
}
async function getGroupMembers(group) {
if (argv.V) {
logger.debug(`GitlabSync:getGroupMembers(group = ${group})`);
}
var members = gMems[group.id];
if (members === undefined) {
try {
members = await api.GroupMembers.all(group.id);
} catch (err) {
if (argv.V) {
logger.error(err);
}
}
if (members === null) {
logger.warn(`Unable to find group members for group with ID '${group.id}'`);
return;
}
gMems[group.id] = members;
}
return members;
}
/** HELPERS */
function getUserList(project) {
if (argv.V) {
logger.debug(`GitlabSync:getUserList(project = ${JSON.stringify(project)})`);
}
var l = {};
// add the contributors with reporter access
for (var contributorIdx in project.contributors) {
l[project.contributors[contributorIdx].username] = {
url: project.contributors[contributorIdx].url,
access_level: 20,
};
}
// add the committers with developer access
for (var committerIdx in project.committers) {
l[project.committers[committerIdx].username] = {
url: project.committers[committerIdx].url,
access_level: 30,
};
}
// add the project leads not yet tracked with reporter access
for (var plIdx in project.project_leads) {
l[project.project_leads[plIdx].username] = {
url: project.project_leads[plIdx].url,
access_level: 40,
};
}
// add the bots with developer access
var botList = bots[project.project_id];
for (var botIdx in botList) {
l[botList[botIdx]] = {
access_level: 30,
};
}
return l;
}
function sanitizeGroupName(pid) {
if (argv.V) {
logger.debug(`GitlabSync:sanitizeGroupName(pid = ${pid})`);
}
if (pid !== undefined) {
return pid.toLowerCase().replace(/[^\s\da-zA-Z-.]/g, '-');
}
return '';
}
function getCompositeProjectKey(projectName, parentId) {
return projectName + ':' + parentId;
.epilog('Copyright 2019 Eclipse Foundation inc.').argv;
import { GitlabSyncRunner } from './gl/GitlabSyncRunner';
const runner = new GitlabSyncRunner({
devMode: argv.D,
host: argv.host,
dryRun: argv.dryRun,
provider: argv.provider,
verbose: argv.verbose,
project: argv.project,
secretLocation: argv.secretLocation,
});
run();
async function run() {
await runner.run();
}
src/SecretReader.js
+ 1
− 1
View file @ 00ed4cb0
......@@ -66,7 +66,7 @@ class SecretReader {
try {
var data = fs.readFileSync(filepath, { encoding: encoding });
if (data !== undefined && (data = data.trim()) !== '') {
return data;
return data.toString();
}
} catch (err) {
if (err.code === 'ENOENT') {
......
src/eclipse/EclipseAPI.ts 0 → 100644
+ 288
− 0
View file @ 00ed4cb0
import axios, { AxiosRequestConfig } from 'axios';
import parse from 'parse-link-header';
import { AccessToken, ClientCredentials } from 'simple-oauth2';
import { Logger } from 'winston';
import { BotDefinition, EclipseProject, Repo, EclipseUser } from '../interfaces/EclipseApi';
import { getLogger } from '../helpers/logger';
const HOUR_IN_SECONDS = 3600;
const EXIT_ERROR_STATE = 1;
/**
* Root config used for interacting with the Eclipse API. OAuth is
*/
export interface EclipseApiConfig {
oauth: EclipseApiOAuthConfig;
verbose?: boolean;
testMode?: boolean;
}
/**
* Configuration interface for oauth/security binding.
*/
export interface EclipseApiOAuthConfig {
client_secret: string;
client_id: string;
endpoint: string;
redirect: string;
scope: string;
timeout?: number;
}
export class EclipseAPI {
config: EclipseApiConfig;
client?: ClientCredentials;
accessToken: AccessToken | null = null;
logger: Logger;
constructor(config: EclipseApiConfig) {
this.config = config;
// generate creds if auth is set
if (this.config.oauth !== null) {
this.client = new ClientCredentials({
client: {
id: this.config.oauth.client_id,
secret: this.config.oauth.client_secret,
},
auth: {
tokenHost: this.config.oauth.endpoint,
tokenPath: '/oauth2/token',
authorizePath: '/oauth2/authorize',
},
});
}
this.logger = getLogger('info', 'EclipseAPI');
}
/**
* Retrieves Eclipse projects with the given query string parameters, with an option to paginate and return all results.
*
* @param queryStringParams optional query string to use when querying projects
* @param paginate Optional, false if only 1 page should be queried, true otherwise.
* @returns promise to return either a page or all pages of eclipse projects given a query string.
*/
async eclipseAPI(queryStringParams = '', paginate = true): Promise<EclipseProject[]> {
if (this.config.verbose) {
this.logger.debug(`EclipseAPI:eclipseAPI(queryStringParams = ${queryStringParams}, paginate = ${paginate})`);
}
// if test mode is enabled, return data that doesn't impact production
if (this.config.testMode) {
return testProjects;
}
let hasMore = true;
let result = [];
let data = [];
// add timestamp to url to avoid browser caching
var url = 'https://projects.eclipse.org/api/projects' + queryStringParams;
// loop through all available users, and add them to a list to be returned
do {
this.logger.silly('Loading next page...');
// get the current page of results, incrementing page count after call
result = await axios
.get(url)
.then(r => {
// return the data to the user
let links = parse(r.headers.link);
// check if we should continue processing
if (links === null || links!.self.url === links!.last.url) {
hasMore = false;
} else {
url = links!.next.url;
}
return r.data;
})
.catch(err => {
this.logger.error(`Error while retrieving results from Eclipse Projects API (${url}): ${err}`);
});
// collect the results
if (result != null && result.length > 0) {
for (var i = 0; i < result.length; i++) {
data.push(result[i]);
}
}
} while (hasMore && paginate);
return data;
}
/**
* Retrieves an eclipse foundation user using the given username, returning null if the user cannot be found.
*
* @param username the username to retrieve data for
* @returns the Eclipse Foundation user account data, enhanced with sensitive information if oauth is configured. Returns null if user cannot be found.
*/
async eclipseUser(username: string): Promise<EclipseUser | null> {
if (this.config.verbose) {
this.logger.debug(`EclipseAPI:eclipseUser(username = ${username})`);
}
return await axios
.get('https://api.eclipse.org/account/profile/' + username, await this.getAuthenticationHeaders())
.then(result => result.data)
.catch(err => {
this.logger.error(`${err}`);
return null;
});
}
async eclipseBots(): Promise<BotDefinition[]> {
if (this.config.verbose) {
this.logger.debug('EclipseAPI:eclipseBots()');
}
var botsRaw = await axios
.get('https://api.eclipse.org/bots')
.then(result => result.data)
.catch(err => this.logger.error(`${err}`));
if (botsRaw === undefined || botsRaw.length <= 0) {
this.logger.error('Could not retrieve bots from API');
process.exit(EXIT_ERROR_STATE);
}
return botsRaw;
}
/**
* Maps bot users by listing bots per project.
*
* @param botsRaw the raw bot definition list to convert to a project bot mapping.
* @param site the site targeted for bots to limit results
* @returns a mapping of project to configured bot usernames.
*/
processBots(botsRaw: BotDefinition[], site: string = 'github.com'): Record<string, string[]> {
if (this.config.verbose) {
this.logger.debug(`EclipseAPI:processBots(botsRaw = ${JSON.stringify(botsRaw)}, site = ${site})`);
}
let rgx = new RegExp(`^${site}.*`);
var botMap: Record<string, string[]> = {};
for (var botIdx in botsRaw) {
let bot = botsRaw[botIdx];
// get the list of bots for project if already created
var projBots = botMap[bot.projectId];
if (projBots === undefined) {
projBots = [];
}
// get usernames for site + sub resource bots
let botKeys = Object.keys(bot);
for (let idx in botKeys) {
let key = botKeys[idx];
// if there is a match (either direct or sub resource match) push the bot name to the list
let match = key.match(rgx);
if (match) {
projBots.push(bot[key].username);
}
}
// dont add empty arrays to output
if (projBots.length === 0) {
continue;
}
botMap[bot.projectId] = projBots;
}
return botMap;
}
/**
* If OAuth has been configured, then retrieves access token and sets into request config to use for calls to Eclipse API endpoints.
* This call is needed to be able to retrieve sensitive user profile information.
*
* @returns request configs including authentication headers if auth is configured, or empty config otherwise.
*/
async getAuthenticationHeaders(): Promise<AxiosRequestConfig> {
let token = await this._getAccessToken();
if (token === null) {
this.logger.info('Authentication token cannot be retrieved, information returned mey be limited');
return {};
} else {
return {
headers: {
Authorization: `Bearer ${token}`,
},
};
}
}
/**
* Retrieves an oauth token and caches it internally to authenticate requests to the Eclipse API. This by default caches
* for an hour to reduce turn around on the authentication API.
*
* @returns the access token if found, otherwise null.
*/
async _getAccessToken(): Promise<string | null> {
// check that we have auth configs and that current token is expired before attempting retrieval
if (
this.config.oauth !== null &&
this.client !== null &&
(this.accessToken === null || this.accessToken!.expired(this.config.oauth.timeout ?? HOUR_IN_SECONDS))
) {
// wrap retrieval in try-catch to give more information to the logs on error.
try {
this.accessToken = await this.client!.getToken({
scope: this.config.oauth.scope,
});
} catch (error) {
this.logger.error(`${error}`);
process.exit(EXIT_ERROR_STATE);
}
return this.accessToken.token.access_token;
}
return null;
}
}
const testProjects: EclipseProject[] = [
{
project_id: 'spider.pig',
url: '',
website_repo: [],
website_url: '',
short_project_id: 'spider.pig',
name: 'Spider pig does what a spider pig does',
summary: 'Can he fly? No, hes a pig. Look out, here comes the spider pig',
logo: '',
tags: ['simpsons', 'doh', 'spider pig'],
gerrit_repos: [
{
url: 'https://github.com/eclipsefdn-webdev/spider-pig',
},
],
github_repos: [
{
url: 'https://github.com/eclipsefdn-webdev/spider-pig',
},
],
gitlab_repos: [
{
url: 'https://gitlab.eclipse.org/eclipsefdn/webdev/gitlab-testing',
},
],
contributors: [],
committers: [
{
username: 'malowe',
url: 'https://api.eclipse.org/account/profile/malowe',
},
{
username: 'epoirier',
url: 'https://api.eclipse.org/account/profile/epoirier',
},
],
project_leads: [
{
username: 'malowe',
url: 'https://api.eclipse.org/account/profile/malowe',
},
{
username: 'cguindon',
url: 'https://api.eclipse.org/account/profile/cguindon',
},
],
working_groups: [
{
name: 'Cloud Development Tools',
id: 'cloud-development-tools',
},
],
spec_project_working_group: [],
state: 'Regular',
releases: [],
},
];
src/gl/AxiosRequester.ts 0 → 100644
+ 21
− 0
View file @ 00ed4cb0
import {RequesterType} from '@gitbeaker/requester-utils';
import axios from 'axios';
export class AxiosRequester implements RequesterType {
get(endpoint: string, options?: Record<string, unknown>): Promise<any> {
throw new Error('Method not implemented.');
}
post(endpoint: string, options?: Record<string, unknown>): Promise<any> {
throw new Error('Method not implemented.');
}
put(endpoint: string, options?: Record<string, unknown>): Promise<any> {
throw new Error('Method not implemented.');
}
delete(endpoint: string, options?: Record<string, unknown>): Promise<any> {
throw new Error('Method not implemented.');
}
stream?(endpoint: string, options?: Record<string, unknown>): NodeJS.ReadableStream {
throw new Error('Method not implemented.');
}
}
\ No newline at end of file
src/gl/GitlabSync.ts 0 → 100644
+ 68
− 0
View file @ 00ed4cb0
// set up yargs command line parsing
import { GitlabSyncRunner } from './GitlabSyncRunner';
import yargs from 'yargs';
let args = yargs(process.argv)
.usage('Usage: $0 [options]')
.example('$0', '')
.option('d', {
alias: 'dryrun',
description: 'Runs script as dry run, not writing any changes to API',
boolean: true,
default: false,
})
.option('D', {
alias: 'devMode',
description: 'Runs script in dev mode, which returns API data that does not impact production organizations/teams.',
boolean: true,
default: false,
})
.option('V', {
alias: 'verbose',
description: 'Sets the script to run in verbose mode',
boolean: true,
default: false,
})
.option('H', {
alias: 'host',
description: 'GitLab host base URL',
default: 'http://gitlab.eclipse.org/',
string: true,
})
.option('p', {
alias: 'provider',
description: 'The OAuth provider name set in GitLab',
default: 'oauth2_generic',
string: true,
})
.option('P', {
alias: 'project',
description: 'The short project ID of the target for the current sync run',
string: true,
})
.option('s', {
alias: 'secretLocation',
description: 'The location of the access-token file containing an API access token',
string: true,
})
.help('h')
.alias('h', 'help')
.version('0.1')
.alias('v', 'version')
.epilog('Copyright 2019 Eclipse Foundation inc.').argv;
const runner =
run();
async function run() {
const argv = await args;
await new GitlabSyncRunner({
devMode: argv.D,
host: argv.H,
dryRun: argv.d,
provider: argv.p,
verbose: argv.V,
project: argv.P,
secretLocation: argv.s,
}).run();
}
src/gl/GitlabSyncRunner.ts 0 → 100644
+ 714
− 0
View file @ 00ed4cb0
This diff is collapsed.
src/helpers/SecretReader.ts 0 → 100644
+ 88
− 0
View file @ 00ed4cb0
/** ***************************************************************
Copyright (C) 2022 Eclipse Foundation, Inc.
This program and the accompanying materials are made
available under the terms of the Eclipse Public License 2.0
which is available at https://www.eclipse.org/legal/epl-2.0/
Contributors:
Martin Lowe <martin.lowe@eclipse-foundation.org>
SPDX-License-Identifier: EPL-2.0
******************************************************************/
import { getLogger, isNodeErr } from './logger';
import fs from 'fs';
import { Logger } from 'winston';
const DEFAULT_FILE_ENCODING: string = 'utf-8';
const DEFAULT_SECRET_LOCATION: string = '/run/secrets/';
export interface SecretReaderConfig {
root?: string;
encoding?: string;
}
/**
* Contains functionality for reading secret files in and returning them
* to the user. This defaults to the location used in Kubernetes containers
* for secrets. This can be configured by passing an object with updated values.
*
* Multiple secrets can be read using the same reader assuming that they are
* in the same directory. Additional secrets would need to be read in using a
* new reader in such a case.
*/
export class SecretReader {
verbose: boolean = false;
logger: Logger;
config: SecretReaderConfig;
constructor(config: SecretReaderConfig) {
// set internally and modify for defaults
this.config = config;
// set defaults if value is missing
this.config.root = this.config.root ?? DEFAULT_SECRET_LOCATION;
this.config.encoding = this.config.encoding ?? DEFAULT_FILE_ENCODING;
// throws if there is no access
fs.accessSync(this.config.root, fs.constants.R_OK);
this.logger = getLogger('info', 'SecretReader');
}
readSecret(name: string, encoding: string = this.config.encoding ?? DEFAULT_FILE_ENCODING): string | null {
if (this.verbose === true) {
this.logger.debug(`SecretReader:readSecret(name = ${name}, encoding = ${encoding})`);
}
var filepath = `${this.config.root}/${name}`;
try {
var data = fs.readFileSync(filepath, { encoding: encoding as BufferEncoding });
let out: string;
if (data !== undefined && (out = data.trim()) !== '') {
return out;
}
} catch (e) {
// cast and message with error
if (isNodeErr(e)) {
if (e.code === 'ENOENT') {
this.logger.error(`File at path ${filepath} does not exist`);
} else if (e.code === 'EACCES') {
this.logger.error(`File at path ${filepath} cannot be read`);
} else {
this.logger.error('An unknown error occurred while reading the secret');
}
} else if (typeof e === 'string') {
this.logger.error('An unknown error occurred while reading the secret');
}
}
return null;
}
}
/**
* Get modifiable deep copy of the base configuration for this class.
*/
export function getBaseConfig(): SecretReaderConfig {
return {
root: DEFAULT_SECRET_LOCATION,
encoding: DEFAULT_FILE_ENCODING,
};
}
src/helpers/logger.ts 0 → 100644
+ 42
− 0
View file @ 00ed4cb0
/** **************************************************************
Copyright (C) 2021 Eclipse Foundation, Inc.
This program and the accompanying materials are made
available under the terms of the Eclipse Public License 2.0
which is available at https://www.eclipse.org/legal/epl-2.0/
Contributors:
Martin Lowe <martin.lowe@eclipse-foundation.org>
SPDX-License-Identifier: EPL-2.0
******************************************************************/
// import winston for logging implementation
import { createLogger, format, Logger, transports } from 'winston';
/**
Exports central implementation of logging to be used across JS. This way logging can be consistent across the logs easily w/o repitition.
Example of this format:
2021-01-25T15:55:29 [main] INFO Generating teams for eclipsefdn-webdev/spider-pig
2021-01-25T15:55:30 [SecretReader] ERROR An unknown error occurred while reading the secret
*/
export function getLogger(level: string, name = 'main') {
let logger = createLogger({
level: level,
format: format.combine(
format.timestamp({
format: 'YYYY-MM-DDTHH:mm:ss',
}),
format.printf(info => {
return `${info.timestamp} [${name}] ${info.level.toUpperCase()} ${info.message}`;
})),
transports: [
new transports.Console({ level: level }),
],
});
return logger;
}
export function isNodeErr(error: unknown): error is NodeJS.ErrnoException {
return error instanceof Error;
}
src/interfaces/EclipseApi.ts 0 → 100644
+ 100
− 0
View file @ 00ed4cb0
//
// PROJECTS API
//
export interface Repo {
url: string;
// post-processing fields
repo?: string;
org?: string;
}
export interface UserRef {
username: string;
full_name?: string;
url: string;
}
export interface Release {
name: string;
url: string;
}
export interface WorkingGroup {
name: string;
id: string;
}
export interface EclipseProject {
project_id: string;
short_project_id: string;
name: string;
summary: string;
url: string;
website_url: string;
website_repo: string[];
logo: string;
tags: string[];
github_repos: Repo[];
gitlab_repos: Repo[];
gerrit_repos: Repo[];
contributors: UserRef[];
committers: UserRef[];
project_leads: UserRef[];
working_groups: WorkingGroup[];
spec_project_working_group: WorkingGroup | Array<any>;
state: string;
releases: Release[];
// post processing storage fields
pp_repos?: string[];
pp_orgs?: string[];
}
//
// ECLIPSE ACCOUNT
//
export interface EclipseUser {
uid: string;
name: string;
mail: string;
picture: string;
eca: {
signed: boolean;
can_contribute_spec_project: boolean;
};
is_committer: boolean;
friends: {
friend_id: string;
};
first_name: string;
last_name: string;
full_name: string;
github_handle: string;
twitter_handle: string;
org: string;
org_id: number;
job_title: string;
website: string;
country: {
code: string;
name: string;
};
bio: string;
}
//
// BOTS API
//
export interface BotInstance {
username: string;
email: string;
}
export interface BotProperties {
id: number;
projectId: string;
username: string;
email: string;
}
export type BotDefinition = BotProperties & Record<string, BotInstance>;
src/logger.js
+ 3
− 3
View file @ 00ed4cb0
......@@ -11,7 +11,7 @@
SPDX-License-Identifier: EPL-2.0
******************************************************************/
// import winston for logging implementation
const { createLogger, format, transports } = require('winston');
import { createLogger, format, transports } from 'winston';
/**
Exports central implementation of logging to be used across JS. This way logging can be consistent across the logs easily w/o repitition.
......@@ -21,7 +21,7 @@ Example of this format:
2021-01-25T15:55:29 [main] INFO Generating teams for eclipsefdn-webdev/spider-pig
2021-01-25T15:55:30 [SecretReader] ERROR An unknown error occurred while reading the secret
*/
module.exports.getLogger = function(level, name = 'main') {
export function getLogger(level, name = 'main') {
let logger = createLogger({
level: level,
format: format.combine(
......@@ -36,4 +36,4 @@ module.exports.getLogger = function(level, name = 'main') {
],
});
return logger;
};
}
tsconfig.json 0 → 100644
+ 14
− 0
View file @ 00ed4cb0
{
"compilerOptions": {
"module": "CommonJS",
"esModuleInterop": true,
"sourceMap": true
},
"include": [
"src/**/*"
],
"exclude": [
"node_modules",
"**/*.spec.ts"
]
}
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Copyright © Eclipse Foundation, Inc. All Rights Reserved.     Privacy Policy | Terms of Use | Copyright Agent