Commit 0d0c5d74 authored by Martin Lowe's avatar Martin Lowe 🇨🇦
Browse files

Update run cmd, restruc TS files, add tests

parent d9f45491
This diff is collapsed.
......@@ -11,7 +11,7 @@
"start": "node src/Sync.js -c",
"pretest": "eslint --ignore-path .gitignore .",
"test": "mocha --timeout 60000",
"lab-sync": "node src/GitlabSync.js -c",
"lab-sync": "ts-node src/scripts/gl/GitlabSync.ts -c",
"import-backup": "node src/auto_backup/Import.js"
},
"license": "EPL-2.0",
......@@ -21,29 +21,30 @@
"homepage": "https://github.com/eclipsefdn/eclipsefdn-github-sync#readme",
"dependencies": {
"@gitbeaker/core": "^35.6.0",
"@gitbeaker/node": "github:autumnfound/gitbeaker#malowe/master/2483",
"@gitbeaker/node": "^35.6.0",
"@octokit/plugin-retry": "^3.0.3",
"@octokit/plugin-throttling": "^3.3.0",
"@octokit/rest": "^18.0.3",
"@types/got": "^9.6.12",
"@types/node": "^17.0.32",
"@types/yargs": "^17.0.10",
"axios": "^0.21.4",
"flat-cache": "^2.0.1",
"got": "^12.0.4",
"nodemailer": "^6.5.0",
"openid-client": "^3.15.6",
"parse-link-header": "^2.0.0",
"simple-oauth2": "^4.1.0",
"ts-node": "^10.7.0",
"uuid": "^8.3.2",
"winston": "3.2.1",
"yargs": "^13.3.0"
},
"devDependencies": {
"@babel/eslint-parser": "^7.17.0",
"@types/mocha": "^9.1.1",
"@types/parse-link-header": "^2.0.0",
"@types/simple-oauth2": "^4.1.1",
"@types/uuid": "^8.3.4",
"@typescript-eslint/parser": "^5.23.0",
"chai": "^4.2.0",
"eslint": "^7.5.0",
"eslint-config-strongloop": "^2.1.0",
......
// set up yargs command line parsing
var argv = require('yargs')
.usage('Usage: $0 [options]')
.example('$0', '')
.option('d', {
alias: 'dryrun',
description: 'Runs script as dry run, not writing any changes to API',
boolean: true,
default: false,
})
.option('D', {
alias: 'devMode',
description: 'Runs script in dev mode, which returns API data that does not impact production organizations/teams.',
boolean: true,
default: false,
})
.option('V', {
alias: 'verbose',
description: 'Sets the script to run in verbose mode',
boolean: true,
default: false,
})
.option('H', {
alias: 'host',
description: 'GitLab host base URL',
default: 'http://gitlab.eclipse.org/',
})
.option('p', {
alias: 'provider',
description: 'The OAuth provider name set in GitLab',
default: 'oauth2_generic',
})
.option('P', {
alias: 'project',
description: 'The short project ID of the target for the current sync run',
})
.option('s', {
alias: 'secretLocation',
description: 'The location of the access-token file containing an API access token',
})
.help('h')
.alias('h', 'help')
.version('0.1')
.alias('v', 'version')
.epilog('Copyright 2019 Eclipse Foundation inc.').argv;
import { GitlabSyncRunner } from './gl/GitlabSyncRunner';
const runner = new GitlabSyncRunner({
devMode: argv.D,
host: argv.host,
dryRun: argv.dryRun,
provider: argv.provider,
verbose: argv.verbose,
project: argv.project,
secretLocation: argv.secretLocation,
});
run();
async function run() {
await runner.run();
}
......@@ -2,7 +2,7 @@ import axios, { AxiosRequestConfig } from 'axios';
import parse from 'parse-link-header';
import { AccessToken, ClientCredentials } from 'simple-oauth2';
import { Logger } from 'winston';
import { BotDefinition, EclipseProject, Repo, EclipseUser } from '../interfaces/EclipseApi';
import { BotDefinition, EclipseProject, EclipseUser } from '../interfaces/EclipseApi';
import { getLogger } from '../helpers/logger';
const HOUR_IN_SECONDS = 3600;
......@@ -12,7 +12,7 @@ const EXIT_ERROR_STATE = 1;
* Root config used for interacting with the Eclipse API. OAuth is
*/
export interface EclipseApiConfig {
oauth: EclipseApiOAuthConfig;
oauth?: EclipseApiOAuthConfig;
verbose?: boolean;
testMode?: boolean;
}
......@@ -38,7 +38,7 @@ export class EclipseAPI {
constructor(config: EclipseApiConfig) {
this.config = config;
// generate creds if auth is set
if (this.config.oauth !== null) {
if (this.config.oauth !== null && this.config.oauth !== undefined) {
this.client = new ClientCredentials({
client: {
id: this.config.oauth.client_id,
......
/** **************************************************************
Copyright (C) 2021 Eclipse Foundation, Inc.
Copyright (C) 2021, 2022 Eclipse Foundation, Inc.
This program and the accompanying materials are made
available under the terms of the Eclipse Public License 2.0
......
//
// PROJECTS API
//
/** ***************************************************************
Copyright (C) 2022 Eclipse Foundation, Inc.
This program and the accompanying materials are made
available under the terms of the Eclipse Public License 2.0
which is available at https://www.eclipse.org/legal/epl-2.0/
Contributors:
Martin Lowe <martin.lowe@eclipse-foundation.org>
SPDX-License-Identifier: EPL-2.0
******************************************************************/
export interface Repo {
url: string;
// post-processing fields
repo?: string;
org?: string;
}
......@@ -44,9 +51,6 @@ export interface EclipseProject {
spec_project_working_group: WorkingGroup | Array<any>;
state: string;
releases: Release[];
// post processing storage fields
pp_repos?: string[];
pp_orgs?: string[];
}
//
......
......@@ -11,7 +11,7 @@
SPDX-License-Identifier: EPL-2.0
******************************************************************/
// import winston for logging implementation
import { createLogger, format, transports } from 'winston';
const { createLogger, format, transports } = require('winston');
/**
Exports central implementation of logging to be used across JS. This way logging can be consistent across the logs easily w/o repitition.
......@@ -21,7 +21,7 @@ Example of this format:
2021-01-25T15:55:29 [main] INFO Generating teams for eclipsefdn-webdev/spider-pig
2021-01-25T15:55:30 [SecretReader] ERROR An unknown error occurred while reading the secret
*/
export function getLogger(level, name = 'main') {
module.exports.getLogger = function(level, name = 'main') {
let logger = createLogger({
level: level,
format: format.combine(
......@@ -36,4 +36,4 @@ export function getLogger(level, name = 'main') {
],
});
return logger;
}
};
......@@ -34,9 +34,9 @@ var argv = require('yargs')
.epilog('Copyright 2019 Eclipse Foundation inc.')
.argv;
const Wrapper = require('../GitWrapper.js');
const EclipseAPI = require('../EclipseAPI.js');
const { SecretReader, getBaseConfig } = require('../SecretReader.js');
const Wrapper = require('../../GitWrapper.js');
const EclipseAPI = require('../../EclipseAPI.js');
const { SecretReader, getBaseConfig } = require('../../SecretReader.js');
var wrap, eclipseApi;
var inviteCtr = 0;
......
// set up yargs command line parsing
/** ***************************************************************
Copyright (C) 2022 Eclipse Foundation, Inc.
This program and the accompanying materials are made
available under the terms of the Eclipse Public License 2.0
which is available at https://www.eclipse.org/legal/epl-2.0/
Contributors:
Martin Lowe <martin.lowe@eclipse-foundation.org>
SPDX-License-Identifier: EPL-2.0
******************************************************************/
import { GitlabSyncRunner } from './GitlabSyncRunner';
import yargs from 'yargs';
......
/** ***************************************************************
Copyright (C) 2022 Eclipse Foundation, Inc.
This program and the accompanying materials are made
available under the terms of the Eclipse Public License 2.0
which is available at https://www.eclipse.org/legal/epl-2.0/
Contributors:
Martin Lowe <martin.lowe@eclipse-foundation.org>
SPDX-License-Identifier: EPL-2.0
******************************************************************/
import { Logger } from 'winston';
import { Projects, Resources } from '@gitbeaker/core/dist/types';
import { Resources } from '@gitbeaker/core/dist/types';
import { AccessLevel, GroupSchema, MemberSchema, ProjectSchema, UserSchema } from '@gitbeaker/core/dist/types/types';
import { v4 } from 'uuid';
import { EclipseAPI, EclipseApiConfig } from '../eclipse/EclipseAPI';
import { getLogger } from '../helpers/logger';
import { SecretReader, getBaseConfig } from '../helpers/SecretReader';
import { EclipseProject, EclipseUser } from '../interfaces/EclipseApi';
import { EclipseAPI, EclipseApiConfig } from '../../eclipse/EclipseAPI';
import { getLogger } from '../../helpers/logger';
import { SecretReader, getBaseConfig } from '../../helpers/SecretReader';
import { EclipseProject, EclipseUser } from '../../interfaces/EclipseApi';
// used to make use of default requested based on Got rather than recreating our own
import { Gitlab } from '@gitbeaker/core';
......
......@@ -36,9 +36,9 @@ const SPLICE_CURRENT_ENTRY_ONLY = 1;
const SUBSTRING_SECOND_LAST_CHARACTER_TARGET = 2;
// custom wrappers
const Wrapper = require('../GitWrapper.js');
const CachedHttp = require('../HttpWrapper.js');
const EclipseAPI = require('../EclipseAPI.js');
const Wrapper = require('../../GitWrapper.js');
const CachedHttp = require('../../HttpWrapper.js');
const EclipseAPI = require('../../EclipseAPI.js');
const axios = require('axios');
var readline = require('readline');
......
......@@ -33,9 +33,9 @@ var argv = require('yargs')
const MB_IN_BYTES = 1024;
// custom wrappers
const Wrapper = require('../GitWrapper.js');
const CachedHttp = require('../HttpWrapper.js');
const EclipseAPI = require('../EclipseAPI.js');
const Wrapper = require('../../GitWrapper.js');
const CachedHttp = require('../../HttpWrapper.js');
const EclipseAPI = require('../../EclipseAPI.js');
const readline = require('readline');
// create global placeholder for wrapper
var wrap;
......
{
"extension": ["ts"],
"spec": "test/ts/**/*.ts",
"require": "ts-node/register"
}
\ No newline at end of file
import { expect } from 'chai';
import { EclipseAPI } from '../../src/eclipse/EclipseAPI';
describe('EclipseAPI', function () {
var eclipseAPI = new EclipseAPI({});
describe('#eclipseAPI', function () {
describe('success', function () {
var result;
before(async function () {
// get eclipse projects, disable pagination as this is a long process
result = await eclipseAPI.eclipseAPI('', false);
});
it('should contain JSON data', function () {
expect(result).to.be.an('array');
});
it('should contain project_id field', function () {
if (result.length > 0) {
expect(result[0]).to.have.property('project_id').that.is.a('string');
}
});
it('should contain github_repos field', function () {
if (result.length > 0) {
expect(result[0]).to.have.property('github_repos').that.is.an('array');
}
});
it('should contain gitlab_repos field', function () {
if (result.length > 0) {
expect(result[0]).to.have.property('gitlab_repos').that.is.an('array');
}
});
});
});
describe('#eclipseBots', function () {
describe('success', function () {
var result;
before(async function () {
// get the eclipse bots for api
result = await eclipseAPI.eclipseBots();
});
it('should contain JSON data', function () {
expect(result).to.be.an('array');
});
it('should contain projectId string value', function () {
// validate data required in further calls.
expect(result[0]).to.have.property('projectId').that.is.a('string');
});
});
});
describe('#processBots', function () {
describe('success', function () {
var result;
var bots;
var siteName;
before(async function () {
// get current bots and find a site name to filter on
bots = [
{
id: 1,
projectId: 'ecd.che',
username: 'genie.che',
email: 'che-bot@eclipse.org',
'github.com': {
username: 'che-bot',
email: 'che-bot@eclipse.org',
},
'github.com-openshift-ci-robot': {
username: 'openshift-ci-robot',
email: 'openshift-ci-robot@users.noreply.github.com',
},
'github.com-openshift-merge-robot': {
username: 'openshift-merge-robot',
email: 'openshift-merge-robot@users.noreply.github.com',
},
'non-gh-bot-sample': {
username: 'non-gh-bot',
email: 'non-gh-bot@test.com',
},
},
{
id: 11,
projectId: 'eclipse.jdt',
username: 'genie.jdt',
email: 'jdt-bot@eclipse.org',
'oss.sonatype.org': {
username: 'jdt-dev',
email: 'jdt-dev@eclipse.org',
},
},
];
siteName = 'github.com';
// get bots with filtered list
result = eclipseAPI.processBots(bots, siteName);
});
it('should contain object data', function () {
expect(result).to.be.an('object');
});
it('should contain keys that match bot projects in the original data', function () {
// get the project IDs of bots that have an entry for the current site
var projectNames = [];
bots.forEach(b => (b[siteName] != undefined ? projectNames.push(b.projectId) : null));
expect(result).to.have.keys(projectNames);
});
it('should contain keys that have at least 1 user associated', function () {
for (var resultIdx in result) {
expect(result[resultIdx]).to.be.an('array').that.is.not.empty;
}
// expect that the JDT project isn't included (no github.com bot)
expect(result['eclipse.jdt']).to.be.undefined;
});
it('should contain keys that only have valid users associated', function () {
// check that the ECD Che project gets included
expect(result['ecd.che']).to.include.members(['openshift-ci-robot', 'openshift-merge-robot', 'che-bot']);
});
});
});
});
import { expect } from 'chai';
import { SecretReader, getBaseConfig } from '../../src/helpers/SecretReader';
describe('SecretReader', function() {
var reader = new SecretReader({ root: `${__dirname}/../secrets` });
describe('#readSecret', function() {
describe('success', function() {
var result = reader.readSecret('sample-secret');
it('should be a string', function() {
expect(result).to.be.a('string');
});
it('should be trimmed', function() {
expect(result).to.equal(result.trim());
});
it('should be equal to', function() {
expect(result).to.equal('sample-secret');
});
});
describe('alternate encoding', function() {
var result = reader.readSecret('non-utf-file', 'ascii');
it('should be a string', function() {
expect(result).to.be.a('string');
});
it('should be trimmed', function() {
expect(result).to.equal(result.trim());
});
it('should be equal to', function() {
expect(result).to.equal('us-ascii-test');
});
});
describe('no file', function() {
var result = reader.readSecret('random-file-name');
it('should be null', function() {
expect(result).to.equal(null);
});
});
});
describe('#getBaseConfig', function() {
describe('success', function() {
var result = getBaseConfig();
it('should be an object', function() {
expect(result).to.be.an('object');
});
it('should have a root property', function() {
expect(result.root).to.be.a('string').and.to.not.equal(undefined);
});
it('should have an encoding property', function() {
expect(result.encoding).to.be.a('string').and.to.not.equal(undefined);
});
});
describe('immutability', function() {
var result = getBaseConfig();
// not with these tests, root needs to be provided as it can't be guaranteed that the base
// secrets dir exists on all machines.
it('should not change when creating a new reader', function() {
// take a deep copy to avoid issues with references and data changes
var resultBase = JSON.parse(JSON.stringify(result));
// try with empty configs
try {
new SecretReader({});
} catch (err) {
// expected on most machines, but not all
}
var result1 = getBaseConfig();
// with new properties
new SecretReader({ root: __dirname });
var result2 = getBaseConfig();
// with modified base properties
new SecretReader({ root: __dirname + '/../secrets', encoding: 'test' });
var result3 = getBaseConfig();
expect(resultBase.root).to.equal(result1.root).and.to.equal(result2.root).and.to.equal(result3.root).and.to.not.equal(undefined);
expect(resultBase.encoding).to.equal(result1.encoding).and.to.equal(result2.encoding).and.to.equal(result3.encoding).and.to.not.equal(undefined);
});
it('should not change when modifying value', function() {
var result1 = getBaseConfig();
var originalRoot = result1.root;
result1.root = 'different_value';
var result2 = getBaseConfig();
// they should be different objects
expect(result1).to.not.equal(result2);
// second call should equal first calls values, and not the changed value
expect(result2.root).to.equal(originalRoot).and.not.equal(result1.root);
});
});
});
});
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment