diff --git a/package-lock.json b/package-lock.json
index d96f072047706163159ef826143b5ee939c4416d..194524c2b12f896c6f20fe955e3f1186b4b26b28 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -14,10 +14,12 @@
"@octokit/plugin-retry": "^3.0.3",
"@octokit/plugin-throttling": "^3.3.0",
"@octokit/rest": "^18.0.3",
+ "@types/got": "^9.6.12",
"@types/node": "^17.0.32",
"@types/yargs": "^17.0.10",
"axios": "^0.21.4",
"flat-cache": "^2.0.1",
+ "got": "^12.0.4",
"nodemailer": "^6.5.0",
"openid-client": "^3.15.6",
"parse-link-header": "^2.0.0",
@@ -738,6 +740,17 @@
"resolved": "https://registry.npmjs.org/@sideway/pinpoint/-/pinpoint-2.0.0.tgz",
"integrity": "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ=="
},
+ "node_modules/@sindresorhus/is": {
+ "version": "4.6.0",
+ "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz",
+ "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==",
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sindresorhus/is?sponsor=1"
+ }
+ },
"node_modules/@sinonjs/commons": {
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz",
@@ -773,6 +786,28 @@
"integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==",
"dev": true
},
+ "node_modules/@szmarczak/http-timer": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz",
+ "integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==",
+ "dependencies": {
+ "defer-to-connect": "^2.0.1"
+ },
+ "engines": {
+ "node": ">=14.16"
+ }
+ },
+ "node_modules/@types/cacheable-request": {
+ "version": "6.0.2",
+ "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz",
+ "integrity": "sha512-B3xVo+dlKM6nnKTcmm5ZtY/OL8bOAOd2Olee9M1zft65ox50OzjEHW91sDiU9j6cvW8Ejg1/Qkf4xd2kugApUA==",
+ "dependencies": {
+ "@types/http-cache-semantics": "*",
+ "@types/keyv": "*",
+ "@types/node": "*",
+ "@types/responselike": "*"
+ }
+ },
"node_modules/@types/got": {
"version": "9.6.12",
"resolved": "https://registry.npmjs.org/@types/got/-/got-9.6.12.tgz",
@@ -796,6 +831,24 @@
"node": ">= 0.12"
}
},
+ "node_modules/@types/http-cache-semantics": {
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz",
+ "integrity": "sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ=="
+ },
+ "node_modules/@types/json-buffer": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/@types/json-buffer/-/json-buffer-3.0.0.tgz",
+ "integrity": "sha512-3YP80IxxFJB4b5tYC2SUPwkg0XQLiu0nWvhRgEatgjf+29IcWO9X1k8xRv5DGssJ/lCrjYTjQPcobJr2yWIVuQ=="
+ },
+ "node_modules/@types/keyv": {
+ "version": "3.1.4",
+ "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz",
+ "integrity": "sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==",
+ "dependencies": {
+ "@types/node": "*"
+ }
+ },
"node_modules/@types/node": {
"version": "17.0.32",
"resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.32.tgz",
@@ -807,6 +860,14 @@
"integrity": "sha512-KbqcQLdRaawDOfXnwqr6nvhe1MV+Uv/Ww+ViSx7Ujgw9X5qCgObLP52B1ZSJqZD8FK1y/4o+bJQTUrZOynegcg==",
"dev": true
},
+ "node_modules/@types/responselike": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz",
+ "integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==",
+ "dependencies": {
+ "@types/node": "*"
+ }
+ },
"node_modules/@types/simple-oauth2": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/@types/simple-oauth2/-/simple-oauth2-4.1.1.tgz",
@@ -1065,6 +1126,31 @@
"node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
}
},
+ "node_modules/cacheable-lookup": {
+ "version": "6.0.4",
+ "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-6.0.4.tgz",
+ "integrity": "sha512-mbcDEZCkv2CZF4G01kr8eBd/5agkt9oCqz75tJMSIsquvRZ2sL6Hi5zGVKi/0OSC9oO1GHfJ2AV0ZIOY9vye0A==",
+ "engines": {
+ "node": ">=10.6.0"
+ }
+ },
+ "node_modules/cacheable-request": {
+ "version": "7.0.2",
+ "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.2.tgz",
+ "integrity": "sha512-pouW8/FmiPQbuGpkXQ9BAPv/Mo5xDGANgSNXzTzJ8DrKGuXOssM4wIQRjfanNRh3Yu5cfYPvcorqbhg2KIJtew==",
+ "dependencies": {
+ "clone-response": "^1.0.2",
+ "get-stream": "^5.1.0",
+ "http-cache-semantics": "^4.0.0",
+ "keyv": "^4.0.0",
+ "lowercase-keys": "^2.0.0",
+ "normalize-url": "^6.0.1",
+ "responselike": "^2.0.0"
+ },
+ "engines": {
+ "node": ">=8"
+ }
+ },
"node_modules/call-bind": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
@@ -1300,6 +1386,18 @@
"node": ">= 0.8"
}
},
+ "node_modules/compress-brotli": {
+ "version": "1.3.8",
+ "resolved": "https://registry.npmjs.org/compress-brotli/-/compress-brotli-1.3.8.tgz",
+ "integrity": "sha512-lVcQsjhxhIXsuupfy9fmZUFtAIdBmXA7EGY6GBdgZ++qkM9zG4YFT8iU7FoBxzryNDMOpD1HIFHUSX4D87oqhQ==",
+ "dependencies": {
+ "@types/json-buffer": "~3.0.0",
+ "json-buffer": "~3.0.1"
+ },
+ "engines": {
+ "node": ">= 12"
+ }
+ },
"node_modules/concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -1368,6 +1466,31 @@
"node": ">=0.10"
}
},
+ "node_modules/decompress-response": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
+ "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
+ "dependencies": {
+ "mimic-response": "^3.1.0"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/decompress-response/node_modules/mimic-response": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
+ "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"node_modules/deep-eql": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz",
@@ -1386,6 +1509,14 @@
"integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
"dev": true
},
+ "node_modules/defer-to-connect": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
+ "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==",
+ "engines": {
+ "node": ">=10"
+ }
+ },
"node_modules/define-properties": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
@@ -2051,6 +2182,11 @@
"node": ">= 6"
}
},
+ "node_modules/form-data-encoder": {
+ "version": "1.7.1",
+ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.1.tgz",
+ "integrity": "sha512-EFRDrsMm/kyqbTQocNvRXMLjc7Es2Vk+IQFx/YW7hkUH1eBl4J1fqiP34l74Yt0pFLCNpc06fkbVk00008mzjg=="
+ },
"node_modules/fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -2190,6 +2326,62 @@
"node": ">=4"
}
},
+ "node_modules/got": {
+ "version": "12.0.4",
+ "resolved": "https://registry.npmjs.org/got/-/got-12.0.4.tgz",
+ "integrity": "sha512-2Eyz4iU/ktq7wtMFXxzK7g5p35uNYLLdiZarZ5/Yn3IJlNEpBd5+dCgcAyxN8/8guZLszffwe3wVyw+DEVrpBg==",
+ "dependencies": {
+ "@sindresorhus/is": "^4.6.0",
+ "@szmarczak/http-timer": "^5.0.1",
+ "@types/cacheable-request": "^6.0.2",
+ "@types/responselike": "^1.0.0",
+ "cacheable-lookup": "^6.0.4",
+ "cacheable-request": "^7.0.2",
+ "decompress-response": "^6.0.0",
+ "form-data-encoder": "1.7.1",
+ "get-stream": "^6.0.1",
+ "http2-wrapper": "^2.1.10",
+ "lowercase-keys": "^3.0.0",
+ "p-cancelable": "^3.0.0",
+ "responselike": "^2.0.0"
+ },
+ "engines": {
+ "node": ">=14.16"
+ },
+ "funding": {
+ "url": "https://github.com/sindresorhus/got?sponsor=1"
+ }
+ },
+ "node_modules/got/node_modules/get-stream": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
+ "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/got/node_modules/lowercase-keys": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",
+ "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==",
+ "engines": {
+ "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/got/node_modules/p-cancelable": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz",
+ "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==",
+ "engines": {
+ "node": ">=12.20"
+ }
+ },
"node_modules/growl": {
"version": "1.10.5",
"resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz",
@@ -2268,6 +2460,18 @@
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz",
"integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ=="
},
+ "node_modules/http2-wrapper": {
+ "version": "2.1.11",
+ "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.1.11.tgz",
+ "integrity": "sha512-aNAk5JzLturWEUiuhAN73Jcbq96R7rTitAoXV54FYMatvihnpD2+6PUgU4ce3D/m5VDbw+F5CsyKSF176ptitQ==",
+ "dependencies": {
+ "quick-lru": "^5.1.1",
+ "resolve-alpn": "^1.2.0"
+ },
+ "engines": {
+ "node": ">=10.19.0"
+ }
+ },
"node_modules/ignore": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
@@ -2655,6 +2859,11 @@
"node": ">=4"
}
},
+ "node_modules/json-buffer": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
+ "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
+ },
"node_modules/json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
@@ -2686,6 +2895,15 @@
"integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==",
"dev": true
},
+ "node_modules/keyv": {
+ "version": "4.2.7",
+ "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.2.7.tgz",
+ "integrity": "sha512-HeOstD8SXvtWoQhMMBCelcUuZsiV7T7MwsADtOXT0KuwYP9nCxrSoMDeLXNDTLN3VFSuRp38JzoGbbTboq3QQw==",
+ "dependencies": {
+ "compress-brotli": "^1.3.8",
+ "json-buffer": "3.0.1"
+ }
+ },
"node_modules/kuler": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/kuler/-/kuler-1.0.1.tgz",
@@ -3081,6 +3299,17 @@
"node": ">=0.10.0"
}
},
+ "node_modules/normalize-url": {
+ "version": "6.1.0",
+ "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz",
+ "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==",
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"node_modules/object-hash": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.2.0.tgz",
@@ -3563,6 +3792,17 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/quick-lru": {
+ "version": "5.1.1",
+ "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
+ "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==",
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"node_modules/readable-stream": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -3622,6 +3862,11 @@
"resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz",
"integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg=="
},
+ "node_modules/resolve-alpn": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz",
+ "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g=="
+ },
"node_modules/resolve-from": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
@@ -3631,6 +3876,14 @@
"node": ">=4"
}
},
+ "node_modules/responselike": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz",
+ "integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==",
+ "dependencies": {
+ "lowercase-keys": "^2.0.0"
+ }
+ },
"node_modules/rimraf": {
"version": "2.6.3",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz",
@@ -5067,6 +5320,11 @@
"resolved": "https://registry.npmjs.org/@sideway/pinpoint/-/pinpoint-2.0.0.tgz",
"integrity": "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ=="
},
+ "@sindresorhus/is": {
+ "version": "4.6.0",
+ "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz",
+ "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw=="
+ },
"@sinonjs/commons": {
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz",
@@ -5102,6 +5360,25 @@
"integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==",
"dev": true
},
+ "@szmarczak/http-timer": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz",
+ "integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==",
+ "requires": {
+ "defer-to-connect": "^2.0.1"
+ }
+ },
+ "@types/cacheable-request": {
+ "version": "6.0.2",
+ "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz",
+ "integrity": "sha512-B3xVo+dlKM6nnKTcmm5ZtY/OL8bOAOd2Olee9M1zft65ox50OzjEHW91sDiU9j6cvW8Ejg1/Qkf4xd2kugApUA==",
+ "requires": {
+ "@types/http-cache-semantics": "*",
+ "@types/keyv": "*",
+ "@types/node": "*",
+ "@types/responselike": "*"
+ }
+ },
"@types/got": {
"version": "9.6.12",
"resolved": "https://registry.npmjs.org/@types/got/-/got-9.6.12.tgz",
@@ -5124,6 +5401,24 @@
}
}
},
+ "@types/http-cache-semantics": {
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz",
+ "integrity": "sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ=="
+ },
+ "@types/json-buffer": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/@types/json-buffer/-/json-buffer-3.0.0.tgz",
+ "integrity": "sha512-3YP80IxxFJB4b5tYC2SUPwkg0XQLiu0nWvhRgEatgjf+29IcWO9X1k8xRv5DGssJ/lCrjYTjQPcobJr2yWIVuQ=="
+ },
+ "@types/keyv": {
+ "version": "3.1.4",
+ "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz",
+ "integrity": "sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==",
+ "requires": {
+ "@types/node": "*"
+ }
+ },
"@types/node": {
"version": "17.0.32",
"resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.32.tgz",
@@ -5135,6 +5430,14 @@
"integrity": "sha512-KbqcQLdRaawDOfXnwqr6nvhe1MV+Uv/Ww+ViSx7Ujgw9X5qCgObLP52B1ZSJqZD8FK1y/4o+bJQTUrZOynegcg==",
"dev": true
},
+ "@types/responselike": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz",
+ "integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==",
+ "requires": {
+ "@types/node": "*"
+ }
+ },
"@types/simple-oauth2": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/@types/simple-oauth2/-/simple-oauth2-4.1.1.tgz",
@@ -5335,6 +5638,25 @@
"picocolors": "^1.0.0"
}
},
+ "cacheable-lookup": {
+ "version": "6.0.4",
+ "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-6.0.4.tgz",
+ "integrity": "sha512-mbcDEZCkv2CZF4G01kr8eBd/5agkt9oCqz75tJMSIsquvRZ2sL6Hi5zGVKi/0OSC9oO1GHfJ2AV0ZIOY9vye0A=="
+ },
+ "cacheable-request": {
+ "version": "7.0.2",
+ "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.2.tgz",
+ "integrity": "sha512-pouW8/FmiPQbuGpkXQ9BAPv/Mo5xDGANgSNXzTzJ8DrKGuXOssM4wIQRjfanNRh3Yu5cfYPvcorqbhg2KIJtew==",
+ "requires": {
+ "clone-response": "^1.0.2",
+ "get-stream": "^5.1.0",
+ "http-cache-semantics": "^4.0.0",
+ "keyv": "^4.0.0",
+ "lowercase-keys": "^2.0.0",
+ "normalize-url": "^6.0.1",
+ "responselike": "^2.0.0"
+ }
+ },
"call-bind": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
@@ -5521,6 +5843,15 @@
"delayed-stream": "~1.0.0"
}
},
+ "compress-brotli": {
+ "version": "1.3.8",
+ "resolved": "https://registry.npmjs.org/compress-brotli/-/compress-brotli-1.3.8.tgz",
+ "integrity": "sha512-lVcQsjhxhIXsuupfy9fmZUFtAIdBmXA7EGY6GBdgZ++qkM9zG4YFT8iU7FoBxzryNDMOpD1HIFHUSX4D87oqhQ==",
+ "requires": {
+ "@types/json-buffer": "~3.0.0",
+ "json-buffer": "~3.0.1"
+ }
+ },
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -5574,6 +5905,21 @@
"resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz",
"integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU="
},
+ "decompress-response": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
+ "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
+ "requires": {
+ "mimic-response": "^3.1.0"
+ },
+ "dependencies": {
+ "mimic-response": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
+ "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="
+ }
+ }
+ },
"deep-eql": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz",
@@ -5589,6 +5935,11 @@
"integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
"dev": true
},
+ "defer-to-connect": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
+ "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg=="
+ },
"define-properties": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
@@ -6096,6 +6447,11 @@
"mime-types": "^2.1.12"
}
},
+ "form-data-encoder": {
+ "version": "1.7.1",
+ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.1.tgz",
+ "integrity": "sha512-EFRDrsMm/kyqbTQocNvRXMLjc7Es2Vk+IQFx/YW7hkUH1eBl4J1fqiP34l74Yt0pFLCNpc06fkbVk00008mzjg=="
+ },
"fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -6194,6 +6550,43 @@
"dev": true,
"peer": true
},
+ "got": {
+ "version": "12.0.4",
+ "resolved": "https://registry.npmjs.org/got/-/got-12.0.4.tgz",
+ "integrity": "sha512-2Eyz4iU/ktq7wtMFXxzK7g5p35uNYLLdiZarZ5/Yn3IJlNEpBd5+dCgcAyxN8/8guZLszffwe3wVyw+DEVrpBg==",
+ "requires": {
+ "@sindresorhus/is": "^4.6.0",
+ "@szmarczak/http-timer": "^5.0.1",
+ "@types/cacheable-request": "^6.0.2",
+ "@types/responselike": "^1.0.0",
+ "cacheable-lookup": "^6.0.4",
+ "cacheable-request": "^7.0.2",
+ "decompress-response": "^6.0.0",
+ "form-data-encoder": "1.7.1",
+ "get-stream": "^6.0.1",
+ "http2-wrapper": "^2.1.10",
+ "lowercase-keys": "^3.0.0",
+ "p-cancelable": "^3.0.0",
+ "responselike": "^2.0.0"
+ },
+ "dependencies": {
+ "get-stream": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
+ "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg=="
+ },
+ "lowercase-keys": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",
+ "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ=="
+ },
+ "p-cancelable": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz",
+ "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw=="
+ }
+ }
+ },
"growl": {
"version": "1.10.5",
"resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz",
@@ -6245,6 +6638,15 @@
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz",
"integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ=="
},
+ "http2-wrapper": {
+ "version": "2.1.11",
+ "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.1.11.tgz",
+ "integrity": "sha512-aNAk5JzLturWEUiuhAN73Jcbq96R7rTitAoXV54FYMatvihnpD2+6PUgU4ce3D/m5VDbw+F5CsyKSF176ptitQ==",
+ "requires": {
+ "quick-lru": "^5.1.1",
+ "resolve-alpn": "^1.2.0"
+ }
+ },
"ignore": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
@@ -6504,6 +6906,11 @@
"dev": true,
"peer": true
},
+ "json-buffer": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
+ "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
+ },
"json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
@@ -6529,6 +6936,15 @@
"integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==",
"dev": true
},
+ "keyv": {
+ "version": "4.2.7",
+ "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.2.7.tgz",
+ "integrity": "sha512-HeOstD8SXvtWoQhMMBCelcUuZsiV7T7MwsADtOXT0KuwYP9nCxrSoMDeLXNDTLN3VFSuRp38JzoGbbTboq3QQw==",
+ "requires": {
+ "compress-brotli": "^1.3.8",
+ "json-buffer": "3.0.1"
+ }
+ },
"kuler": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/kuler/-/kuler-1.0.1.tgz",
@@ -6842,6 +7258,11 @@
"integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
"dev": true
},
+ "normalize-url": {
+ "version": "6.1.0",
+ "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz",
+ "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A=="
+ },
"object-hash": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.2.0.tgz",
@@ -7198,6 +7619,11 @@
"strict-uri-encode": "^2.0.0"
}
},
+ "quick-lru": {
+ "version": "5.1.1",
+ "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
+ "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA=="
+ },
"readable-stream": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -7239,12 +7665,25 @@
"resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz",
"integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg=="
},
+ "resolve-alpn": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz",
+ "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g=="
+ },
"resolve-from": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
"integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
"dev": true
},
+ "responselike": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz",
+ "integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==",
+ "requires": {
+ "lowercase-keys": "^2.0.0"
+ }
+ },
"rimraf": {
"version": "2.6.3",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz",
diff --git a/package.json b/package.json
index c844ae5a81a8e93acc85ded8f48fee2544c7cdee..0f2be11f6e3b823af79410c1c29d31c045ac7227 100644
--- a/package.json
+++ b/package.json
@@ -25,10 +25,12 @@
"@octokit/plugin-retry": "^3.0.3",
"@octokit/plugin-throttling": "^3.3.0",
"@octokit/rest": "^18.0.3",
+ "@types/got": "^9.6.12",
"@types/node": "^17.0.32",
"@types/yargs": "^17.0.10",
"axios": "^0.21.4",
"flat-cache": "^2.0.1",
+ "got": "^12.0.4",
"nodemailer": "^6.5.0",
"openid-client": "^3.15.6",
"parse-link-header": "^2.0.0",
diff --git a/src/eclipse/EclipseAPI.ts b/src/eclipse/EclipseAPI.ts
index 908398e5430ecf9c02ee02fefa1e2ae0f19e4b9f..d282a57653e9e4da6cfb6ea37e11944575a257b4 100644
--- a/src/eclipse/EclipseAPI.ts
+++ b/src/eclipse/EclipseAPI.ts
@@ -251,7 +251,7 @@ const testProjects: EclipseProject[] = [
],
gitlab_repos: [
{
- url: 'https://gitlab.eclipse.org/eclipsefdn/webdev/gitlab-testing',
+ url: 'http://localhost/eclipse/spider.pig',
},
],
contributors: [],
diff --git a/src/gl/AxiosRequester.ts b/src/gl/AxiosRequester.ts
index ccff72bacf5235af5d92811a4fd2dbf58f9e94f6..8185cf41e85bb810bddaa8e4a4865bba8d397348 100644
--- a/src/gl/AxiosRequester.ts
+++ b/src/gl/AxiosRequester.ts
@@ -1,21 +1,95 @@
-import {RequesterType} from '@gitbeaker/requester-utils';
import axios from 'axios';
+import { decamelizeKeys } from 'xcase';
+import {
+ DefaultResourceOptions,
+ DefaultRequestReturn,
+ DefaultRequestOptions,
+ createRequesterFn,
+ defaultOptionsHandler as baseOptionsHandler,
+} from '@gitbeaker/requester-utils';
-export class AxiosRequester implements RequesterType {
- get(endpoint: string, options?: Record<string, unknown>): Promise<any> {
- throw new Error('Method not implemented.');
- }
- post(endpoint: string, options?: Record<string, unknown>): Promise<any> {
- throw new Error('Method not implemented.');
- }
- put(endpoint: string, options?: Record<string, unknown>): Promise<any> {
- throw new Error('Method not implemented.');
- }
- delete(endpoint: string, options?: Record<string, unknown>): Promise<any> {
- throw new Error('Method not implemented.');
- }
- stream?(endpoint: string, options?: Record<string, unknown>): NodeJS.ReadableStream {
- throw new Error('Method not implemented.');
+export function defaultOptionsHandler(
+ resourceOptions: DefaultResourceOptions,
+ { body, query, sudo, method }: DefaultRequestOptions = {}
+): DefaultRequestReturn & {
+ json?: Record<string, unknown>;
+ https?: { rejectUnauthorized: boolean };
+} {
+ const options: DefaultRequestReturn & {
+ json?: Record<string, unknown>;
+ https?: { rejectUnauthorized: boolean };
+ } = baseOptionsHandler(resourceOptions, { body, query, sudo, method });
+
+ // FIXME: Not the best comparison, but...it will have to do for now.
+ if (typeof body === 'object' && body.constructor.name !== 'FormData') {
+ options.json = decamelizeKeys(body);
+
+ delete options.body;
+ }
+
+ if (resourceOptions.url.includes('https') && resourceOptions.rejectUnauthorized != null && resourceOptions.rejectUnauthorized === false) {
+ options.https = {
+ rejectUnauthorized: resourceOptions.rejectUnauthorized,
+ };
+ }
+
+ return options;
+}
+
+export function processBody({ data, headers }: { data: Buffer; headers: Record<string, unknown> }) {
+ // Split to remove potential charset info from the content type
+ const contentType = ((headers['content-type'] as string) || '').split(';')[0].trim();
+
+ if (contentType === 'application/json') {
+ return data.length === 0 ? {} : data;
+ }
+
+ if (contentType.startsWith('text/')) {
+ return data.toString();
+ }
+
+ return Buffer.from(data);
+}
+
+export async function handler(endpoint: string, options: Record<string, unknown>) {
+ const retryCodes = [429, 502];
+ const maxRetries = 10;
+ let response;
+ // map the Got prefix to the axios base url
+ options['baseURL'] = options['prefixUrl'];
+ // map the qs glob into an object
+ options['params'] = Object.fromEntries(new URLSearchParams(options['searchParams'] as string));
+ options['data'] = options['json'];
+ delete options['json'];
+ for (let i = 0; i < maxRetries; i += 1) {
+ const waitTime = 2 ** i;
+
+ try {
+ response = await axios(endpoint, { ...options }); // eslint-disable-line
+ break;
+ } catch (e) {
+ if (e.response) {
+ if (retryCodes.includes(e.response.statusCode)) {
+ await new Promise(resolve => setTimeout(resolve, waitTime));
+ continue; // eslint-disable-line
+ }
+
+ if (typeof e.response.body === 'string' && e.response.body.length > 0) {
+ try {
+ const output = JSON.parse(e.response.body);
+ e.description = output.error || output.message;
+ } catch (err) {
+ e.description = e.response.body;
+ }
+ }
+ }
+
+ throw e;
}
+ }
+ const { status, headers } = response;
+ const body = processBody(response);
+ return { body, headers, status: status };
+}
-}
\ No newline at end of file
+export const requesterFn = createRequesterFn(defaultOptionsHandler, handler);
diff --git a/src/gl/GitlabSync.ts b/src/gl/GitlabSync.ts
index 9596a57205745d726c9fbcfb20de28e2f5aee5db..f08c1915bd80111abe6ccbc55b7c1b45f6400651 100644
--- a/src/gl/GitlabSync.ts
+++ b/src/gl/GitlabSync.ts
@@ -50,8 +50,6 @@ let args = yargs(process.argv)
.version('0.1')
.alias('v', 'version')
.epilog('Copyright 2019 Eclipse Foundation inc.').argv;
-
-const runner =
run();
async function run() {
diff --git a/src/gl/GitlabSyncRunner.ts b/src/gl/GitlabSyncRunner.ts
index 558f5d57395f536a7b09735eaec8c9ffb9285b1f..9832c56fb8895a340df8845b8c3c772b346db213 100644
--- a/src/gl/GitlabSyncRunner.ts
+++ b/src/gl/GitlabSyncRunner.ts
@@ -1,14 +1,15 @@
import { Logger } from 'winston';
+import { Projects, Resources } from '@gitbeaker/core/dist/types';
import { AccessLevel, GroupSchema, MemberSchema, ProjectSchema, UserSchema } from '@gitbeaker/core/dist/types/types';
import { v4 } from 'uuid';
import { EclipseAPI, EclipseApiConfig } from '../eclipse/EclipseAPI';
import { getLogger } from '../helpers/logger';
import { SecretReader, getBaseConfig } from '../helpers/SecretReader';
import { EclipseProject, EclipseUser } from '../interfaces/EclipseApi';
-import { Resources } from '@gitbeaker/core/dist/types';
// used to make use of default requested based on Got rather than recreating our own
-import {Gitlab} from '@gitbeaker/core';
+import { Gitlab } from '@gitbeaker/core';
+import { requesterFn } from './AxiosRequester';
const ADMIN_PERMISSIONS_LEVEL = 50;
@@ -34,6 +35,7 @@ interface GitlabSyncRunnerConfig {
verbose: boolean;
devMode: boolean;
dryRun: boolean;
+ rootGroup?: string;
}
export class GitlabSyncRunner {
@@ -49,26 +51,35 @@ export class GitlabSyncRunner {
bots: Record<string, string[]> = {};
// caches to optimize calling
- namedProjects: Record<string, ProjectSchema> = {};
namedUsers: Record<string, UserSchema> = {};
groupCache: GroupCache = {
_self: null,
children: {},
projectTargets: [],
};
+ projectsCache: ProjectSchema[] = [];
eclipseProjectCache: Record<string, EclipseProject> = {};
gMems: Record<number, MemberSchema[]> = {};
- constructor(
- config: GitlabSyncRunnerConfig = {
- host: 'http://gitlab.eclipse.org/',
- provider: 'oauth2_generic',
- verbose: false,
- devMode: false,
- dryRun: false,
- }
- ) {
- this.config = config;
+ /**
+ * Sets the internal config with a few default values and creates the bindings for the APIs that are
+ * accessed during the run of this script.
+ *
+ * @param config the initial script configuration object.
+ */
+ constructor(config: GitlabSyncRunnerConfig) {
+ this.config = Object.assign(
+ {
+ host: 'http://gitlab.eclipse.org/',
+ provider: 'oauth2_generic',
+ verbose: false,
+ devMode: false,
+ dryRun: false,
+ rootGroup: 'eclipse',
+ },
+ config
+ );
+
this.logger = getLogger(this.config.verbose ? 'debug' : 'info', 'main');
this._prepareSecret();
@@ -76,7 +87,7 @@ export class GitlabSyncRunner {
this.api = new Gitlab({
host: this.config.host,
token: this.accessToken,
- requesterFn:
+ requesterFn: requesterFn,
});
let eclipseAPIConfig: EclipseApiConfig = JSON.parse(this.eclipseToken);
eclipseAPIConfig.testMode = this.config.devMode;
@@ -84,6 +95,11 @@ export class GitlabSyncRunner {
this.eApi = new EclipseAPI(eclipseAPIConfig);
}
+ /**
+ * Prepares the secrets required for the script to run. Specifically the eclipseToken used for Eclipse
+ * API access and the accessToken which is used for sudo+api access on the Gitlab instance targeted by
+ * this script.
+ */
_prepareSecret() {
// retrieve the secret API file root if set
var settings = getBaseConfig();
@@ -108,19 +124,23 @@ export class GitlabSyncRunner {
}
}
- async run() {
+ /**
+ * Run the full sync script, syncing the PMI to the Gitlab instance targeted by the script. This script
+ * will sync the namespace groups named in projects with the users that are set as members of the project.
+ * It will also clear users added outside of this process with the exception of bot users to maintain more
+ * strict control of the access permissions.
+ *
+ * @returns a promise that is completed once the run completes.
+ */
+ async run(): Promise<void> {
// prepopulate caches to optimally retrieve info used in sync ops
await this.prepareCaches();
// fetch org group from results, create if missing
this.logger.info('Starting sync');
- var g = await this.getGroup('eclipse');
- if (g === undefined) {
- if (this.config.dryRun) {
- this.logger.error('Unable to start sync of GitLab content. Base Eclipse group could not be found and dryrun is set');
- } else {
- this.logger.error('Unable to start sync of GitLab content. Base Eclipse group could not be created');
- }
+ var g = this.getRootGroup();
+ if (g._self === null) {
+ this.logger.error(`Unable to start sync of GitLab content. Base group (${this.config.rootGroup}) could not be found`);
return;
}
@@ -142,16 +162,18 @@ export class GitlabSyncRunner {
let [host, namespace] = this.splitNamespaceUrl(project.gitlab_repos[idx].url);
// make sure namespace URL is valid
if (host === null || namespace === null) {
- continue;
+ this.logger.error(`Could not generate namespace/host from namespace URL: ${project.gitlab_repos[idx].url}`);
}
// check if hosts are the same ignoring case, skipping if they are different
- if (host.localeCompare(this.config.host, undefined, { sensitivity: 'base' }) !== 0) {
+ if (host.localeCompare(new URL(this.config.host).hostname, undefined, { sensitivity: 'base' }) !== 0) {
+ this.logger.error(`Found host '${host}' when processing for '${new URL(this.config.host).hostname}', skipping`);
continue;
}
// check group cache to ensure well formed.
let namespaceGroup = this.getCachedGroup(namespace);
if (namespaceGroup === null || namespaceGroup._self === null) {
+ this.logger.error(`Could not find group with namespace ${namespace}`);
continue;
}
// update the group to add the users for the current project
@@ -165,13 +187,15 @@ export class GitlabSyncRunner {
await this.addUserToGroup(user, namespaceGroup._self!, userList[uname].accessLevel);
// if not tracked, track current project for group for post-sync cleanup
- if (namespaceGroup.projectTargets.indexOf(project.short_project_id) !== -1) {
+ if (namespaceGroup.projectTargets.indexOf(project.short_project_id) === -1) {
namespaceGroup.projectTargets.push(project.short_project_id);
}
}
}
}
+ // perform cleanup operations to clean out extra users
this.cleanupGroups();
+ this.cleanupProjects();
}
/**
@@ -179,30 +203,31 @@ export class GitlabSyncRunner {
*/
async prepareCaches() {
// get raw project data and post process to add additional context
- let data = await this.eApi.eclipseAPI();
+ try {
+ let data = await this.eApi.eclipseAPI();
- // get the bots for the projects
- let rawBots = await this.eApi.eclipseBots();
- this.bots = this.eApi.processBots(rawBots, 'gitlab.eclipse.org');
+ // get the bots for the projects
+ let rawBots = await this.eApi.eclipseBots();
+ this.bots = this.eApi.processBots(rawBots, 'gitlab.eclipse.org');
- // get all current groups for the instance
- var groups = await this.api.Groups.all();
- var projects = await this.api.Projects.all();
- var users = await this.api.Users.all();
+ // get all current groups for the instance
+ this.projectsCache = await this.api.Projects.all();
+ var groups = await this.api.Groups.all();
+ var users = await this.api.Users.all();
- // generates the nested cache
- this.generateGroupsCache(groups);
- // maps path + parent to create unique tokens to match projects
- for (var projectIdx in projects) {
- this.namedProjects[this.getCompositeProjectKey(projects[projectIdx].name, projects[projectIdx].namespace.id)] = projects[projectIdx];
- }
- // map the users to their usernames for easy lookups
- for (var userIdx in users) {
- this.namedUsers[users[userIdx].username] = users[userIdx];
- }
- for (let projectIdx in data) {
- let p = data[projectIdx];
- this.eclipseProjectCache[p.short_project_id] = p;
+ // generates the nested cache
+ this.generateGroupsCache(groups);
+ // map the users to their usernames for easy lookups
+ for (var userIdx in users) {
+ this.namedUsers[users[userIdx].username] = users[userIdx];
+ }
+ for (let projectIdx in data) {
+ let p = data[projectIdx];
+ this.eclipseProjectCache[p.short_project_id] = p;
+ }
+ } catch (e) {
+ this.logger.error(`Cannot fetch resources associated with sync operations, exiting: ${e}`);
+ process.exit(1);
}
}
@@ -210,7 +235,8 @@ export class GitlabSyncRunner {
* Iterate through each group, checking self and ancestor project users and comparing against the current groups users to ensure that there are no
* additional users added with permissions.
*/
- cleanupGroups(currentLevel: GroupCache = this.groupCache, collectedProjects: string[] = []) {
+ cleanupGroups(currentLevel: GroupCache = this.getRootGroup(), collectedProjects: string[] = []) {
+ this.logger.debug(`cleanupGroups(currentLevel = ${currentLevel._self?.full_path}, collectedProjects = ${collectedProjects})`);
let self = currentLevel._self;
if (self === null) {
this.logger.error('Error encountered during group cleanup process, ending early');
@@ -233,7 +259,11 @@ export class GitlabSyncRunner {
async removeAdditionalUsers(expectedUsers: Record<string, EclipseUser>, group: GroupSchema, ...projectIDs: string[]) {
if (this.config.verbose) {
- this.logger.debug(`GitlabSync:removeAdditionalUsers(expectedUsers = ${expectedUsers}, group = ${group}, projectIDs = ${projectIDs})`);
+ this.logger.debug(
+ `GitlabSync:removeAdditionalUsers(expectedUsers = ${JSON.stringify(expectedUsers)}, group = ${
+ group.full_path
+ }, projectIDs = ${projectIDs})`
+ );
}
// get the current list of users for the group
var members = await this.getGroupMembers(group);
@@ -269,7 +299,29 @@ export class GitlabSyncRunner {
});
}
- async cleanUpProjectUsers(project: ProjectSchema, projectID: string) {
+ /**
+ * Iterates over the projects cache and cleans out the users and keeps bots for build operations. Skips over projects
+ * outside the scope of the designated root group to avoid over processing groups.
+ */
+ async cleanupProjects() {
+ this.projectsCache.forEach(p => {
+ let group = this.getCachedGroup(p.namespace.full_path);
+ if (group !== null) {
+ this.cleanUpProjectUsers(p, ...group!.projectTargets);
+ } else {
+ this.logger.info(`Skipping processing of project '${p.name}'`);
+ }
+ });
+ }
+
+ /**
+ * Removes any non-owner user that isn't a bot from projects. Membership is managed at the group level, not the direct
+ * project level.
+ *
+ * @param project the Gitlab project to sanitize
+ * @param projectIDs the Eclipse projects that impact the Gitlab project.
+ */
+ async cleanUpProjectUsers(project: ProjectSchema, ...projectIDs: string[]) {
if (this.config.verbose) {
this.logger.debug(`GitlabSync:cleanUpProjectUsers(project = ${project.id})`);
}
@@ -277,7 +329,7 @@ export class GitlabSyncRunner {
for (var idx in projectMembers) {
let member = projectMembers[idx];
// skip bot user or admin users
- if (this.isBot(member.username, [projectID]) || member.access_level === ADMIN_PERMISSIONS_LEVEL) {
+ if (this.isBot(member.username, projectIDs) || member.access_level === ADMIN_PERMISSIONS_LEVEL) {
continue;
}
if (this.config.dryRun) {
@@ -296,9 +348,18 @@ export class GitlabSyncRunner {
}
}
+ /**
+ * Ensures that the user exists within the group with the given access level (no more or less). If a user has too high
+ * permissions, the membership is modified to have the given access instead.
+ *
+ * @param user the user that is being given permissions
+ * @param group group that the user should be added to
+ * @param perms the permission set to give the user
+ * @returns the membership information for the user wrt to this Gitlab group.
+ */
async addUserToGroup(user: UserSchema, group: GroupSchema, perms: AccessLevel): Promise<MemberSchema | null> {
if (this.config.verbose) {
- this.logger.debug(`GitlabSync:addUserToGroup(user = ${user}, group = ${group}, perms = ${perms})`);
+ this.logger.debug(`GitlabSync:addUserToGroup(user = ${user?.username}, group = ${group?.full_path}, perms = ${perms})`);
}
// get the members for the current group
var members = await this.getGroupMembers(group);
@@ -308,7 +369,8 @@ export class GitlabSyncRunner {
}
// check if user is already present
- members!.forEach(async (member, idx) => {
+ for (let i = 0; i < members.length; i++) {
+ let member = members[i];
if (member.username === user.username) {
this.logger.verbose(`User '${user.username}' is already a member of ${group.name}`);
if (member.access_level !== perms) {
@@ -323,7 +385,7 @@ export class GitlabSyncRunner {
try {
var updatedMember = await this.api.GroupMembers.edit(group.id, user.id, perms);
// update inner array
- members![idx] = updatedMember;
+ members![i] = updatedMember;
this.gMems[group.id] = members!;
} catch (err) {
if (this.config.verbose) {
@@ -334,9 +396,9 @@ export class GitlabSyncRunner {
}
}
// return a copy of the updated user
- return members![idx];
+ return members![i];
}
- });
+ }
// check if dry run before updating
if (this.config.dryRun) {
this.logger.info(
@@ -363,113 +425,6 @@ export class GitlabSyncRunner {
return null;
}
- async getProject(name: string, parent: GroupSchema): Promise<ProjectSchema | null> {
- if (this.config.verbose) {
- this.logger.debug(`GitlabSync:getProject(name = ${name}, parent = ${parent})`);
- }
- if (name.trim() === '.github') {
- this.logger.warn("Skipping project with name '.github'. No current equivalent to default repository in GitLab.");
- return null;
- }
-
- var p = this.namedProjects[this.getCompositeProjectKey(name, parent.id)];
- if (p === undefined) {
- this.logger.verbose(`Creating new project with name '${name}'`);
- // create the request options for the new user, any needed for addition of random properties after
- let opts;
- if (parent !== undefined) {
- opts = {
- path: name,
- visibility: 'public',
- namespace_id: parent.id,
- };
- } else {
- opts = {
- path: name,
- visibility: 'public',
- };
- }
- // check if dry run before creating new project
- if (this.config.dryRun) {
- this.logger.info(`Dryrun flag active, would have created new project '${name}' with options ${JSON.stringify(opts)}`);
- return null;
- }
-
- // create the new project, and track it
- if (this.config.verbose) {
- this.logger.debug(`Creating project with options: ${JSON.stringify(opts)}`);
- }
- try {
- p = await this.api.Projects.create(opts);
- } catch (err) {
- if (this.config.verbose) {
- this.logger.error(`${err}`);
- }
- }
- if (p === null || p instanceof Array) {
- this.logger.warn(`Error while creating project '${name}'`);
- return null;
- }
- if (this.config.verbose) {
- this.logger.debug(`Created project: ${JSON.stringify(p)}`);
- }
- // set it back
- this.namedProjects[this.getCompositeProjectKey(name, parent.id)] = p;
- }
- return p;
- }
-
- async getGroup(namespace: string, visibility: string = 'public'): Promise<GroupSchema | null> {
- if (this.config.verbose) {
- this.logger.debug(`GitlabSync:getGroup(namespace = ${namespace}, visibility = ${visibility})`);
- }
- let g: GroupSchema | null = null;
- let cachedGroup = this.getCachedGroup(namespace);
- if (cachedGroup === null) {
- let parentNamespace = namespace.substring(0, namespace.lastIndexOf('/'));
- let parent = this.getCachedGroup(parentNamespace);
- if (parent === null || parent._self === null) {
- this.logger.error(`Could not find the parent of group to fetch (${parentNamespace}), cannot fetch group ${namespace}`);
- return null;
- }
- let groupName = namespace.substring(namespace.lastIndexOf('/'), namespace.length - 1);
- this.logger.verbose(`Creating new group with name '${groupName}'`);
- var opts = {
- project_creation_level: 'maintainer',
- visibility: visibility,
- request_access_enabled: false,
- parent_id: parent._self.id,
- };
- // check if dry run before creating group
- if (this.config.dryRun) {
- this.logger.info(`Dryrun flag active, would have created new group '${groupName}' with options ${JSON.stringify(opts)}`);
- return null;
- }
-
- // if verbose is set display user opts
- if (this.config.verbose) {
- this.logger.debug(`Creating group with options: ${JSON.stringify(opts)}`);
- }
- try {
- g = await this.api.Groups.create(groupName, this.sanitizeGroupName(groupName), opts);
- } catch (err) {
- if (this.config.verbose) {
- this.logger.error(`${err}`);
- }
- }
- if (g === null || g instanceof Array) {
- this.logger.warn(`Error while creating group '${groupName}'`);
- return null;
- }
- if (this.config.verbose) {
- this.logger.debug(`Created group: ${JSON.stringify(g)}`);
- }
- // set it back
- this.addGroup(g);
- }
- return g;
- }
-
async getUser(uname: string, url: string): Promise<UserSchema | null> {
if (this.config.verbose) {
this.logger.debug(`GitlabSync:getUser(uname = ${uname}, url = ${url})`);
@@ -535,7 +490,7 @@ export class GitlabSyncRunner {
async getGroupMembers(group: GroupSchema): Promise<MemberSchema[] | null> {
if (this.config.verbose) {
- this.logger.debug(`GitlabSync:getGroupMembers(group = ${group})`);
+ this.logger.debug(`GitlabSync:getGroupMembers(group = ${group?.full_path})`);
}
var members = this.gMems[group.id];
if (members === undefined) {
@@ -574,27 +529,46 @@ export class GitlabSyncRunner {
}
}
+ getRootGroup(): GroupCache {
+ let rootGroupCache = this.groupCache.children[this.config.rootGroup];
+ if (rootGroupCache === undefined) {
+ this.logger.error(`Could not find root group '${this.config.rootGroup}' for group caching, exiting`);
+ process.exit(1);
+ }
+ return rootGroupCache;
+ }
+
getCachedGroup(namespace: string): GroupCache | null {
if (this.config.verbose) {
this.logger.debug(`GitlabSync:getCachedGroup(${namespace})`);
}
+ if (!namespace.startsWith(this.config.rootGroup)) {
+ this.logger.info(`Returning null for ${namespace} as it is outside of the root group ${this.config.rootGroup}`);
+ return null;
+ }
return this.tunnelAndRetrieve(namespace.split('/'), this.groupCache);
}
addGroup(g: GroupSchema, parent: GroupCache = this.groupCache): GroupCache | null {
+ if (this.config.verbose) {
+ this.logger.debug(`GitlabSync:addGroup(g = ${g.id})`);
+ }
// type the namespace as it is a raw type, make sure its the type we expect
- if (typeof g.path_with_namespace !== 'string') {
- this.logger.error(`Could not cast namespace to string: ${g.path_with_namespace}`);
- // TODO this should short circuit as not having groups is a non-starter
+ if (typeof g.full_path !== 'string') {
+ this.logger.error(`Could not cast namespace to string: ${g.full_path}`);
return null;
}
- let namespace = g.path_with_namespace as string;
+ let namespace = g.full_path;
// split into group namespace paths (eclipse/sample/group.path into ['eclipse','sample','group.path'])
let namespaceParts = namespace.split('/');
return this.tunnelAndInsert(namespaceParts, g, this.groupCache);
}
tunnelAndInsert(namespaceParts: string[], g: GroupSchema, parent: GroupCache): GroupCache {
+ if (this.config.verbose) {
+ this.logger.debug(`GitlabSync:tunnelAndInsert(namespaceParts = '${namespaceParts}', g = ${g.id})`);
+ }
+
let child = parent.children[namespaceParts[0]];
if (child === undefined) {
child = {
@@ -606,29 +580,46 @@ export class GitlabSyncRunner {
}
// check if we should continue tunneling or insert and finish processing
if (namespaceParts.length > 1) {
- return this.tunnelAndInsert(namespaceParts.slice(1, namespaceParts.length - 1), g, child);
+ return this.tunnelAndInsert(namespaceParts.slice(1, namespaceParts.length), g, child);
} else {
child._self = g;
return child;
}
}
+ /**
+ * Recursive access to the nested group cache. Retrieves the group described by the namespace parts and returns
+ * it, returning null if it can't be found.
+ *
+ * @param namespaceParts the full path for a group namespace split into parts
+ * @param parent the parent to search through for the next part of the recursive call.
+ * @returns The group cache for the designated group, or null if it can't be found.
+ */
tunnelAndRetrieve(namespaceParts: string[], parent: GroupCache): GroupCache | null {
+ if (this.config.verbose) {
+ this.logger.debug(`GitlabSync:tunnelAndRetrieve(namespaceParts = '${namespaceParts}')`);
+ }
let child = parent.children[namespaceParts[0]];
if (child === undefined) {
return null;
}
// check if we should continue tunneling or insert and finish processing
if (namespaceParts.length > 1) {
- return this.tunnelAndRetrieve(namespaceParts.slice(1, namespaceParts.length - 1), child);
+ return this.tunnelAndRetrieve(namespaceParts.slice(1, namespaceParts.length), child);
} else {
return child;
}
}
+ /**
+ * Gets list of users with access permissions for the given Eclipse project.
+ *
+ * @param project the Eclipse project to parse user entries for
+ * @returns the mapping of users to access permissions and entity access URL.
+ */
getUserList(project: EclipseProject): Record<string, EclipseUserAccess> {
if (this.config.verbose) {
- this.logger.debug(`GitlabSync:getUserList(project = ${JSON.stringify(project)})`);
+ this.logger.debug(`GitlabSync:getUserList(project = ${project.short_project_id})`);
}
var l: Record<string, EclipseUserAccess> = {};
// add the contributors with reporter access
@@ -665,6 +656,12 @@ export class GitlabSyncRunner {
return l;
}
+ /**
+ * Sanitizes and normalizes strings for use in creating/accessing groups.
+ *
+ * @param pid the project ID to normalize
+ * @returns normalized group name for value.
+ */
sanitizeGroupName(pid: string): string {
if (this.config.verbose) {
this.logger.debug(`GitlabSync:sanitizeGroupName(pid = ${pid})`);
@@ -675,10 +672,6 @@ export class GitlabSyncRunner {
return '';
}
- getCompositeProjectKey(projectName: string, parentId: number): string {
- return projectName + ':' + parentId;
- }
-
/**
* Uses TS URL type to ingest a namespace URL and return the host and namespace for use downstream
*
@@ -688,7 +681,7 @@ export class GitlabSyncRunner {
splitNamespaceUrl(rawUrl: string): [string | null, string | null] {
try {
let url = new URL(rawUrl);
- return [url.host, url.pathname.substring(1, url.pathname.length - 1)];
+ return [url.host, url.pathname.substring(1, url.pathname.length)];
} catch (e) {
// cast and message with error
let message = '';
@@ -701,6 +694,14 @@ export class GitlabSyncRunner {
}
return [null, null];
}
+
+ /**
+ * Checks whether a user is a bot for the given projects.
+ *
+ * @param uname potential bot username
+ * @param projectIDs the projects that the user could be a bot for.
+ * @returns true if the user is a designated bot for the projects, otherwise false.
+ */
isBot(uname: string, projectIDs: string[]): boolean {
for (let pidx in projectIDs) {
var botList = this.bots[projectIDs[pidx]];
diff --git a/tsconfig.json b/tsconfig.json
index 45480bf8b2a2e474d24e758c85faac35bc6bcb01..cfc7ab0c8c1d10e2d742308058554e406e040bf2 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -1,6 +1,5 @@
{
"compilerOptions": {
- "module": "CommonJS",
"esModuleInterop": true,
"sourceMap": true
},