From dc09a0138996d0a490030b4e66a7d84815682a12 Mon Sep 17 00:00:00 2001 From: tuanh <tuan.hoangdinhanh@vn.bosch.com> Date: Tue, 14 Jan 2025 16:23:19 +0700 Subject: [PATCH 1/8] update logic of compute / process api data --- src/controllers/model.controller.js | 5 ++- src/services/api.service.js | 1 - src/services/model.service.js | 58 +++++++++++++++++++++++------ 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/controllers/model.controller.js b/src/controllers/model.controller.js index c0bdf51..89da0c8 100644 --- a/src/controllers/model.controller.js +++ b/src/controllers/model.controller.js @@ -12,8 +12,9 @@ const createModel = catchAsync(async (req, res) => { if (api_data_url) { const result = await modelService.processApiDataUrl(api_data_url); if (result) { - extended_apis = result.extended_apis || extended_apis; - reqBody.api_version = result.api_version || reqBody.api_version; + extended_apis = result.extended_apis; + reqBody.api_version = result.api_version; + reqBody.main_api = result.main_api; } } diff --git a/src/services/api.service.js b/src/services/api.service.js index a71fcbb..7c181ca 100644 --- a/src/services/api.service.js +++ b/src/services/api.service.js @@ -135,7 +135,6 @@ const computeVSSApi = async (modelId) => { const extendedApis = await ExtendedApi.find({ model: modelId, - isWishlist: true, }); extendedApis.forEach((extendedApi) => { try { diff --git a/src/services/model.service.js b/src/services/model.service.js index 7d540f7..8dd5769 100644 --- a/src/services/model.service.js +++ b/src/services/model.service.js @@ -329,11 +329,20 @@ const getAccessibleModels = async (userId) => { const convertToExtendedApiFormat = (api) => { const { name, ...rest } = api; return { - apiName: name, ...rest, + apiName: name, }; }; +const traverse = (api, callback, prefix = '') => { + if (api.children) { + for (const [key, child] of Object.entries(api.children)) { + traverse(child, callback, `${prefix}.${key}`); + } + } + callback(api, prefix); +}; + /** * * @param {string} apiDataUrl @@ -343,22 +352,29 @@ const processApiDataUrl = async (apiDataUrl) => { try { const response = await fetch(apiDataUrl); const data = await response.json(); - const wishlist = []; + const extendedApis = []; const mainApi = Object.keys(data).at(0) || 'Vehicle'; - Object.entries(data[mainApi].children).forEach(([key, value]) => { - if (value.isWishlist) { - wishlist.push(convertToExtendedApiFormat(data[mainApi].children[key])); - delete data[mainApi].children[key]; - } - }); + // Detached wishlist APIs + traverse( + data[mainApi], + (api, prefix) => { + for (const [key, value] of Object.entries(api.children || {})) { + if (value.isWishlist) { + extendedApis.push(convertToExtendedApiFormat(value)); + delete api.children[key]; + } + } + }, + mainApi + ); - const result = {}; - if (wishlist.length > 0) { - result.extended_apis = wishlist; - } + const result = { + main_api: mainApi, + }; + // Check if this is COVESA VSS version const versionList = require('../../data/vss.json'); for (const version of versionList) { const file = require(`../../data/${version.name}.json`); @@ -369,6 +385,24 @@ const processApiDataUrl = async (apiDataUrl) => { } } + // If not COVESA VSS version, then add the rest APIs + if (!result.api_version) { + traverse( + data[mainApi], + (api, prefix) => { + for (const [key, value] of Object.entries(api.children || {})) { + extendedApis.push(convertToExtendedApiFormat(value)); + delete api.children[key]; + } + }, + mainApi + ); + } + + if (extendedApis.length > 0) { + result.extended_apis = extendedApis; + } + return result; } catch (error) { logger.warn(`Error in processing api data url: ${error}`); -- GitLab From 416dac1f4659c4579875ba9253a2fb564819c359 Mon Sep 17 00:00:00 2001 From: tuanh <tuan.hoangdinhanh@vn.bosch.com> Date: Tue, 14 Jan 2025 16:39:52 +0700 Subject: [PATCH 2/8] update compute api logic --- src/services/api.service.js | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/services/api.service.js b/src/services/api.service.js index 7c181ca..7e94afe 100644 --- a/src/services/api.service.js +++ b/src/services/api.service.js @@ -140,14 +140,15 @@ const computeVSSApi = async (modelId) => { try { const name = extendedApi.apiName.split('.').slice(1).join('.'); if (!name) return; - ret[mainApi].children[name] = { - description: extendedApi.description, - type: extendedApi.type || 'branch', - id: extendedApi._id, - datatype: extendedApi.datatype, - name: extendedApi.apiName, - isWishlist: extendedApi.isWishlist, - }; + if (!ret[mainApi].children[name]) + ret[mainApi].children[name] = { + description: extendedApi.description, + type: extendedApi.type || 'branch', + id: extendedApi._id, + datatype: extendedApi.datatype, + name: extendedApi.apiName, + isWishlist: extendedApi.isWishlist, + }; if (extendedApi.unit) { ret[mainApi].children[name].unit = extendedApi.unit; } -- GitLab From e88bf33b50bd981aa67595186e4e099cee7f5efc Mon Sep 17 00:00:00 2001 From: tuanh <tuan.hoangdinhanh@vn.bosch.com> Date: Tue, 14 Jan 2025 16:47:00 +0700 Subject: [PATCH 3/8] revert logic --- src/services/api.service.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/services/api.service.js b/src/services/api.service.js index 7e94afe..a71fcbb 100644 --- a/src/services/api.service.js +++ b/src/services/api.service.js @@ -135,20 +135,20 @@ const computeVSSApi = async (modelId) => { const extendedApis = await ExtendedApi.find({ model: modelId, + isWishlist: true, }); extendedApis.forEach((extendedApi) => { try { const name = extendedApi.apiName.split('.').slice(1).join('.'); if (!name) return; - if (!ret[mainApi].children[name]) - ret[mainApi].children[name] = { - description: extendedApi.description, - type: extendedApi.type || 'branch', - id: extendedApi._id, - datatype: extendedApi.datatype, - name: extendedApi.apiName, - isWishlist: extendedApi.isWishlist, - }; + ret[mainApi].children[name] = { + description: extendedApi.description, + type: extendedApi.type || 'branch', + id: extendedApi._id, + datatype: extendedApi.datatype, + name: extendedApi.apiName, + isWishlist: extendedApi.isWishlist, + }; if (extendedApi.unit) { ret[mainApi].children[name].unit = extendedApi.unit; } -- GitLab From 09fb3e6d5d56c6871a9f8045b37f5b2e70bba5d1 Mon Sep 17 00:00:00 2001 From: tuanh <tuan.hoangdinhanh@vn.bosch.com> Date: Wed, 15 Jan 2025 16:38:19 +0700 Subject: [PATCH 4/8] add list all models route --- src/controllers/model.controller.js | 43 +++++++++++++++++++++++++++++ src/routes/v2/model.route.js | 8 ++++++ 2 files changed, 51 insertions(+) diff --git a/src/controllers/model.controller.js b/src/controllers/model.controller.js index 89da0c8..48aba7d 100644 --- a/src/controllers/model.controller.js +++ b/src/controllers/model.controller.js @@ -93,6 +93,48 @@ const listModels = catchAsync(async (req, res) => { res.json(models); }); +const listAllModels = catchAsync(async (req, res) => { + const ownedModels = await modelService.queryModels( + { + created_by: req.user?.id, + }, + { + limit: 1000, + }, + {}, + req.user?.id + ); + + const contributedModels = await modelService.queryModels( + { + is_contributor: req.user?.id, + }, + { + limit: 1000, + }, + {}, + req.user?.id + ); + + const publicReleasedModels = await modelService.queryModels( + { + visibility: 'public', + state: 'released', + }, + { + limit: 1000, + }, + {}, + req.user?.id + ); + + res.status(200).send({ + ownedModels: ownedModels.results, + contributedModels: contributedModels.results, + publicReleasedModels: publicReleasedModels.results, + }); +}); + const getModel = catchAsync(async (req, res) => { const hasWritePermission = await permissionService.hasPermission(req.user?.id, PERMISSIONS.WRITE_MODEL, req.params.id); @@ -175,4 +217,5 @@ module.exports = { addAuthorizedUser, deleteAuthorizedUser, getComputedVSSApi, + listAllModels, }; diff --git a/src/routes/v2/model.route.js b/src/routes/v2/model.route.js index 322016f..4888c65 100644 --- a/src/routes/v2/model.route.js +++ b/src/routes/v2/model.route.js @@ -6,6 +6,7 @@ const auth = require('../../middlewares/auth'); const { checkPermission } = require('../../middlewares/permission'); const { PERMISSIONS } = require('../../config/roles'); const config = require('../../config/config'); +const { model } = require('mongoose'); const router = express.Router(); @@ -20,6 +21,13 @@ router modelController.listModels ); +router.route('/all').get( + auth({ + optional: !config.strictAuth, + }), + modelController.listAllModels +); + router .route('/:id') .get( -- GitLab From 84d5136ee70a64d337b3e3251b34a07b198f22ee Mon Sep 17 00:00:00 2001 From: tuanh <tuan.hoangdinhanh@vn.bosch.com> Date: Thu, 16 Jan 2025 17:02:48 +0700 Subject: [PATCH 5/8] Finish list all models for user route --- src/controllers/model.controller.js | 39 ++++++++++- src/services/api.service.js | 96 +++++++++++++++++++++++--- src/services/model.service.js | 101 ++++++++++++++++++++++++++++ src/services/permission.service.js | 26 ++++--- 4 files changed, 235 insertions(+), 27 deletions(-) diff --git a/src/controllers/model.controller.js b/src/controllers/model.controller.js index 48aba7d..c7b1735 100644 --- a/src/controllers/model.controller.js +++ b/src/controllers/model.controller.js @@ -128,10 +128,43 @@ const listAllModels = catchAsync(async (req, res) => { req.user?.id ); + const cacheResult = new Map(); + + const processStats = async (model) => { + if (!model) { + throw new Error("Error in processStats: model can't be null"); + } + const modelId = model._id || model.id; + if (cacheResult.has(modelId)) { + model.stats = cacheResult.get(modelId); + return; + } + const stats = await modelService.getModelStats(model); + model.stats = stats; + cacheResult.set(modelId, stats); + }; + + // Add stats to each model + for (const model of ownedModels.results) { + await processStats(model); + } + for (const model of contributedModels.results) { + await processStats(model); + } + for (const model of publicReleasedModels.results) { + await processStats(model); + } + res.status(200).send({ - ownedModels: ownedModels.results, - contributedModels: contributedModels.results, - publicReleasedModels: publicReleasedModels.results, + ownedModels: { + results: ownedModels.results, + }, + contributedModels: { + results: contributedModels.results, + }, + publicReleasedModels: { + results: publicReleasedModels.results, + }, }); }); diff --git a/src/services/api.service.js b/src/services/api.service.js index a71fcbb..3ed36a9 100644 --- a/src/services/api.service.js +++ b/src/services/api.service.js @@ -108,6 +108,82 @@ const getVSSVersion = async (name) => { return data; }; +/** + * + * @param {Object} cvi + * @returns {Array} + */ +const parseCvi = (cvi) => { + const traverse = (node, prefix = 'Vehicle') => { + let ret = []; + + ret.push({ ...node, name: prefix }); + + if (node.children) { + for (const [key, child] of Object.entries(node.children)) { + const newPrefix = `${prefix}.${key}`; + node.children[key].name = newPrefix; + ret = ret.concat(traverse(child, newPrefix)); + } + } + + return ret; + }; + + const mainApi = Object.keys(cvi).at(0) || 'Vehicle'; + + const ret = traverse(cvi[mainApi], mainApi); + + ret.forEach((item) => { + if (item.type == 'branch') return; + let arName = item.name.split('.'); + if (arName.length > 1) { + item.shortName = '.' + arName.slice(1).join('.'); + } else { + item.shortName = item.name; // Ensure root elements have their name as shortName + } + }); + + ret.sort((a, b) => { + const aParts = a.name.split('.'); + const bParts = b.name.split('.'); + + for (let i = 0; i < Math.max(aParts.length, bParts.length); i++) { + if (aParts[i] !== bParts[i]) { + return (aParts[i] || '').localeCompare(bParts[i] || ''); + } + } + + return 0; + }); + + return ret; +}; + +/** + * + * @param {string} code + * @param {Array<any>} apiList + * @returns {Array<any>} + */ +const getUsedApis = (code, apiList) => { + try { + let apis = []; + apiList.forEach((item) => { + if (item.shortName) { + if (code.includes(item.shortName)) { + apis.push(item.name); + } + } + }); + + return apis; + } catch (error) { + logger.error(`Error while parsing/counting APIs number: ${error}`); + return []; + } +}; + /** * * @param {string} modelId @@ -179,13 +255,13 @@ const computeVSSApi = async (modelId) => { return ret; }; -module.exports = { - createApi, - getApi, - getApiByModelId, - updateApi, - deleteApi, - listVSSVersions, - getVSSVersion, - computeVSSApi, -}; +module.exports.createApi = createApi; +module.exports.getApi = getApi; +module.exports.getApiByModelId = getApiByModelId; +module.exports.updateApi = updateApi; +module.exports.deleteApi = deleteApi; +module.exports.listVSSVersions = listVSSVersions; +module.exports.getVSSVersion = getVSSVersion; +module.exports.computeVSSApi = computeVSSApi; +module.exports.parseCvi = parseCvi; +module.exports.getUsedApis = getUsedApis; diff --git a/src/services/model.service.js b/src/services/model.service.js index 8dd5769..52bd38c 100644 --- a/src/services/model.service.js +++ b/src/services/model.service.js @@ -1,6 +1,7 @@ const httpStatus = require('http-status'); const { userService } = require('.'); const prototypeService = require('./prototype.service'); +const apiService = require('./api.service'); const permissionService = require('./permission.service'); const { Model, Role } = require('../models'); const ApiError = require('../utils/ApiError'); @@ -34,6 +35,105 @@ const createModel = async (userId, modelBody) => { return model._id; }; +/** + * + * @param {Object}model + */ +const getModelStats = async (model) => { + // Number of used APIS / total apis + const stats = { + apis: {}, + prototypes: {}, + architecture: {}, + collaboration: {}, + }; + + if (!model) return stats; + + let prototypes = null; + const modelId = model._id || model.id; + + // Query prototypes + try { + prototypes = await prototypeService.queryPrototypes({ model_id: modelId }, { limit: 1000 }); + stats.prototypes.count = prototypes.results.length || 0; + } catch (error) { + logger.warn(`Error in querying prototypes ${error}`); + } + + // Query APIs + try { + const cvi = await apiService.computeVSSApi(modelId); + const apiList = apiService.parseCvi(cvi); + stats.apis.total = { count: apiList?.length || 0 }; + + const mergedCode = prototypes.results.map((prototype) => prototype.code).join('\n'); + const usedApis = apiService.getUsedApis(mergedCode, apiList); + stats.apis.used = { + count: usedApis.length, + }; + } catch (error) { + logger.warn(`Error in computing VSS API ${error}`); + } + + // Query architecture of prototypes + try { + const prototypeArchitectureCount = + prototypes?.results?.reduce((acc, prototype) => { + const architecture = JSON.parse(prototype.skeleton || '{}'); + return acc + (architecture?.nodes?.length || 0); + }, 0) || 0; + stats.architecture.prototypes = { + count: prototypeArchitectureCount, + }; + } catch (error) { + logger.warn(`Error in parsing prototype architecture ${error}`); + } + + // Query architecture of model + try { + const architecture = JSON.parse(model.skeleton || '{}'); + stats.architecture.model = { + count: architecture?.nodes?.length || 0, + }; + } catch (error) { + logger.warn(`Error in parsing architecture of ${error}`); + } + + // Calculate total architectures in model + stats.architecture.total = { + count: (stats.architecture.prototypes?.count || 0) + (stats.architecture.model?.count || 0), + }; + + // Query contributors collaboration + try { + const contributors = await permissionService.listAuthorizedUser({ + role: 'model_contributor', + ref: modelId, + }); + stats.collaboration.contributors = { + count: contributors?.length || 0, + }; + } catch (error) { + logger.warn(`Error in querying collaborators ${error}`); + } + + // Query members collaboration + try { + const members = await permissionService.listAuthorizedUser({ + role: 'model_member', + ref: modelId, + }); + stats.collaboration.members = { + count: members?.length || 0, + }; + } catch (error) { + logger.warn(`Error in querying members ${error}`); + } + + return stats; +}; + /** * Query for models with filters * @param {Object} filter @@ -419,3 +519,4 @@ module.exports.addAuthorizedUser = addAuthorizedUser; module.exports.deleteAuthorizedUser = deleteAuthorizedUser; module.exports.getAccessibleModels = getAccessibleModels; module.exports.processApiDataUrl = processApiDataUrl; +module.exports.getModelStats = getModelStats; diff --git a/src/services/permission.service.js b/src/services/permission.service.js index 4992463..c296c4e 100644 --- a/src/services/permission.service.js +++ b/src/services/permission.service.js @@ -273,17 +273,15 @@ const canAccessModel = async (userId, modelId) => { return hasPermission(userId, PERMISSIONS.READ_MODEL, modelId); }; -module.exports = { - listAuthorizedUser, - assignRoleToUser, - getUserRoles, - getRoleUsers, - hasPermission, - removeRoleFromUser, - getMappedRoles, - containsPermission, - getRoles, - getPermissions, - listReadableModelIds, - canAccessModel, -}; +module.exports.listAuthorizedUser = listAuthorizedUser; +module.exports.assignRoleToUser = assignRoleToUser; +module.exports.getUserRoles = getUserRoles; +module.exports.getRoleUsers = getRoleUsers; +module.exports.hasPermission = hasPermission; +module.exports.removeRoleFromUser = removeRoleFromUser; +module.exports.getMappedRoles = getMappedRoles; +module.exports.containsPermission = containsPermission; +module.exports.getRoles = getRoles; +module.exports.getPermissions = getPermissions; +module.exports.listReadableModelIds = listReadableModelIds; +module.exports.canAccessModel = canAccessModel; -- GitLab From a5c1f39b231190688f66b020ba9f77ae75a48645 Mon Sep 17 00:00:00 2001 From: tuanh <tuan.hoangdinhanh@vn.bosch.com> Date: Mon, 20 Jan 2025 12:10:36 +0700 Subject: [PATCH 6/8] update etas genai route --- src/controllers/genai.controller.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/controllers/genai.controller.js b/src/controllers/genai.controller.js index c2c62c7..6085633 100644 --- a/src/controllers/genai.controller.js +++ b/src/controllers/genai.controller.js @@ -279,11 +279,8 @@ const generateAIContent = async (req, res) => { createdAt: new Date(), }); } - const instance = getInstance(environment); - setupClient(token); - const response = await axios.post( `https://${instance}/r2mm/GENERATE_AI`, { prompt }, @@ -295,7 +292,6 @@ const generateAIContent = async (req, res) => { }, } ); - return res.status(200).json(response.data); } catch (error) { console.error('Error generating AI content:', error?.response || error); -- GitLab From 22b5e99dce541cd1d228164fedbc9965c894c3c1 Mon Sep 17 00:00:00 2001 From: tuanh <tuan.hoangdinhanh@vn.bosch.com> Date: Mon, 20 Jan 2025 14:22:41 +0700 Subject: [PATCH 7/8] update logic of etas genai --- src/controllers/genai.controller.js | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/controllers/genai.controller.js b/src/controllers/genai.controller.js index 6085633..6de36a3 100644 --- a/src/controllers/genai.controller.js +++ b/src/controllers/genai.controller.js @@ -268,7 +268,6 @@ const getInstance = (environment = 'prod') => { const generateAIContent = async (req, res) => { try { const { environment } = req.params; - const { prompt } = req.body; const authorizationData = etasAuthorizationData.getAuthorizationData(); let token = authorizationData.accessToken; if (!token || moment().diff(authorizationData.createdAt, 'seconds') >= authorizationData.expiresIn) { @@ -281,17 +280,14 @@ const generateAIContent = async (req, res) => { } const instance = getInstance(environment); setupClient(token); - const response = await axios.post( - `https://${instance}/r2mm/GENERATE_AI`, - { prompt }, - { - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', - Accept: 'application/json, text/plain, */*', - }, - } - ); + const response = await axios.post(`https://${instance}/generation`, req.body, { + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + Accept: 'application/json, text/plain, */*', + maxBodyLength: Infinity, + }, + }); return res.status(200).json(response.data); } catch (error) { console.error('Error generating AI content:', error?.response || error); -- GitLab From 2419560598b4204b74b56cbf6e9c411a9553160c Mon Sep 17 00:00:00 2001 From: tuanh <tuan.hoangdinhanh@vn.bosch.com> Date: Tue, 21 Jan 2025 12:09:00 +0700 Subject: [PATCH 8/8] update logic of etas genai --- src/controllers/genai.controller.js | 33 +++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/controllers/genai.controller.js b/src/controllers/genai.controller.js index 6de36a3..bd63ec2 100644 --- a/src/controllers/genai.controller.js +++ b/src/controllers/genai.controller.js @@ -267,6 +267,14 @@ const getInstance = (environment = 'prod') => { const generateAIContent = async (req, res) => { try { + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }); + res.write(''); + res.flush(); + const { environment } = req.params; const authorizationData = etasAuthorizationData.getAuthorizationData(); let token = authorizationData.accessToken; @@ -279,22 +287,29 @@ const generateAIContent = async (req, res) => { }); } const instance = getInstance(environment); - setupClient(token); const response = await axios.post(`https://${instance}/generation`, req.body, { headers: { Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', - Accept: 'application/json, text/plain, */*', - maxBodyLength: Infinity, }, + responseType: 'stream', + }); + const stream = response.data; + stream.on('data', (data) => { + res.write(data); + res.flush(); + }); + stream.on('end', () => { + res.end(); }); - return res.status(200).json(response.data); } catch (error) { console.error('Error generating AI content:', error?.response || error); - if (axios.isAxiosError(error)) { - return res.status(error.response.status || 502).json(error.response.data); - } - return res.status(500).json({ message: 'Failed to generate AI content' }); + res.write( + `data: ${JSON.stringify({ + code: 500, + message: 'Error generating AI content', + })}\n\n` + ); + res.end(); } }; -- GitLab