chore: add OpenTelemetry endpoint to all services

apps/connection-manager/src/application.ts

@@ -5,7 +5,7 @@ import { Inject, Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { RouterModule } from '@nestjs/core';
 import { ClientProxy, ClientsModule, Transport } from '@nestjs/microservices';
-import { HealthModule } from '@ocm/shared';
+import { HealthModule, OpenTelemetryModule } from '@ocm/shared';
 
 import { NATS_CLIENT } from './common/constants.js';
 import { httpConfig } from './config/http.config.js';
@@ -61,6 +61,8 @@ import { InvitationsModule } from './invitations/invitations.module.js';
       },
     }),
 
+    OpenTelemetryModule,
+
     ConnectionsModule,
     InvitationsModule,
 

apps/connection-manager/src/main.ts

 /* c8 ignore start */
 import type { ConfigType } from '@nestjs/config';
+import type { IncomingMessage, ServerResponse } from 'http';
 
 import { Logger, VersioningType } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
+import { initOpenTelemetry } from '@ocm/shared';
 import helmet from 'helmet';
 import { createRequire } from 'module';
 import { resolve } from 'node:path';
@@ -14,8 +16,15 @@ import { httpConfig } from './config/http.config.js';
 const pkgPath = resolve('package.json');
 const pkg = createRequire(import.meta.url)(pkgPath);
 
+const { sdk, prometheusExporter } = initOpenTelemetry(pkg.name);
+sdk.start();
+
 const app = await NestFactory.create(Application);
 
+app.use('/metrics', (req: IncomingMessage, res: ServerResponse) => {
+  prometheusExporter.getMetricsRequestHandler(req, res);
+});
+
 app.use(helmet());
 
 app.enableVersioning({

apps/credential-manager/src/application.ts

@@ -5,7 +5,7 @@ import { Inject, Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { RouterModule } from '@nestjs/core';
 import { ClientProxy, ClientsModule, Transport } from '@nestjs/microservices';
-import { HealthModule } from '@ocm/shared';
+import { HealthModule, OpenTelemetryModule } from '@ocm/shared';
 
 import { NATS_CLIENT } from './common/constants.js';
 import { httpConfig } from './config/http.config.js';
@@ -64,6 +64,8 @@ import { PoliciesModule } from './policies/policies.module.js';
       },
     }),
 
+    OpenTelemetryModule,
+
     CredentialsModule,
     CredentialOffersModule,
     CredentialRequestsModule,

apps/credential-manager/src/main.ts

 /* c8 ignore start */
 import type { ConfigType } from '@nestjs/config';
+import type { IncomingMessage, ServerResponse } from 'http';
 
 import { Logger, VersioningType } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
+import { initOpenTelemetry } from '@ocm/shared';
 import helmet from 'helmet';
 import { createRequire } from 'module';
 import { resolve } from 'node:path';
@@ -14,8 +16,15 @@ import { httpConfig } from './config/http.config.js';
 const pkgPath = resolve('package.json');
 const pkg = createRequire(import.meta.url)(pkgPath);
 
+const { sdk, prometheusExporter } = initOpenTelemetry(pkg.name);
+sdk.start();
+
 const app = await NestFactory.create(Application);
 
+app.use('/metrics', (req: IncomingMessage, res: ServerResponse) => {
+  prometheusExporter.getMetricsRequestHandler(req, res);
+});
+
 app.use(helmet());
 
 app.enableVersioning({

apps/did-manager/src/application.ts

@@ -5,7 +5,7 @@ import { Inject, Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { RouterModule } from '@nestjs/core';
 import { ClientProxy, ClientsModule, Transport } from '@nestjs/microservices';
-import { HealthModule } from '@ocm/shared';
+import { HealthModule, OpenTelemetryModule } from '@ocm/shared';
 
 import { NATS_CLIENT } from './common/constants.js';
 import { httpConfig } from './config/http.config.js';
@@ -60,6 +60,8 @@ import { DIDsModule } from './dids/dids.module.js';
       },
     }),
 
+    OpenTelemetryModule,
+
     DIDsModule,
 
     RouterModule.register([

apps/did-manager/src/main.ts

 /* c8 ignore start */
 import type { ConfigType } from '@nestjs/config';
+import type { IncomingMessage, ServerResponse } from 'http';
 
 import { Logger, VersioningType } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
+import { initOpenTelemetry } from '@ocm/shared';
 import helmet from 'helmet';
 import { createRequire } from 'module';
 import { resolve } from 'node:path';
@@ -14,8 +16,15 @@ import { httpConfig } from './config/http.config.js';
 const pkgPath = resolve('package.json');
 const pkg = createRequire(import.meta.url)(pkgPath);
 
+const { sdk, prometheusExporter } = initOpenTelemetry(pkg.name);
+sdk.start();
+
 const app = await NestFactory.create(Application);
 
+app.use('/metrics', (req: IncomingMessage, res: ServerResponse) => {
+  prometheusExporter.getMetricsRequestHandler(req, res);
+});
+
 app.use(helmet());
 
 app.enableVersioning({

apps/proof-manager/src/application.ts

@@ -5,7 +5,7 @@ import { Inject, Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { RouterModule } from '@nestjs/core';
 import { ClientProxy, ClientsModule, Transport } from '@nestjs/microservices';
-import { HealthModule } from '@ocm/shared';
+import { HealthModule, OpenTelemetryModule } from '@ocm/shared';
 
 import { NATS_CLIENT } from './common/constants.js';
 import { httpConfig } from './config/http.config.js';
@@ -60,6 +60,8 @@ import { ProofsModule } from './proofs/proofs.module.js';
       },
     }),
 
+    OpenTelemetryModule,
+
     ProofsModule,
 
     RouterModule.register([

apps/proof-manager/src/main.ts

 /* c8 ignore start */
 import type { ConfigType } from '@nestjs/config';
+import type { IncomingMessage, ServerResponse } from 'http';
 
 import { Logger, VersioningType } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
+import { initOpenTelemetry } from '@ocm/shared';
 import helmet from 'helmet';
 import { createRequire } from 'module';
 import { resolve } from 'node:path';
@@ -14,8 +16,15 @@ import { httpConfig } from './config/http.config.js';
 const pkgPath = resolve('package.json');
 const pkg = createRequire(import.meta.url)(pkgPath);
 
+const { sdk, prometheusExporter } = initOpenTelemetry(pkg.name);
+sdk.start();
+
 const app = await NestFactory.create(Application);
 
+app.use('/metrics', (req: IncomingMessage, res: ServerResponse) => {
+  prometheusExporter.getMetricsRequestHandler(req, res);
+});
+
 app.use(helmet());
 
 app.enableVersioning({

apps/schema-manager/src/application.ts

@@ -5,7 +5,7 @@ import { Inject, Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { RouterModule } from '@nestjs/core';
 import { ClientProxy, ClientsModule, Transport } from '@nestjs/microservices';
-import { HealthModule } from '@ocm/shared';
+import { HealthModule, OpenTelemetryModule } from '@ocm/shared';
 
 import { NATS_CLIENT } from './common/constants.js';
 import { httpConfig } from './config/http.config.js';
@@ -61,6 +61,8 @@ import { SchemasModule } from './schemas/schemas.module.js';
       },
     }),
 
+    OpenTelemetryModule,
+
     SchemasModule,
     CredentialDefinitionsModule,
 

apps/schema-manager/src/main.ts

 /* c8 ignore start */
 import type { ConfigType } from '@nestjs/config';
+import type { IncomingMessage, ServerResponse } from 'http';
 
 import { Logger, VersioningType } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
+import { initOpenTelemetry } from '@ocm/shared';
 import helmet from 'helmet';
 import { createRequire } from 'module';
 import { resolve } from 'node:path';
@@ -14,8 +16,15 @@ import { httpConfig } from './config/http.config.js';
 const pkgPath = resolve('package.json');
 const pkg = createRequire(import.meta.url)(pkgPath);
 
+const { sdk, prometheusExporter } = initOpenTelemetry(pkg.name);
+sdk.start();
+
 const app = await NestFactory.create(Application);
 
+app.use('/metrics', (req: IncomingMessage, res: ServerResponse) => {
+  prometheusExporter.getMetricsRequestHandler(req, res);
+});
+
 app.use(helmet());
 
 app.enableVersioning({

apps/shared/package.json

@@ -22,11 +22,22 @@
     "@nestjs/axios": "3.0.2",
     "@nestjs/swagger": "7.3.0",
     "@nestjs/terminus": "10.2.3",
+    "@opentelemetry/api": "^1.8.0",
+    "@opentelemetry/auto-instrumentations-node": "0.43.0",
+    "@opentelemetry/context-async-hooks": "1.22.0",
+    "@opentelemetry/core": "1.22.0",
+    "@opentelemetry/exporter-prometheus": "0.49.1",
+    "@opentelemetry/exporter-trace-otlp-http": "0.49.1",
+    "@opentelemetry/propagator-b3": "1.22.0",
+    "@opentelemetry/propagator-jaeger": "1.22.0",
+    "@opentelemetry/sdk-node": "0.49.1",
+    "@opentelemetry/sdk-trace-base": "1.22.0",
     "axios": "1.6.7",
-    "joi": "17.12.1",
     "class-transformer": "0.5.1",
     "class-validator": "0.14.1",
+    "joi": "17.12.1",
     "nats": "2.19.0",
+    "nestjs-otel": "5.1.5",
     "rxjs": "7.8.1",
     "winston": "3.11.0"
   },

apps/shared/src/index.ts

@@ -29,3 +29,5 @@ export * from './rxjs/extract-response-data.js';
 export * from './rxjs/handle-empty-response.js';
 export * from './rxjs/handle-request-timeout.js';
 export * from './rxjs/handle-ssi-response.js';
+
+export * from './opentelemetry/index.js';

apps/shared/src/opentelemetry/index.ts

+import { Logger, Module } from '@nestjs/common';
+import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
+import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
+import {
+  CompositePropagator,
+  W3CBaggagePropagator,
+  W3CTraceContextPropagator,
+} from '@opentelemetry/core';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { B3InjectEncoding, B3Propagator } from '@opentelemetry/propagator-b3';
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { NoopSpanProcessor } from '@opentelemetry/sdk-trace-base';
+import { OpenTelemetryModule as NestJSOpenTelemetryModule } from 'nestjs-otel';
+import process from 'process';
+
+@Module({
+  imports: [
+    NestJSOpenTelemetryModule.forRoot({
+      metrics: {
+        hostMetrics: true,
+        apiMetrics: {
+          enable: true,
+          ignoreRoutes: ['/favicon.ico'],
+          ignoreUndefinedRoutes: false,
+        },
+      },
+    }),
+  ],
+})
+export class OpenTelemetryModule {}
+
+export const initOpenTelemetry = (serviceName: string) => {
+  const prometheusExporter = new PrometheusExporter({
+    preventServerStart: true,
+  });
+
+  const otelSDK = new NodeSDK({
+    serviceName,
+    metricReader: prometheusExporter,
+    spanProcessors: [new NoopSpanProcessor()],
+    contextManager: new AsyncLocalStorageContextManager(),
+    instrumentations: [
+      getNodeAutoInstrumentations({
+        '@opentelemetry/instrumentation-fs': { enabled: false },
+      }),
+    ],
+    textMapPropagator: new CompositePropagator({
+      propagators: [
+        new W3CTraceContextPropagator(),
+        new W3CBaggagePropagator(),
+        new B3Propagator(),
+        new B3Propagator({
+          injectEncoding: B3InjectEncoding.MULTI_HEADER,
+        }),
+      ],
+    }),
+  });
+
+  process.on('SIGTERM', async () => {
+    try {
+      await otelSDK.shutdown();
+      Logger.log('OpenTelemetry SDK shut down successfully');
+    } catch (error) {
+      Logger.log('Error shutting down OpenTelemetry SDK', error);
+    } finally {
+      process.exit(0);
+    }
+  });
+
+  return { sdk: otelSDK, prometheusExporter };
+};

apps/shared/src/tracing/otel.ts

+import { Logger } from '@nestjs/common';
+import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
+import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
+import {
+  CompositePropagator,
+  W3CBaggagePropagator,
+  W3CTraceContextPropagator,
+} from '@opentelemetry/core';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { B3InjectEncoding, B3Propagator } from '@opentelemetry/propagator-b3';
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { NoopSpanProcessor } from '@opentelemetry/sdk-trace-base';
+import process from 'process';
+
+export const initOpenTelemetry = (serviceName: string) => {
+  const prometheusExporter = new PrometheusExporter({
+    preventServerStart: true,
+  });
+
+  const otelSDK = new NodeSDK({
+    serviceName,
+    metricReader: prometheusExporter,
+    spanProcessors: [new NoopSpanProcessor()],
+    contextManager: new AsyncLocalStorageContextManager(),
+    instrumentations: [
+      getNodeAutoInstrumentations({
+        '@opentelemetry/instrumentation-fs': { enabled: false },
+      }),
+    ],
+    textMapPropagator: new CompositePropagator({
+      propagators: [
+        new W3CTraceContextPropagator(),
+        new W3CBaggagePropagator(),
+        new B3Propagator(),
+        new B3Propagator({
+          injectEncoding: B3InjectEncoding.MULTI_HEADER,
+        }),
+      ],
+    }),
+  });
+
+  process.on('SIGTERM', async () => {
+    try {
+      await otelSDK.shutdown();
+      Logger.log('OpenTelemetry SDK shut down successfully');
+    } catch (error) {
+      Logger.log('Error shutting down OpenTelemetry SDK', error);
+    } finally {
+      process.exit(0);
+    }
+  });
+
+  return { sdk: otelSDK, prometheusExporter };
+};

apps/ssi-abstraction/src/application.ts

@@ -5,7 +5,11 @@ import { Inject, Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { RouterModule } from '@nestjs/core';
 import { ClientProxy, ClientsModule, Transport } from '@nestjs/microservices';
-import { EventDidsRegisterEndorserDid, HealthModule } from '@ocm/shared';
+import {
+  EventDidsRegisterEndorserDid,
+  HealthModule,
+  OpenTelemetryModule,
+} from '@ocm/shared';
 import { firstValueFrom } from 'rxjs';
 
 import { AgentModule } from './agent/agent.module.js';
@@ -71,6 +75,8 @@ import { validationSchema } from './config/validation.js';
       },
     }),
 
+    OpenTelemetryModule,
+
     AgentModule,
     ConnectionsModule,
     SchemasModule,

apps/ssi-abstraction/src/main.ts

 /* c8 ignore start */
 import type { ConfigType } from '@nestjs/config';
+import type { IncomingMessage, ServerResponse } from 'node:http';
 
 import { Logger } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
 import { Transport } from '@nestjs/microservices';
+import { initOpenTelemetry } from '@ocm/shared';
+import { createRequire } from 'module';
+import { resolve } from 'node:path';
 
 import { Application } from './application.js';
 import { httpConfig } from './config/http.config.js';
 import { natsConfig } from './config/nats.config.js';
 
+const pkgPath = resolve('package.json');
+const pkg = createRequire(import.meta.url)(pkgPath);
+
+const { sdk, prometheusExporter } = initOpenTelemetry(pkg.name);
+sdk.start();
+
 const app = await NestFactory.create(Application);
 
+app.use('/metrics', (req: IncomingMessage, res: ServerResponse) => {
+  prometheusExporter.getMetricsRequestHandler(req, res);
+});
+
 const { url, user, password } = app.get(natsConfig.KEY) as ConfigType<
   typeof natsConfig
 >;

apps/tenant-manager/src/application.ts

@@ -5,7 +5,7 @@ import { Inject, Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { RouterModule } from '@nestjs/core';
 import { ClientProxy, ClientsModule, Transport } from '@nestjs/microservices';
-import { HealthModule } from '@ocm/shared';
+import { HealthModule, OpenTelemetryModule } from '@ocm/shared';
 
 import { NATS_CLIENT } from './common/constants.js';
 import { httpConfig } from './config/http.config.js';
@@ -60,6 +60,8 @@ import { TenantsModule } from './tenants/tenants.module.js';
       },
     }),
 
+    OpenTelemetryModule,
+
     TenantsModule,
 
     RouterModule.register([

apps/tenant-manager/src/main.ts

 /* c8 ignore start */
 import type { ConfigType } from '@nestjs/config';
+import type { IncomingMessage, ServerResponse } from 'http';
 
 import { Logger, VersioningType } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
+import { initOpenTelemetry } from '@ocm/shared';
 import helmet from 'helmet';
 import { createRequire } from 'module';
 import { resolve } from 'node:path';
@@ -14,8 +16,15 @@ import { httpConfig } from './config/http.config.js';
 const pkgPath = resolve('package.json');
 const pkg = createRequire(import.meta.url)(pkgPath);
 
+const { sdk, prometheusExporter } = initOpenTelemetry(pkg.name);
+sdk.start();
+
 const app = await NestFactory.create(Application);
 
+app.use('/metrics', (req: IncomingMessage, res: ServerResponse) => {
+  prometheusExporter.getMetricsRequestHandler(req, res);
+});
+
 app.use(helmet());
 
 app.enableVersioning({