feat(ssi-abstraction): client part of S3 bucket for tails file storage

Signed-off-by: Berend Sliedrecht <berend@animo.id>

apps/ssi-abstraction/src/agent/agent.service.ts

@@ -51,7 +51,7 @@ import { parseDid } from '../common/utils.js';
 import { LEDGERS } from '../config/ledger.js';
 
 import { AgentLogger } from './logger.js';
-import { TailsFileService } from './revocation/TailsFileService.js';
+import { S3TailsFileService } from './revocation/TailsFileService.js';
 
 export type TenantAgent = Agent<Omit<AgentService['modules'], 'tenants'>>;
 export type AppAgent = Agent<AgentService['modules']>;
@@ -102,7 +102,14 @@ export class AgentService implements OnApplicationShutdown {
     const { autoAcceptConnection, autoAcceptCredential, autoAcceptProof } =
       this.configService.get('agent');
 
-    const tailsServerBaseUrl = this.configService.get('tailsServerBaseUrl');
+    const tailsServerBaseUrl = this.configService.getOrThrow(
+      'tailsServer.baseUrl',
+    );
+    const tailsServerBucketName = this.configService.getOrThrow(
+      'tailsServer.bucketName',
+    );
+    const s3Secret = this.configService.getOrThrow('s3.secret');
+    const s3AccessKey = this.configService.getOrThrow('s3.accessKey');
 
     return {
       connections: new ConnectionsModule({
@@ -138,7 +145,12 @@ export class AgentService implements OnApplicationShutdown {
       anoncreds: new AnonCredsModule({
         anoncreds,
         registries: [new IndyVdrAnonCredsRegistry()],
-        tailsFileService: new TailsFileService({ tailsServerBaseUrl }),
+        tailsFileService: new S3TailsFileService({
+          tailsServerBaseUrl,
+          s3AccessKey,
+          s3Secret,
+          tailsServerBucketName,
+        }),
       }),
       indyVdr: new IndyVdrModule({ indyVdr, networks: this.ledgers }),
 

apps/ssi-abstraction/src/agent/revocation/TailsFileService.ts

@@ -5,15 +5,72 @@ import { BasicTailsFileService } from '@credo-ts/anoncreds';
 import FormData from 'form-data';
 import fs from 'fs';
 
-export class TailsFileService extends BasicTailsFileService {
-  private tailsServerBaseUrl?: string;
+export type UploadToS3Options = {
+  s3Url: string;
+  bucketName: string;
+  fileId: string;
+  content: Uint8Array;
+  accessKey: string;
+  secret: string;
+};
 
-  public constructor(options?: {
+// Upload to S3 and return the URL to fetch it from
+export const uploadToS3 = async ({
+  s3Url,
+  content,
+  fileId,
+  bucketName,
+  accessKey,
+  secret,
+}: UploadToS3Options) => {
+  // TODO: double check all headers
+  const headers = new Headers();
+  headers.set('Host', s3Url);
+  headers.set('Date', generateRfc1123Date());
+  headers.set('Content-Type', 'application/octet-stream');
+  headers.set('Authorization', accessKey);
+  headers.set('Secret', secret);
+
+  const sanitizedUrl = s3Url.endsWith('/')
+    ? s3Url.slice(0, s3Url.length - 1)
+    : s3Url;
+
+  const url = `${sanitizedUrl}/${bucketName}/${fileId}`;
+
+  // TODO: check whether we need to include the sig or not
+  const result = await axios.put(url, content, {
+    headers,
+  });
+
+  if (result.status > 299) {
+    throw new Error(`Error uploading to S3. Error: ${JSON.stringify(result)}`);
+  }
+
+  return url;
+};
+
+export class S3TailsFileService extends BasicTailsFileService {
+  private tailsServerBaseUrl: string;
+  private tailsServerBucketName: string;
+  private s3Secrets: {
+    s3AccessKey: string;
+    s3Secret: string;
+  };
+
+  public constructor(options: {
     tailsDirectoryPath?: string;
-    tailsServerBaseUrl?: string;
+    tailsServerBaseUrl: string;
+    tailsServerBucketName: string;
+    s3AccessKey: string;
+    s3Secret: string;
   }) {
     super(options);
-    this.tailsServerBaseUrl = options?.tailsServerBaseUrl;
+    this.tailsServerBaseUrl = options.tailsServerBaseUrl;
+    this.tailsServerBucketName = options.tailsServerBucketName;
+    this.s3Secrets = {
+      s3AccessKey: options.s3AccessKey,
+      s3Secret: options.s3Secret,
+    };
   }
 
   public async uploadTailsFile(
@@ -22,6 +79,11 @@ export class TailsFileService extends BasicTailsFileService {
       revocationRegistryDefinition: AnonCredsRevocationRegistryDefinition;
     },
   ) {
+    const headers = this.prepareS3Headers(
+      this.tailsServerBaseUrl,
+      this.s3Secrets.s3AccessKey,
+      this.s3Secrets.s3Secret,
+    );
     const revocationRegistryDefinition = options.revocationRegistryDefinition;
     const localTailsFilePath = revocationRegistryDefinition.value.tailsLocation;
     const pathParts = localTailsFilePath.split('/');
@@ -31,11 +93,14 @@ export class TailsFileService extends BasicTailsFileService {
     const readStream = fs.createReadStream(localTailsFilePath);
     data.append('file', readStream);
 
+    const tailsFileUrl = `${this.tailsServerBaseUrl}/${this.tailsServerBucketName}/${tailsFileId}`;
+
     const response = await agentContext.config.agentDependencies.fetch(
-      `${this.tailsServerBaseUrl}/${tailsFileId}`,
+      tailsFileUrl,
       {
         method: 'PUT',
         body: data,
+        headers,
       },
     );
 
@@ -44,7 +109,34 @@ export class TailsFileService extends BasicTailsFileService {
     }
 
     return {
-      tailsFileUrl: `${this.tailsServerBaseUrl}/${encodeURIComponent(tailsFileId)}`,
+      tailsFileUrl,
     };
   }
+
+  private prepareS3Headers(url: string, accessKey: string, secret: string) {
+    const rfc1123Date =
+      new Date()
+        .toLocaleString('en-GB', {
+          timeZone: 'UTC',
+          hour12: false,
+          weekday: 'short',
+          year: 'numeric',
+          month: 'short',
+          day: '2-digit',
+          hour: '2-digit',
+          minute: '2-digit',
+          second: '2-digit',
+        })
+        .replace(/(?:(\d),)/, '$1') + ' GMT';
+
+    // TODO: double check all headers
+    const headers = new Headers();
+    headers.set('Host', url);
+    headers.set('Date', rfc1123Date);
+    headers.set('Content-Type', 'application/octet-stream');
+    headers.set('Authorization', accessKey);
+    headers.set('Secret', secret);
+
+    return headers;
+  }
 }

apps/ssi-abstraction/src/config/__tests__/mockConfig.ts

@@ -8,7 +8,14 @@ import { validationSchema } from '../validation.js';
 const mockConfig = (port: number = 3001, withLedger = false): AppConfig => ({
   agentHost: '',
   port: 3000,
-  tailsServerBaseUrl: 'http://localhost:8080',
+  s3: {
+    secret: 'some-secret',
+    accessKey: 'some-access-key',
+  },
+  tailsServer: {
+    baseUrl: 'http://localhost:8080',
+    bucketName: 'tails',
+  },
   jwtSecret: '',
   nats: {
     url: 'localhost',

apps/ssi-abstraction/src/config/config.ts

@@ -4,7 +4,16 @@ export interface AppConfig {
   agentHost: string;
   port: number;
   jwtSecret: string;
-  tailsServerBaseUrl: string;
+
+  tailsServer: {
+    baseUrl: string;
+    bucketName: string;
+  };
+
+  s3: {
+    secret: string;
+    accessKey: string;
+  };
 
   nats: {
     url: string;
@@ -30,7 +39,6 @@ export const config = (): AppConfig => ({
   agentHost: process.env.AGENT_HOST || '',
   port: parseInt(process.env.PORT || '3000'),
   jwtSecret: process.env.JWT_SECRET || '',
-  tailsServerBaseUrl: process.env.TAILS_SERVER_BASE_URL || '',
 
   nats: {
     url: process.env.NATS_URL || '',
@@ -38,6 +46,16 @@ export const config = (): AppConfig => ({
     password: process.env.NATS_PASSWORD || '',
   },
 
+  s3: {
+    secret: process.env.S3_SECRET || '',
+    accessKey: process.env.S3_ACCESS_KEY || '',
+  },
+
+  tailsServer: {
+    baseUrl: process.env.TAILS_SERVER_BASE_URL || '',
+    bucketName: process.env.TAILS_SERVER_BUCKET_NAME || '',
+  },
+
   agent: {
     name: process.env.AGENT_NAME || '',
     walletId: process.env.AGENT_WALLET_ID || '',

apps/ssi-abstraction/src/config/validation.ts

@@ -4,8 +4,14 @@ export const validationSchema = Joi.object({
   NATS_URL: Joi.string().required(),
   NATS_USER: Joi.string().optional(),
   NATS_PASSWORD: Joi.string().optional(),
+
   PORT: Joi.number().required(),
+
   TAILS_SERVER_BASE_URL: Joi.string().required(),
+  TAILS_SERVER_BUCKET_NAME: Joi.string().required(),
+
+  S3_SECRET: Joi.string().required(),
+  S3_ACCESS_KEY: Joi.string().required(),
 
   AGENT_NAME: Joi.string().required(),
   AGENT_WALLET_ID: Joi.string().required(),