feat: Add support for S3 as an asset storage layer (#1703)

* feat: Add support for S3 as an asset storage layer. Fixes #305

* some minor fixes

* use bulk deletion api

* stream the file to s3

4 files changed, 1341 insertions, 0 deletions

diff --git a/packages/e2e_tests/tests/assetdb/assetdb-utils.ts b/packages/e2e_tests/tests/assetdb/assetdb-utils.ts
new file mode 100644
index 00000000..a8e29ab4
--- /dev/null
+++ b/packages/e2e_tests/tests/assetdb/assetdb-utils.ts
@@ -0,0 +1,289 @@
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import {
+  CreateBucketCommand,
+  DeleteBucketCommand,
+  DeleteObjectCommand,
+  ListObjectsV2Command,
+  S3Client,
+} from "@aws-sdk/client-s3";
+
+import {
+  ASSET_TYPES,
+  AssetMetadata,
+  AssetStore,
+  LocalFileSystemAssetStore,
+  S3AssetStore,
+} from "@karakeep/shared/assetdb";
+
+export interface TestAssetData {
+  userId: string;
+  assetId: string;
+  content: Buffer;
+  metadata: AssetMetadata;
+}
+
+export function createTestAssetData(
+  overrides: Partial<TestAssetData> = {},
+): TestAssetData {
+  const defaultData: TestAssetData = {
+    userId: `user_${Math.random().toString(36).substring(7)}`,
+    assetId: `asset_${Math.random().toString(36).substring(7)}`,
+    content: Buffer.from(`Test content ${Math.random()}`),
+    metadata: {
+      contentType: ASSET_TYPES.TEXT_HTML,
+      fileName: "test.html",
+    },
+  };
+
+  return { ...defaultData, ...overrides };
+}
+
+export function createTestImageData(): TestAssetData {
+  // Create a minimal PNG image (1x1 pixel)
+  const pngData = Buffer.from([
+    0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
+    0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
+    0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53, 0xde, 0x00, 0x00, 0x00,
+    0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, 0x0f, 0x00, 0x00,
+    0x01, 0x00, 0x01, 0x5c, 0xc2, 0x8a, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x49,
+    0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
+  ]);
+
+  return createTestAssetData({
+    content: pngData,
+    metadata: {
+      contentType: ASSET_TYPES.IMAGE_PNG,
+      fileName: "test.png",
+    },
+  });
+}
+
+export function createTestPdfData(): TestAssetData {
+  // Create a minimal PDF
+  const pdfContent = `%PDF-1.4
+1 0 obj
+<<
+/Type /Catalog
+/Pages 2 0 R
+>>
+endobj
+
+2 0 obj
+<<
+/Type /Pages
+/Kids [3 0 R]
+/Count 1
+>>
+endobj
+
+3 0 obj
+<<
+/Type /Page
+/Parent 2 0 R
+/MediaBox [0 0 612 792]
+>>
+endobj
+
+xref
+0 4
+0000000000 65535 f 
+0000000010 00000 n 
+0000000053 00000 n 
+0000000125 00000 n 
+trailer
+<<
+/Size 4
+/Root 1 0 R
+>>
+startxref
+173
+%%EOF`;
+
+  return createTestAssetData({
+    content: Buffer.from(pdfContent),
+    metadata: {
+      contentType: ASSET_TYPES.APPLICATION_PDF,
+      fileName: "test.pdf",
+    },
+  });
+}
+
+export async function createTempDirectory(): Promise<string> {
+  const tempDir = await fs.promises.mkdtemp(
+    path.join(os.tmpdir(), "assetdb-test-"),
+  );
+  return tempDir;
+}
+
+export async function cleanupTempDirectory(tempDir: string): Promise<void> {
+  try {
+    await fs.promises.rm(tempDir, { recursive: true, force: true });
+  } catch (error) {
+    console.warn(`Failed to cleanup temp directory ${tempDir}:`, error);
+  }
+}
+
+export function createLocalFileSystemStore(
+  rootPath: string,
+): LocalFileSystemAssetStore {
+  return new LocalFileSystemAssetStore(rootPath);
+}
+
+export function createS3Store(bucketName: string): S3AssetStore {
+  const s3Client = new S3Client({
+    region: "us-east-1",
+    endpoint: "http://localhost:9000", // MinIO endpoint for testing
+    credentials: {
+      accessKeyId: "minioadmin",
+      secretAccessKey: "minioadmin",
+    },
+    forcePathStyle: true,
+  });
+
+  return new S3AssetStore(s3Client, bucketName);
+}
+
+export async function createTestBucket(bucketName: string): Promise<S3Client> {
+  const s3Client = new S3Client({
+    region: "us-east-1",
+    endpoint: "http://localhost:9000",
+    credentials: {
+      accessKeyId: "minioadmin",
+      secretAccessKey: "minioadmin",
+    },
+    forcePathStyle: true,
+  });
+
+  try {
+    await s3Client.send(new CreateBucketCommand({ Bucket: bucketName }));
+  } catch (error: unknown) {
+    const err = error as { name?: string };
+    if (
+      err.name !== "BucketAlreadyOwnedByYou" &&
+      err.name !== "BucketAlreadyExists"
+    ) {
+      throw error;
+    }
+  }
+
+  return s3Client;
+}
+
+export async function cleanupTestBucket(
+  s3Client: S3Client,
+  bucketName: string,
+): Promise<void> {
+  try {
+    // List and delete all objects
+    let continuationToken: string | undefined;
+    do {
+      const listResponse = await s3Client.send(
+        new ListObjectsV2Command({
+          Bucket: bucketName,
+          ContinuationToken: continuationToken,
+        }),
+      );
+
+      if (listResponse.Contents && listResponse.Contents.length > 0) {
+        const deletePromises = listResponse.Contents.map(
+          (obj: { Key?: string }) =>
+            s3Client.send(
+              new DeleteObjectCommand({
+                Bucket: bucketName,
+                Key: obj.Key!,
+              }),
+            ),
+        );
+        await Promise.all(deletePromises);
+      }
+
+      continuationToken = listResponse.NextContinuationToken;
+    } while (continuationToken);
+
+    // Delete the bucket
+    await s3Client.send(new DeleteBucketCommand({ Bucket: bucketName }));
+  } catch (error) {
+    console.warn(`Failed to cleanup S3 bucket ${bucketName}:`, error);
+  }
+}
+
+export async function createTempFile(
+  content: Buffer,
+  fileName: string,
+): Promise<string> {
+  const tempDir = await createTempDirectory();
+  const filePath = path.join(tempDir, fileName);
+  await fs.promises.writeFile(filePath, content);
+  return filePath;
+}
+
+export async function streamToBuffer(
+  stream: NodeJS.ReadableStream,
+): Promise<Buffer> {
+  const chunks: Buffer[] = [];
+  const readable = stream as AsyncIterable<Buffer>;
+
+  for await (const chunk of readable) {
+    chunks.push(chunk);
+  }
+
+  return Buffer.concat(chunks);
+}
+
+export function generateLargeBuffer(sizeInMB: number): Buffer {
+  const sizeInBytes = sizeInMB * 1024 * 1024;
+  const buffer = Buffer.alloc(sizeInBytes);
+
+  // Fill with some pattern to make it compressible but not empty
+  for (let i = 0; i < sizeInBytes; i++) {
+    buffer[i] = i % 256;
+  }
+
+  return buffer;
+}
+
+export async function assertAssetExists(
+  store: AssetStore,
+  userId: string,
+  assetId: string,
+): Promise<void> {
+  const { asset, metadata } = await store.readAsset({ userId, assetId });
+  if (!asset || !metadata) {
+    throw new Error(`Asset ${assetId} for user ${userId} does not exist`);
+  }
+}
+
+export async function assertAssetNotExists(
+  store: AssetStore,
+  userId: string,
+  assetId: string,
+): Promise<void> {
+  try {
+    await store.readAsset({ userId, assetId });
+    throw new Error(`Asset ${assetId} for user ${userId} should not exist`);
+  } catch (error: unknown) {
+    // Expected to throw
+    const err = error as { message?: string };
+    if (err.message?.includes("should not exist")) {
+      throw error;
+    }
+  }
+}
+
+export async function getAllAssetsArray(store: AssetStore): Promise<
+  {
+    userId: string;
+    assetId: string;
+    contentType: string;
+    fileName?: string | null;
+    size: number;
+  }[]
+> {
+  const assets = [];
+  for await (const asset of store.getAllAssets()) {
+    assets.push(asset);
+  }
+  return assets;
+}
diff --git a/packages/e2e_tests/tests/assetdb/interface-compliance.test.ts b/packages/e2e_tests/tests/assetdb/interface-compliance.test.ts
new file mode 100644
index 00000000..d5288c7a
--- /dev/null
+++ b/packages/e2e_tests/tests/assetdb/interface-compliance.test.ts
@@ -0,0 +1,627 @@
+import * as fs from "fs";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { ASSET_TYPES, AssetStore } from "@karakeep/shared/assetdb";
+
+import {
+  assertAssetExists,
+  assertAssetNotExists,
+  cleanupTempDirectory,
+  cleanupTestBucket,
+  createLocalFileSystemStore,
+  createS3Store,
+  createTempDirectory,
+  createTempFile,
+  createTestAssetData,
+  createTestBucket,
+  createTestImageData,
+  createTestPdfData,
+  generateLargeBuffer,
+  getAllAssetsArray,
+  streamToBuffer,
+} from "./assetdb-utils";
+
+interface TestContext {
+  store: AssetStore;
+  cleanup: () => Promise<void>;
+}
+
+async function createLocalContext(): Promise<TestContext> {
+  const tempDir = await createTempDirectory();
+  const store = createLocalFileSystemStore(tempDir);
+
+  return {
+    store,
+    cleanup: async () => {
+      await cleanupTempDirectory(tempDir);
+    },
+  };
+}
+
+async function createS3Context(): Promise<TestContext> {
+  const bucketName = `test-bucket-${Math.random().toString(36).substring(7)}`;
+  const s3Client = await createTestBucket(bucketName);
+  const store = createS3Store(bucketName);
+
+  return {
+    store,
+    cleanup: async () => {
+      await cleanupTestBucket(s3Client, bucketName);
+    },
+  };
+}
+
+describe.each([
+  { name: "LocalFileSystemAssetStore", createContext: createLocalContext },
+  { name: "S3AssetStore", createContext: createS3Context },
+])("AssetStore Interface Compliance - $name", ({ createContext }) => {
+  let context: TestContext;
+
+  beforeEach(async () => {
+    context = await createContext();
+  });
+
+  afterEach(async () => {
+    await context.cleanup();
+  });
+
+  describe("Basic CRUD Operations", () => {
+    it("should save and retrieve an asset", async () => {
+      const testData = createTestAssetData();
+
+      await context.store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const { asset, metadata } = await context.store.readAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      expect(asset).toEqual(testData.content);
+      expect(metadata).toEqual(testData.metadata);
+    });
+
+    it("should delete an asset", async () => {
+      const testData = createTestAssetData();
+
+      await context.store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      await assertAssetExists(context.store, testData.userId, testData.assetId);
+
+      await context.store.deleteAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      await assertAssetNotExists(
+        context.store,
+        testData.userId,
+        testData.assetId,
+      );
+    });
+
+    it("should get asset size", async () => {
+      const testData = createTestAssetData();
+
+      await context.store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const size = await context.store.getAssetSize({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      expect(size).toBe(testData.content.length);
+    });
+
+    it("should read asset metadata", async () => {
+      const testData = createTestAssetData();
+
+      await context.store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const metadata = await context.store.readAssetMetadata({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      expect(metadata).toEqual(testData.metadata);
+    });
+  });
+
+  describe("Streaming Operations", () => {
+    it("should create readable stream", async () => {
+      const testData = createTestAssetData();
+
+      await context.store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const stream = await context.store.createAssetReadStream({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      const streamedContent = await streamToBuffer(stream);
+      expect(streamedContent).toEqual(testData.content);
+    });
+
+    it("should support range requests in streams", async () => {
+      const content = Buffer.from("0123456789abcdef");
+      const testData = createTestAssetData({ content });
+
+      await context.store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const stream = await context.store.createAssetReadStream({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        start: 5,
+        end: 10,
+      });
+
+      const streamedContent = await streamToBuffer(stream);
+      expect(streamedContent.toString()).toBe("56789a");
+    });
+  });
+
+  describe("Asset Types Support", () => {
+    it("should support all required asset types", async () => {
+      const testCases = [
+        { contentType: ASSET_TYPES.IMAGE_JPEG, fileName: "test.jpg" },
+        { contentType: ASSET_TYPES.IMAGE_PNG, fileName: "test.png" },
+        { contentType: ASSET_TYPES.IMAGE_WEBP, fileName: "test.webp" },
+        { contentType: ASSET_TYPES.APPLICATION_PDF, fileName: "test.pdf" },
+        { contentType: ASSET_TYPES.TEXT_HTML, fileName: "test.html" },
+        { contentType: ASSET_TYPES.VIDEO_MP4, fileName: "test.mp4" },
+      ];
+
+      for (const { contentType, fileName } of testCases) {
+        const testData = createTestAssetData({
+          metadata: { contentType, fileName },
+        });
+
+        await context.store.saveAsset({
+          userId: testData.userId,
+          assetId: testData.assetId,
+          asset: testData.content,
+          metadata: testData.metadata,
+        });
+
+        const { metadata } = await context.store.readAsset({
+          userId: testData.userId,
+          assetId: testData.assetId,
+        });
+
+        expect(metadata.contentType).toBe(contentType);
+        expect(metadata.fileName).toBe(fileName);
+      }
+    });
+
+    it("should handle large assets", async () => {
+      const largeContent = generateLargeBuffer(5); // 5MB
+      const testData = createTestAssetData({ content: largeContent });
+
+      await context.store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const { asset } = await context.store.readAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      expect(asset.length).toBe(largeContent.length);
+      expect(asset).toEqual(largeContent);
+    });
+
+    it("should reject unsupported asset types", async () => {
+      const testData = createTestAssetData({
+        metadata: {
+          contentType: "unsupported/type",
+          fileName: "test.unsupported",
+        },
+      });
+
+      await expect(
+        context.store.saveAsset({
+          userId: testData.userId,
+          assetId: testData.assetId,
+          asset: testData.content,
+          metadata: testData.metadata,
+        }),
+      ).rejects.toThrow("Unsupported asset type");
+    });
+  });
+
+  describe("Bulk Operations", () => {
+    it("should delete all user assets", async () => {
+      const userId = "bulk-test-user";
+      const testAssets = [
+        createTestAssetData({ userId }),
+        createTestAssetData({ userId }),
+        createTestAssetData({ userId }),
+      ];
+      const otherUserAsset = createTestAssetData(); // Different user
+
+      // Save all assets
+      await Promise.all([
+        ...testAssets.map((testData) =>
+          context.store.saveAsset({
+            userId: testData.userId,
+            assetId: testData.assetId,
+            asset: testData.content,
+            metadata: testData.metadata,
+          }),
+        ),
+        context.store.saveAsset({
+          userId: otherUserAsset.userId,
+          assetId: otherUserAsset.assetId,
+          asset: otherUserAsset.content,
+          metadata: otherUserAsset.metadata,
+        }),
+      ]);
+
+      // Delete user assets
+      await context.store.deleteUserAssets({ userId });
+
+      // Verify user assets are deleted
+      for (const testData of testAssets) {
+        await assertAssetNotExists(
+          context.store,
+          testData.userId,
+          testData.assetId,
+        );
+      }
+
+      // Verify other user's asset still exists
+      await assertAssetExists(
+        context.store,
+        otherUserAsset.userId,
+        otherUserAsset.assetId,
+      );
+    });
+
+    it("should list all assets", async () => {
+      const testAssets = [
+        createTestAssetData(),
+        createTestImageData(),
+        createTestPdfData(),
+      ];
+
+      // Save all assets
+      await Promise.all(
+        testAssets.map((testData) =>
+          context.store.saveAsset({
+            userId: testData.userId,
+            assetId: testData.assetId,
+            asset: testData.content,
+            metadata: testData.metadata,
+          }),
+        ),
+      );
+
+      const assets = await getAllAssetsArray(context.store);
+
+      expect(assets).toHaveLength(3);
+
+      // Verify all assets are present
+      const assetIds = assets.map((a) => a.assetId);
+      for (const testData of testAssets) {
+        expect(assetIds).toContain(testData.assetId);
+      }
+
+      // Verify asset structure
+      for (const asset of assets) {
+        expect(asset).toHaveProperty("userId");
+        expect(asset).toHaveProperty("assetId");
+        expect(asset).toHaveProperty("contentType");
+        expect(asset).toHaveProperty("size");
+        expect(typeof asset.size).toBe("number");
+        expect(asset.size).toBeGreaterThan(0);
+      }
+    });
+  });
+
+  describe("File Operations", () => {
+    it("should save asset from file and delete original file", async () => {
+      const testData = createTestAssetData();
+      const tempFile = await createTempFile(testData.content, "temp-asset.bin");
+
+      // Verify temp file exists before operation
+      expect(
+        await fs.promises
+          .access(tempFile)
+          .then(() => true)
+          .catch(() => false),
+      ).toBe(true);
+
+      await context.store.saveAssetFromFile({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        assetPath: tempFile,
+        metadata: testData.metadata,
+      });
+
+      // Verify temp file was deleted
+      expect(
+        await fs.promises
+          .access(tempFile)
+          .then(() => true)
+          .catch(() => false),
+      ).toBe(false);
+
+      // Verify asset was saved correctly
+      const { asset, metadata } = await context.store.readAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      expect(asset).toEqual(testData.content);
+      expect(metadata).toEqual(testData.metadata);
+    });
+  });
+
+  describe("Error Handling", () => {
+    it("should throw error for non-existent asset read", async () => {
+      await expect(
+        context.store.readAsset({
+          userId: "non-existent-user",
+          assetId: "non-existent-asset",
+        }),
+      ).rejects.toThrow();
+    });
+
+    it("should throw error for non-existent asset metadata", async () => {
+      await expect(
+        context.store.readAssetMetadata({
+          userId: "non-existent-user",
+          assetId: "non-existent-asset",
+        }),
+      ).rejects.toThrow();
+    });
+
+    it("should throw error for non-existent asset size", async () => {
+      await expect(
+        context.store.getAssetSize({
+          userId: "non-existent-user",
+          assetId: "non-existent-asset",
+        }),
+      ).rejects.toThrow();
+    });
+
+    it("should handle deleting non-existent asset gracefully", async () => {
+      // Filesystem implementation throws errors for non-existent files
+      await expect(
+        context.store.deleteAsset({
+          userId: "non-existent-user",
+          assetId: "non-existent-asset",
+        }),
+      ).resolves.not.toThrow();
+    });
+
+    it("should handle deletion of non-existent user directory gracefully", async () => {
+      // Should not throw error when user directory doesn't exist
+      await expect(
+        context.store.deleteUserAssets({ userId: "non-existent-user" }),
+      ).resolves.not.toThrow();
+    });
+
+    it("should handle non-existent asset stream appropriately", async () => {
+      const streamResult = context.store.createAssetReadStream({
+        userId: "non-existent-user",
+        assetId: "non-existent-asset",
+      });
+
+      await expect(streamResult).rejects.toThrow();
+    });
+  });
+
+  describe("Data Integrity", () => {
+    it("should preserve binary data integrity", async () => {
+      const testData = createTestImageData();
+
+      await context.store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const { asset } = await context.store.readAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      // Verify exact binary match
+      expect(asset).toEqual(testData.content);
+
+      // Verify PNG header is intact
+      expect(asset[0]).toBe(0x89);
+      expect(asset[1]).toBe(0x50);
+      expect(asset[2]).toBe(0x4e);
+      expect(asset[3]).toBe(0x47);
+    });
+
+    it("should preserve metadata exactly", async () => {
+      const testData = createTestAssetData({
+        metadata: {
+          contentType: ASSET_TYPES.APPLICATION_PDF,
+          fileName: "test-document.pdf",
+        },
+      });
+
+      await context.store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const { metadata } = await context.store.readAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      expect(metadata).toEqual(testData.metadata);
+      expect(metadata.contentType).toBe(ASSET_TYPES.APPLICATION_PDF);
+      expect(metadata.fileName).toBe("test-document.pdf");
+    });
+
+    it("should handle null fileName correctly", async () => {
+      const testData = createTestAssetData({
+        metadata: {
+          contentType: ASSET_TYPES.TEXT_HTML,
+          fileName: null,
+        },
+      });
+
+      await context.store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const { metadata } = await context.store.readAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      expect(metadata.fileName).toBeNull();
+    });
+  });
+
+  describe("Concurrent Operations", () => {
+    it("should handle concurrent saves safely", async () => {
+      const testAssets = Array.from({ length: 5 }, () => createTestAssetData());
+
+      await Promise.all(
+        testAssets.map((testData) =>
+          context.store.saveAsset({
+            userId: testData.userId,
+            assetId: testData.assetId,
+            asset: testData.content,
+            metadata: testData.metadata,
+          }),
+        ),
+      );
+
+      // Verify all assets were saved correctly
+      for (const testData of testAssets) {
+        const { asset, metadata } = await context.store.readAsset({
+          userId: testData.userId,
+          assetId: testData.assetId,
+        });
+
+        expect(asset).toEqual(testData.content);
+        expect(metadata).toEqual(testData.metadata);
+      }
+    });
+
+    it("should handle concurrent reads safely", async () => {
+      const testData = createTestAssetData();
+
+      await context.store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      // Perform multiple concurrent reads
+      const readPromises = Array.from({ length: 10 }, () =>
+        context.store.readAsset({
+          userId: testData.userId,
+          assetId: testData.assetId,
+        }),
+      );
+
+      const results = await Promise.all(readPromises);
+
+      // Verify all reads returned the same data
+      for (const { asset, metadata } of results) {
+        expect(asset).toEqual(testData.content);
+        expect(metadata).toEqual(testData.metadata);
+      }
+    });
+  });
+
+  describe("Edge Cases", () => {
+    it("should handle empty assets", async () => {
+      const testData = createTestAssetData({
+        content: Buffer.alloc(0),
+      });
+
+      await context.store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const { asset } = await context.store.readAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      expect(asset.length).toBe(0);
+
+      const size = await context.store.getAssetSize({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      expect(size).toBe(0);
+    });
+
+    it("should handle special characters in user and asset IDs", async () => {
+      const testData = createTestAssetData({
+        userId: "user-with-special_chars.123",
+        assetId: "asset_with-special.chars_456",
+      });
+
+      await context.store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const { asset, metadata } = await context.store.readAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      expect(asset).toEqual(testData.content);
+      expect(metadata).toEqual(testData.metadata);
+    });
+  });
+});
diff --git a/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.ts b/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.ts
new file mode 100644
index 00000000..36ff837f
--- /dev/null
+++ b/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.ts
@@ -0,0 +1,228 @@
+import * as fs from "fs";
+import * as path from "path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { LocalFileSystemAssetStore } from "@karakeep/shared/assetdb";
+
+import {
+  assertAssetNotExists,
+  cleanupTempDirectory,
+  createLocalFileSystemStore,
+  createTempDirectory,
+  createTestAssetData,
+} from "./assetdb-utils";
+
+describe("LocalFileSystemAssetStore - Filesystem-Specific Behaviors", () => {
+  let tempDir: string;
+  let store: LocalFileSystemAssetStore;
+
+  beforeEach(async () => {
+    tempDir = await createTempDirectory();
+    store = createLocalFileSystemStore(tempDir);
+  });
+
+  afterEach(async () => {
+    await cleanupTempDirectory(tempDir);
+  });
+
+  describe("File System Structure", () => {
+    it("should create correct directory structure and files", async () => {
+      const testData = createTestAssetData();
+
+      await store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      // Verify directory structure
+      const assetDir = path.join(tempDir, testData.userId, testData.assetId);
+      expect(
+        await fs.promises
+          .access(assetDir)
+          .then(() => true)
+          .catch(() => false),
+      ).toBe(true);
+
+      // Verify asset.bin file
+      const assetFile = path.join(assetDir, "asset.bin");
+      expect(
+        await fs.promises
+          .access(assetFile)
+          .then(() => true)
+          .catch(() => false),
+      ).toBe(true);
+
+      // Verify metadata.json file
+      const metadataFile = path.join(assetDir, "metadata.json");
+      expect(
+        await fs.promises
+          .access(metadataFile)
+          .then(() => true)
+          .catch(() => false),
+      ).toBe(true);
+
+      // Verify file contents
+      const savedContent = await fs.promises.readFile(assetFile);
+      expect(savedContent).toEqual(testData.content);
+
+      const savedMetadata = JSON.parse(
+        await fs.promises.readFile(metadataFile, "utf8"),
+      );
+      expect(savedMetadata).toEqual(testData.metadata);
+    });
+
+    it("should create nested directory structure for user/asset hierarchy", async () => {
+      const userId = "user123";
+      const assetId = "asset456";
+      const testData = createTestAssetData({ userId, assetId });
+
+      await store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      // Verify the exact directory structure
+      const userDir = path.join(tempDir, userId);
+      const assetDir = path.join(userDir, assetId);
+
+      expect(
+        await fs.promises
+          .access(userDir)
+          .then(() => true)
+          .catch(() => false),
+      ).toBe(true);
+
+      expect(
+        await fs.promises
+          .access(assetDir)
+          .then(() => true)
+          .catch(() => false),
+      ).toBe(true);
+
+      // Verify files exist in the correct location
+      expect(
+        await fs.promises
+          .access(path.join(assetDir, "asset.bin"))
+          .then(() => true)
+          .catch(() => false),
+      ).toBe(true);
+
+      expect(
+        await fs.promises
+          .access(path.join(assetDir, "metadata.json"))
+          .then(() => true)
+          .catch(() => false),
+      ).toBe(true);
+    });
+  });
+
+  describe("Directory Cleanup", () => {
+    it("should remove entire asset directory when deleting asset", async () => {
+      const testData = createTestAssetData();
+
+      await store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const assetDir = path.join(tempDir, testData.userId, testData.assetId);
+
+      // Verify directory exists
+      expect(
+        await fs.promises
+          .access(assetDir)
+          .then(() => true)
+          .catch(() => false),
+      ).toBe(true);
+
+      await store.deleteAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      // Verify entire directory was removed
+      expect(
+        await fs.promises
+          .access(assetDir)
+          .then(() => true)
+          .catch(() => false),
+      ).toBe(false);
+
+      await assertAssetNotExists(store, testData.userId, testData.assetId);
+    });
+
+    it("should remove entire user directory when deleting all user assets", async () => {
+      const userId = "test-user";
+      const testData1 = createTestAssetData({ userId });
+      const testData2 = createTestAssetData({ userId });
+
+      await Promise.all([
+        store.saveAsset({
+          userId: testData1.userId,
+          assetId: testData1.assetId,
+          asset: testData1.content,
+          metadata: testData1.metadata,
+        }),
+        store.saveAsset({
+          userId: testData2.userId,
+          assetId: testData2.assetId,
+          asset: testData2.content,
+          metadata: testData2.metadata,
+        }),
+      ]);
+
+      const userDir = path.join(tempDir, userId);
+
+      // Verify user directory exists
+      expect(
+        await fs.promises
+          .access(userDir)
+          .then(() => true)
+          .catch(() => false),
+      ).toBe(true);
+
+      await store.deleteUserAssets({ userId });
+
+      // Verify entire user directory was removed
+      expect(
+        await fs.promises
+          .access(userDir)
+          .then(() => true)
+          .catch(() => false),
+      ).toBe(false);
+    });
+  });
+
+  describe("File System Permissions", () => {
+    it("should create directories with appropriate permissions", async () => {
+      const testData = createTestAssetData();
+
+      await store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const userDir = path.join(tempDir, testData.userId);
+      const assetDir = path.join(userDir, testData.assetId);
+
+      // Verify directories are readable and writable
+      const userStats = await fs.promises.stat(userDir);
+      const assetStats = await fs.promises.stat(assetDir);
+
+      expect(userStats.isDirectory()).toBe(true);
+      expect(assetStats.isDirectory()).toBe(true);
+
+      // Verify we can read and write to the directories
+      await fs.promises.access(userDir, fs.constants.R_OK | fs.constants.W_OK);
+      await fs.promises.access(assetDir, fs.constants.R_OK | fs.constants.W_OK);
+    });
+  });
+});
diff --git a/packages/e2e_tests/tests/assetdb/s3-store.test.ts b/packages/e2e_tests/tests/assetdb/s3-store.test.ts
new file mode 100644
index 00000000..c573750e
--- /dev/null
+++ b/packages/e2e_tests/tests/assetdb/s3-store.test.ts
@@ -0,0 +1,197 @@
+import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { S3AssetStore } from "@karakeep/shared/assetdb";
+
+import {
+  assertAssetExists,
+  cleanupTestBucket,
+  createS3Store,
+  createTestAssetData,
+  createTestBucket,
+} from "./assetdb-utils";
+
+describe("S3AssetStore - S3-Specific Behaviors", () => {
+  let bucketName: string;
+  let s3Client: S3Client;
+  let store: S3AssetStore;
+
+  beforeEach(async () => {
+    bucketName = `test-bucket-${Math.random().toString(36).substring(7)}`;
+    s3Client = await createTestBucket(bucketName);
+    store = createS3Store(bucketName);
+  });
+
+  afterEach(async () => {
+    await cleanupTestBucket(s3Client, bucketName);
+  });
+
+  describe("S3 Object Structure", () => {
+    it("should store asset as single object with user-defined metadata", async () => {
+      const testData = createTestAssetData();
+
+      await store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      // Verify the object exists with the expected key structure
+      const objectKey = `${testData.userId}/${testData.assetId}`;
+      const headResponse = await s3Client.send(
+        new HeadObjectCommand({
+          Bucket: bucketName,
+          Key: objectKey,
+        }),
+      );
+
+      // Verify metadata is stored in S3 user-defined metadata
+      expect(headResponse.Metadata).toBeDefined();
+      expect(headResponse.Metadata!["x-amz-meta-content-type"]).toBe(
+        testData.metadata.contentType,
+      );
+      expect(headResponse.Metadata!["x-amz-meta-file-name"]).toBe(
+        testData.metadata.fileName || "",
+      );
+
+      // Verify content type is set correctly on the S3 object
+      expect(headResponse.ContentType).toBe(testData.metadata.contentType);
+    });
+
+    it("should handle null fileName in metadata correctly", async () => {
+      const testData = createTestAssetData({
+        metadata: {
+          contentType: "text/html",
+          fileName: null,
+        },
+      });
+
+      await store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      const objectKey = `${testData.userId}/${testData.assetId}`;
+      const headResponse = await s3Client.send(
+        new HeadObjectCommand({
+          Bucket: bucketName,
+          Key: objectKey,
+        }),
+      );
+
+      // Verify null fileName are not stored in S3 metadata
+      expect(headResponse.Metadata!["x-amz-meta-file-name"]).toBeUndefined();
+
+      // Verify reading back gives us null fileName
+      const metadata = await store.readAssetMetadata({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+      expect(metadata.fileName).toBeNull();
+    });
+  });
+
+  describe("S3 Key Structure", () => {
+    it("should use clean userId/assetId key structure", async () => {
+      const userId = "user123";
+      const assetId = "asset456";
+      const testData = createTestAssetData({ userId, assetId });
+
+      await store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      // Verify the exact key structure
+      const expectedKey = `${userId}/${assetId}`;
+      const headResponse = await s3Client.send(
+        new HeadObjectCommand({
+          Bucket: bucketName,
+          Key: expectedKey,
+        }),
+      );
+
+      expect(headResponse.ContentLength).toBe(testData.content.length);
+    });
+
+    it("should handle special characters in user and asset IDs", async () => {
+      const userId = "user/with/slashes";
+      const assetId = "asset-with-special-chars_123";
+      const testData = createTestAssetData({ userId, assetId });
+
+      await store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      await assertAssetExists(store, testData.userId, testData.assetId);
+    });
+  });
+
+  describe("S3 Eventual Consistency", () => {
+    it("should handle immediate read after write (MinIO strong consistency)", async () => {
+      const testData = createTestAssetData();
+
+      await store.saveAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+        asset: testData.content,
+        metadata: testData.metadata,
+      });
+
+      // Immediately try to read - should work with MinIO's strong consistency
+      const { asset, metadata } = await store.readAsset({
+        userId: testData.userId,
+        assetId: testData.assetId,
+      });
+
+      expect(asset).toEqual(testData.content);
+      expect(metadata).toEqual(testData.metadata);
+    });
+  });
+
+  describe("S3 Metadata Conversion", () => {
+    it("should correctly convert between AssetMetadata and S3 metadata", async () => {
+      const testCases = [
+        {
+          contentType: "image/jpeg",
+          fileName: "test-image.jpg",
+        },
+        {
+          contentType: "application/pdf",
+          fileName: "document.pdf",
+        },
+        {
+          contentType: "text/html",
+          fileName: null,
+        },
+      ];
+
+      for (const metadata of testCases) {
+        const testData = createTestAssetData({ metadata });
+
+        await store.saveAsset({
+          userId: testData.userId,
+          assetId: testData.assetId,
+          asset: testData.content,
+          metadata: testData.metadata,
+        });
+
+        // Verify metadata round-trip
+        const retrievedMetadata = await store.readAssetMetadata({
+          userId: testData.userId,
+          assetId: testData.assetId,
+        });
+
+        expect(retrievedMetadata).toEqual(testData.metadata);
+      }
+    });
+  });
+});