feat: Add support for S3 as an asset storage layer (#1703)

* feat: Add support for S3 as an asset storage layer. Fixes #305

* some minor fixes

* use bulk deletion api

* stream the file to s3

1 files changed, 21 insertions, 0 deletions

diff --git a/packages/shared/config.ts b/packages/shared/config.ts
index b899dbeb..715c2848 100644
--- a/packages/shared/config.ts
+++ b/packages/shared/config.ts
@@ -88,6 +88,14 @@ const allEnv = z.object({
 
   // A flag to detect if the user is running in the old separete containers setup
   USING_LEGACY_SEPARATE_CONTAINERS: stringBool("false"),
+
+  // Asset storage configuration
+  ASSET_STORE_S3_ENDPOINT: z.string().optional(),
+  ASSET_STORE_S3_REGION: z.string().optional(),
+  ASSET_STORE_S3_BUCKET: z.string().optional(),
+  ASSET_STORE_S3_ACCESS_KEY_ID: z.string().optional(),
+  ASSET_STORE_S3_SECRET_ACCESS_KEY: z.string().optional(),
+  ASSET_STORE_S3_FORCE_PATH_STYLE: stringBool("false"),
 });
 
 const serverConfigSchema = allEnv.transform((val) => {
@@ -185,6 +193,19 @@ const serverConfigSchema = allEnv.transform((val) => {
       timeoutSec: val.WEBHOOK_TIMEOUT_SEC,
       retryTimes: val.WEBHOOK_RETRY_TIMES,
     },
+    assetStore: {
+      type: val.ASSET_STORE_S3_ENDPOINT
+        ? ("s3" as const)
+        : ("filesystem" as const),
+      s3: {
+        endpoint: val.ASSET_STORE_S3_ENDPOINT,
+        region: val.ASSET_STORE_S3_REGION,
+        bucket: val.ASSET_STORE_S3_BUCKET,
+        accessKeyId: val.ASSET_STORE_S3_ACCESS_KEY_ID,
+        secretAccessKey: val.ASSET_STORE_S3_SECRET_ACCESS_KEY,
+        forcePathStyle: val.ASSET_STORE_S3_FORCE_PATH_STYLE,
+      },
+    },
   };
 });