From 136f126296af65f50da598d084d1485c0e40437a Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sun, 27 Apr 2025 00:02:20 +0100 Subject: feat: Implement generic rule engine (#1318) * Add schema for the new rule engine * Add rule engine backend logic * Implement the worker logic and event firing * Implement the UI changesfor the rule engine * Ensure that when a referenced list or tag are deleted, the corresponding event/action is * Dont show smart lists in rule engine events * Add privacy validations for attached tag and list ids * Move the rules logic into a models --- apps/web/app/layout.tsx | 6 +- apps/web/app/settings/layout.tsx | 6 + apps/web/app/settings/rules/page.tsx | 89 + .../components/dashboard/feeds/FeedSelector.tsx | 53 + .../dashboard/lists/BookmarkListSelector.tsx | 11 +- .../dashboard/rules/RuleEngineActionBuilder.tsx | 216 +++ .../dashboard/rules/RuleEngineConditionBuilder.tsx | 322 ++++ .../dashboard/rules/RuleEngineEventSelector.tsx | 107 ++ .../dashboard/rules/RuleEngineRuleEditor.tsx | 203 ++ .../dashboard/rules/RuleEngineRuleList.tsx | 166 ++ .../components/dashboard/tags/TagAutocomplete.tsx | 126 ++ apps/web/components/dashboard/tags/TagSelector.tsx | 5 +- apps/web/components/settings/AddApiKey.tsx | 6 +- apps/web/components/settings/FeedSettings.tsx | 3 +- apps/web/components/settings/WebhookSettings.tsx | 12 +- .../components/shared/sidebar/MobileSidebar.tsx | 2 +- apps/web/lib/i18n/locales/en/translation.json | 48 + apps/workers/index.ts | 7 +- apps/workers/openaiWorker.ts | 22 +- apps/workers/ruleEngineWorker.ts | 86 + apps/workers/trpc.ts | 21 +- apps/workers/webhookWorker.ts | 13 +- packages/db/drizzle/0045_add_rule_engine.sql | 33 + packages/db/drizzle/meta/0045_snapshot.json | 1951 ++++++++++++++++++++ packages/db/drizzle/meta/_journal.json | 9 +- packages/db/schema.ts | 118 +- packages/shared-react/hooks/rules.ts | 40 + packages/shared/queues.ts | 28 + packages/shared/types/rules.ts | 333 ++++ packages/shared/types/search.ts | 6 +- packages/trpc/lib/__tests__/ruleEngine.test.ts | 664 +++++++ packages/trpc/lib/ruleEngine.ts | 231 +++ packages/trpc/models/lists.ts | 19 +- packages/trpc/models/rules.ts | 233 +++ packages/trpc/package.json | 2 + packages/trpc/routers/_app.ts | 2 + packages/trpc/routers/bookmarks.ts | 27 + packages/trpc/routers/lists.ts | 3 +- packages/trpc/routers/rules.test.ts | 379 ++++ packages/trpc/routers/rules.ts | 120 ++ packages/trpc/routers/tags.ts | 2 +- pnpm-lock.yaml | 97 +- 42 files changed, 5787 insertions(+), 40 deletions(-) create mode 100644 apps/web/app/settings/rules/page.tsx create mode 100644 apps/web/components/dashboard/feeds/FeedSelector.tsx create mode 100644 apps/web/components/dashboard/rules/RuleEngineActionBuilder.tsx create mode 100644 apps/web/components/dashboard/rules/RuleEngineConditionBuilder.tsx create mode 100644 apps/web/components/dashboard/rules/RuleEngineEventSelector.tsx create mode 100644 apps/web/components/dashboard/rules/RuleEngineRuleEditor.tsx create mode 100644 apps/web/components/dashboard/rules/RuleEngineRuleList.tsx create mode 100644 apps/web/components/dashboard/tags/TagAutocomplete.tsx create mode 100644 apps/workers/ruleEngineWorker.ts create mode 100644 packages/db/drizzle/0045_add_rule_engine.sql create mode 100644 packages/db/drizzle/meta/0045_snapshot.json create mode 100644 packages/shared-react/hooks/rules.ts create mode 100644 packages/shared/types/rules.ts create mode 100644 packages/trpc/lib/__tests__/ruleEngine.test.ts create mode 100644 packages/trpc/lib/ruleEngine.ts create mode 100644 packages/trpc/models/rules.ts create mode 100644 packages/trpc/routers/rules.test.ts create mode 100644 packages/trpc/routers/rules.ts diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsx index 6b75edf3..beeecc2b 100644 --- a/apps/web/app/layout.tsx +++ b/apps/web/app/layout.tsx @@ -49,7 +49,11 @@ export default async function RootLayout({ const userSettings = await getUserLocalSettings(); const isRTL = userSettings.lang === "ar"; return ( - + , path: "/settings/webhooks", }, + { + name: t("settings.rules.rules"), + icon: , + path: "/settings/rules", + }, { name: t("settings.manage_assets.manage_assets"), icon: , diff --git a/apps/web/app/settings/rules/page.tsx b/apps/web/app/settings/rules/page.tsx new file mode 100644 index 00000000..98a30bcc --- /dev/null +++ b/apps/web/app/settings/rules/page.tsx @@ -0,0 +1,89 @@ +"use client"; + +import { useState } from "react"; +import { RuleEditor } from "@/components/dashboard/rules/RuleEngineRuleEditor"; +import RuleList from "@/components/dashboard/rules/RuleEngineRuleList"; +import { Button } from "@/components/ui/button"; +import { FullPageSpinner } from "@/components/ui/full-page-spinner"; +import { useTranslation } from "@/lib/i18n/client"; +import { api } from "@/lib/trpc"; +import { Tooltip, TooltipContent, TooltipTrigger } from "components/ui/tooltip"; +import { FlaskConical, PlusCircle } from "lucide-react"; + +import { RuleEngineRule } from "@karakeep/shared/types/rules"; + +export default function RulesSettingsPage() { + const { t } = useTranslation(); + const [editingRule, setEditingRule] = useState< + (Omit & { id: string | null }) | null + >(null); + + const { data: rules, isLoading } = api.rules.list.useQuery(undefined, { + refetchOnWindowFocus: true, + refetchOnMount: true, + }); + + const handleCreateRule = () => { + const newRule = { + id: null, + name: "New Rule", + description: "Description of the new rule", + enabled: true, + event: { type: "bookmarkAdded" as const }, + condition: { type: "alwaysTrue" as const }, + actions: [{ type: "addTag" as const, tagId: "" }], + }; + setEditingRule(newRule); + }; + + const handleDeleteRule = (ruleId: string) => { + if (editingRule?.id === ruleId) { + // If the rule being edited is being deleted, reset the editing rule + setEditingRule(null); + } + }; + + return ( +
+
+
+ + {t("settings.rules.rules")} + + + + + + {t("common.experimental")} + + + + +
+

+ {t("settings.rules.description")} +

+ {!rules || isLoading ? ( + + ) : ( + setEditingRule(r)} + onDeleteRule={handleDeleteRule} + /> + )} +
+ {editingRule && ( + setEditingRule(null)} + /> + )} +
+
+
+ ); +} diff --git a/apps/web/components/dashboard/feeds/FeedSelector.tsx b/apps/web/components/dashboard/feeds/FeedSelector.tsx new file mode 100644 index 00000000..db95a042 --- /dev/null +++ b/apps/web/components/dashboard/feeds/FeedSelector.tsx @@ -0,0 +1,53 @@ +import { + Select, + SelectContent, + SelectGroup, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import LoadingSpinner from "@/components/ui/spinner"; +import { api } from "@/lib/trpc"; +import { cn } from "@/lib/utils"; + +export function FeedSelector({ + value, + onChange, + placeholder = "Select a feed", + className, +}: { + className?: string; + value?: string | null; + onChange: (value: string) => void; + placeholder?: string; +}) { + const { data, isPending } = api.feeds.list.useQuery(undefined, { + select: (data) => data.feeds, + }); + + if (isPending) { + return ; + } + + return ( + + ); +} diff --git a/apps/web/components/dashboard/lists/BookmarkListSelector.tsx b/apps/web/components/dashboard/lists/BookmarkListSelector.tsx index db37efc0..9ce56031 100644 --- a/apps/web/components/dashboard/lists/BookmarkListSelector.tsx +++ b/apps/web/components/dashboard/lists/BookmarkListSelector.tsx @@ -7,8 +7,10 @@ import { SelectValue, } from "@/components/ui/select"; import LoadingSpinner from "@/components/ui/spinner"; +import { cn } from "@/lib/utils"; import { useBookmarkLists } from "@karakeep/shared-react/hooks/lists"; +import { ZBookmarkList } from "@karakeep/shared/types/lists"; export function BookmarkListSelector({ value, @@ -16,12 +18,16 @@ export function BookmarkListSelector({ hideSubtreeOf, hideBookmarkIds = [], placeholder = "Select a list", + className, + listTypes = ["manual", "smart"], }: { + className?: string; value?: string | null; onChange: (value: string) => void; placeholder?: string; hideSubtreeOf?: string; hideBookmarkIds?: string[]; + listTypes?: ZBookmarkList["type"][]; }) { const { data, isPending: isFetchingListsPending } = useBookmarkLists(); let { allPaths } = data ?? {}; @@ -34,6 +40,9 @@ export function BookmarkListSelector({ if (hideBookmarkIds.includes(path[path.length - 1].id)) { return false; } + if (!listTypes.includes(path[path.length - 1].type)) { + return false; + } if (!hideSubtreeOf) { return true; } @@ -42,7 +51,7 @@ export function BookmarkListSelector({ return ( + handleActionTypeChange( + index, + value as RuleEngineAction["type"], + ) + } + > + + + + + + {t("settings.rules.actions_types.add_tag")} + + + {t("settings.rules.actions_types.remove_tag")} + + + {t("settings.rules.actions_types.add_to_list")} + + + {t("settings.rules.actions_types.remove_from_list")} + + + {t( + "settings.rules.actions_types.download_full_page_archive", + )} + + + {t("settings.rules.actions_types.favourite_bookmark")} + + + {t("settings.rules.actions_types.archive_bookmark")} + + + + + {(action.type === "addTag" || + action.type === "removeTag") && ( + + handleActionFieldChange(index, { + type: action.type, + tagId: t, + }) + } + /> + )} + + {(action.type === "addToList" || + action.type === "removeFromList") && ( + + handleActionFieldChange(index, { + type: action.type, + listId: e, + }) + } + /> + )} + + + + + + + )) + )} + + + + ); +} diff --git a/apps/web/components/dashboard/rules/RuleEngineConditionBuilder.tsx b/apps/web/components/dashboard/rules/RuleEngineConditionBuilder.tsx new file mode 100644 index 00000000..8faca013 --- /dev/null +++ b/apps/web/components/dashboard/rules/RuleEngineConditionBuilder.tsx @@ -0,0 +1,322 @@ +import { useState } from "react"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from "@/components/ui/collapsible"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + Archive, + ChevronDown, + ChevronRight, + FileType, + Link, + PlusCircle, + Rss, + Star, + Tag, + Trash2, +} from "lucide-react"; +import { useTranslation } from "react-i18next"; + +import type { RuleEngineCondition } from "@karakeep/shared/types/rules"; + +import { FeedSelector } from "../feeds/FeedSelector"; +import { TagAutocomplete } from "../tags/TagAutocomplete"; + +interface ConditionBuilderProps { + value: RuleEngineCondition; + onChange: (condition: RuleEngineCondition) => void; + level?: number; + onRemove?: () => void; +} + +export function ConditionBuilder({ + value, + onChange, + level = 0, + onRemove, +}: ConditionBuilderProps) { + const { t } = useTranslation(); + const [isOpen, setIsOpen] = useState(true); + + const handleTypeChange = (type: RuleEngineCondition["type"]) => { + switch (type) { + case "urlContains": + onChange({ type: "urlContains", str: "" }); + break; + case "importedFromFeed": + onChange({ type: "importedFromFeed", feedId: "" }); + break; + case "bookmarkTypeIs": + onChange({ type: "bookmarkTypeIs", bookmarkType: "link" }); + break; + case "hasTag": + onChange({ type: "hasTag", tagId: "" }); + break; + case "isFavourited": + onChange({ type: "isFavourited" }); + break; + case "isArchived": + onChange({ type: "isArchived" }); + break; + case "and": + onChange({ type: "and", conditions: [] }); + break; + case "or": + onChange({ type: "or", conditions: [] }); + break; + case "alwaysTrue": + onChange({ type: "alwaysTrue" }); + break; + default: { + const _exhaustiveCheck: never = type; + return null; + } + } + }; + + const renderConditionIcon = (type: RuleEngineCondition["type"]) => { + switch (type) { + case "urlContains": + return ; + case "importedFromFeed": + return ; + case "bookmarkTypeIs": + return ; + case "hasTag": + return ; + case "isFavourited": + return ; + case "isArchived": + return ; + default: + return null; + } + }; + + const renderConditionFields = () => { + switch (value.type) { + case "urlContains": + return ( +
+ onChange({ ...value, str: e.target.value })} + placeholder="URL contains..." + className="w-full" + /> +
+ ); + + case "importedFromFeed": + return ( +
+ onChange({ ...value, feedId: e })} + className="w-full" + /> +
+ ); + + case "bookmarkTypeIs": + return ( +
+ +
+ ); + + case "hasTag": + return ( +
+ onChange({ type: value.type, tagId: t })} + /> +
+ ); + + case "and": + case "or": + return ( +
+ {value.conditions.map((condition, index) => ( + { + const newConditions = [...value.conditions]; + newConditions[index] = newCondition; + onChange({ ...value, conditions: newConditions }); + }} + level={level + 1} + onRemove={() => { + const newConditions = [...value.conditions]; + newConditions.splice(index, 1); + onChange({ ...value, conditions: newConditions }); + }} + /> + ))} + + +
+ ); + + default: + return null; + } + }; + + const ConditionSelector = () => ( + + ); + + return ( + + + {value.type === "and" || value.type === "or" ? ( + +
+
+ + + + + + {value.conditions.length} condition + {value.conditions.length !== 1 ? "s" : ""} + +
+ + {onRemove && ( + + )} +
+ + {renderConditionFields()} + + ) : ( +
+
+
+ {renderConditionIcon(value.type)} + +
+ + {onRemove && ( + + )} +
+ + {renderConditionFields()} +
+ )} + + + ); +} diff --git a/apps/web/components/dashboard/rules/RuleEngineEventSelector.tsx b/apps/web/components/dashboard/rules/RuleEngineEventSelector.tsx new file mode 100644 index 00000000..ae37945e --- /dev/null +++ b/apps/web/components/dashboard/rules/RuleEngineEventSelector.tsx @@ -0,0 +1,107 @@ +import { Card, CardContent } from "@/components/ui/card"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { useTranslation } from "react-i18next"; + +import type { RuleEngineEvent } from "@karakeep/shared/types/rules"; + +import { BookmarkListSelector } from "../lists/BookmarkListSelector"; +import { TagAutocomplete } from "../tags/TagAutocomplete"; + +interface EventSelectorProps { + value: RuleEngineEvent; + onChange: (event: RuleEngineEvent) => void; +} + +export function EventSelector({ value, onChange }: EventSelectorProps) { + const { t } = useTranslation(); + const handleTypeChange = (type: RuleEngineEvent["type"]) => { + switch (type) { + case "bookmarkAdded": + onChange({ type: "bookmarkAdded" }); + break; + case "tagAdded": + onChange({ type: "tagAdded", tagId: "" }); + break; + case "tagRemoved": + onChange({ type: "tagRemoved", tagId: "" }); + break; + case "addedToList": + onChange({ type: "addedToList", listId: "" }); + break; + case "removedFromList": + onChange({ type: "removedFromList", listId: "" }); + break; + case "favourited": + onChange({ type: "favourited" }); + break; + case "archived": + onChange({ type: "archived" }); + break; + default: { + const _exhaustiveCheck: never = type; + return null; + } + } + }; + + return ( + + +
+ + + {/* Additional fields based on event type */} + {(value.type === "tagAdded" || value.type === "tagRemoved") && ( + onChange({ type: value.type, tagId: t ?? "" })} + /> + )} + + {(value.type === "addedToList" || + value.type === "removedFromList") && ( + onChange({ type: value.type, listId: l })} + /> + )} +
+ + + ); +} diff --git a/apps/web/components/dashboard/rules/RuleEngineRuleEditor.tsx b/apps/web/components/dashboard/rules/RuleEngineRuleEditor.tsx new file mode 100644 index 00000000..da10317a --- /dev/null +++ b/apps/web/components/dashboard/rules/RuleEngineRuleEditor.tsx @@ -0,0 +1,203 @@ +import type React from "react"; +import { useEffect, useState } from "react"; +import { ActionBuilder } from "@/components/dashboard/rules/RuleEngineActionBuilder"; +import { ConditionBuilder } from "@/components/dashboard/rules/RuleEngineConditionBuilder"; +import { EventSelector } from "@/components/dashboard/rules/RuleEngineEventSelector"; +import { ActionButton } from "@/components/ui/action-button"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Textarea } from "@/components/ui/textarea"; +import { toast } from "@/components/ui/use-toast"; +import { Save, X } from "lucide-react"; +import { useTranslation } from "react-i18next"; + +import type { + RuleEngineAction, + RuleEngineCondition, + RuleEngineEvent, + RuleEngineRule, +} from "@karakeep/shared/types/rules"; +import { + useCreateRule, + useUpdateRule, +} from "@karakeep/shared-react/hooks/rules"; + +interface RuleEditorProps { + rule: Omit & { id: string | null }; + onCancel: () => void; +} + +export function RuleEditor({ rule, onCancel }: RuleEditorProps) { + const { t } = useTranslation(); + const { mutate: createRule, isPending: isCreating } = useCreateRule({ + onSuccess: () => { + toast({ + description: t("settings.rules.rule_has_been_created"), + }); + onCancel(); + }, + onError: (e) => { + if (e.data?.code == "BAD_REQUEST") { + if (e.data.zodError) { + toast({ + variant: "destructive", + description: Object.values(e.data.zodError.fieldErrors) + .flat() + .join("\n"), + }); + } else { + toast({ + variant: "destructive", + description: e.message, + }); + } + } else { + toast({ + variant: "destructive", + title: t("common.something_went_wrong"), + }); + } + }, + }); + const { mutate: updateRule, isPending: isUpdating } = useUpdateRule({ + onSuccess: () => { + toast({ + description: t("settings.rules.rule_has_been_updated"), + }); + onCancel(); + }, + onError: (e) => { + if (e.data?.code == "BAD_REQUEST") { + if (e.data.zodError) { + toast({ + variant: "destructive", + description: Object.values(e.data.zodError.fieldErrors) + .flat() + .join("\n"), + }); + } else { + toast({ + variant: "destructive", + description: e.message, + }); + } + } else { + toast({ + variant: "destructive", + title: t("common.something_went_wrong"), + }); + } + }, + }); + + const [editedRule, setEditedRule] = useState({ ...rule }); + + useEffect(() => { + setEditedRule({ ...rule }); + }, [rule]); + + const handleEventChange = (event: RuleEngineEvent) => { + setEditedRule({ ...editedRule, event }); + }; + + const handleConditionChange = (condition: RuleEngineCondition) => { + setEditedRule({ ...editedRule, condition }); + }; + + const handleActionsChange = (actions: RuleEngineAction[]) => { + setEditedRule({ ...editedRule, actions }); + }; + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + const rule = editedRule; + if (rule.id) { + updateRule({ + ...rule, + id: rule.id, + }); + } else { + createRule(rule); + } + }; + + return ( +
+ + + + {rule.id + ? t("settings.rules.edit_rule") + : t("settings.rules.ceate_rule")} + + + +
+
+ + + setEditedRule({ ...editedRule, name: e.target.value }) + } + placeholder={t("settings.rules.enter_rule_name")} + required + /> +
+ +
+ +