Learned patterns

apps/web/utils/ai/choose-rule/ai-choose-rule.ts

@@ -8,48 +8,48 @@ import { Braintrust } from "@/utils/braintrust";
 
 const logger = createScopedLogger("ai-choose-rule");
 
-const braintrust = new Braintrust("choose-rule-1");
+const braintrust = new Braintrust("choose-rule-2");
 
 type GetAiResponseOptions = {
   email: EmailForLLM;
   user: UserEmailWithAI;
-  rules: { instructions: string }[];
+  rules: { name: string; instructions: string }[];
 };
 
 async function getAiResponse(options: GetAiResponseOptions) {
   const { email, user, rules } = options;
 
-  const specialRuleNumber = rules.length + 1;
-
   const emailSection = stringifyEmail(email, 500);
 
   const system = `You are an AI assistant that helps people manage their emails.
 
 <instructions>
-IMPORTANT: Follow these instructions carefully when selecting a rule:
-
-<priority>
-1. Match the email to a SPECIFIC user-defined rule that addresses the email's exact content or purpose.
-2. If the email doesn't match any specific rule but the user has a catch-all rule (like "emails that don't match other criteria"), use that catch-all rule.
-3. Only use rule #${specialRuleNumber} (system fallback) if no user-defined rule can reasonably apply.
-</priority>
-
-<guidelines>
-- If a rule says to exclude certain types of emails, DO NOT select that rule for those excluded emails.
-- When multiple rules match, choose the more specific one that best matches the email's content.
-- Rules about requiring replies should be prioritized when the email clearly needs a response.
-- Rule #${specialRuleNumber} should ONLY be selected when there is absolutely no user-defined rule that could apply.
-</guidelines>
+  IMPORTANT: Follow these instructions carefully when selecting a rule:
+
+  <priority>
+  1. Match the email to a SPECIFIC user-defined rule that addresses the email's exact content or purpose.
+  2. If the email doesn't match any specific rule but the user has a catch-all rule (like "emails that don't match other criteria"), use that catch-all rule.
+  3. Only set "noMatchFound" to true if no user-defined rule can reasonably apply.
+  </priority>
+
+  <guidelines>
+  - If a rule says to exclude certain types of emails, DO NOT select that rule for those excluded emails.
+  - When multiple rules match, choose the more specific one that best matches the email's content.
+  - Rules about requiring replies should be prioritized when the email clearly needs a response.
+  </guidelines>
 </instructions>
 
 <user_rules>
-${rules.map((rule, i) => `${i + 1}. ${rule.instructions}`).join("\n")}
+${rules
+  .map(
+    (rule) => `<rule>
+  <name>${rule.name}</name>
+  <instructions>${rule.instructions}</instructions>
+</rule>`,
+  )
+  .join("\n")}
 </user_rules>
 
-<system_fallback>
-${specialRuleNumber}. None of the other rules match or not enough information to make a decision.
-</system_fallback>
-
 ${
   user.about
     ? `<user_info>
@@ -64,7 +64,8 @@ ${
 <outputFormat>
 Respond with a JSON object with the following fields:
 "reason" - the reason you chose that rule. Keep it concise.
-"rule" - the number of the rule you want to apply
+"ruleName" - the exact name of the rule you want to apply
+"noMatchFound" - true if no match was found, false otherwise
 </outputFormat>`;
 
   const prompt = `Select a rule to apply to this email that was sent to me:
@@ -97,7 +98,8 @@ ${emailSection}
     ],
     schema: z.object({
       reason: z.string(),
-      rule: z.number(),
+      ruleName: z.string(),
+      noMatchFound: z.boolean().optional(),
     }),
     userEmail: user.email || "",
     usageLabel: "Choose rule",
@@ -109,23 +111,22 @@ ${emailSection}
     id: email.id,
     input: {
       email: emailSection,
-      rules: rules.map((rule, i) => ({
-        ruleNumber: i + 1,
+      rules: rules.map((rule) => ({
+        name: rule.name,
         instructions: rule.instructions,
       })),
       hasAbout: !!user.about,
       userAbout: user.about,
       userEmail: user.email,
-      specialRuleNumber,
     },
-    expected: aiResponse.object.rule,
+    expected: aiResponse.object.ruleName,
   });
 
   return aiResponse.object;
 }
 
 export async function aiChooseRule<
-  T extends { instructions: string },
+  T extends { name: string; instructions: string },
 >(options: { email: EmailForLLM; rules: T[]; user: UserEmailWithAI }) {
   const { email, rules, user } = options;
 
@@ -137,13 +138,14 @@ export async function aiChooseRule<
     user,
   });
 
-  const ruleNumber = aiResponse ? aiResponse.rule - 1 : undefined;
-  if (typeof ruleNumber !== "number") {
-    logger.warn("No rule selected");
-    return { reason: aiResponse?.reason };
-  }
+  if (aiResponse.noMatchFound)
+    return { rule: undefined, reason: "No match found" };
 
-  const selectedRule = rules[ruleNumber];
+  const selectedRule = aiResponse.ruleName
+    ? rules.find(
+        (rule) => rule.name.toLowerCase() === aiResponse.ruleName.toLowerCase(),
+      )
+    : undefined;
 
   return {
     rule: selectedRule,

apps/web/utils/reply-tracker/inbound.ts

@@ -132,6 +132,7 @@ export async function handleInboundReply(
     email: getEmailForLLM(message),
     rules: replyTrackingRules.map((rule) => ({
       id: rule.id,
+      name: rule.name,
       instructions: rule.instructions || "",
     })),
     user,