Make AI categorization prompts user-configurable

Add SystemPrompt and UserPromptTemplate fields to AI categorization settings so users can customize the prompts sent to Ollama without code changes. Null/empty values fall back to built-in defaults. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-08 19:30:26 -05:00
parent 1c7abb7ffc
commit e742b0fbe7
9 changed files with 1750 additions and 40 deletions
@@ -16,6 +16,29 @@ public class AiCategorizationService : IAiCategorizationService
    private readonly IHttpClientFactory _httpClientFactory;
    private readonly ILogger<AiCategorizationService> _logger;

+    internal const string DefaultSystemPrompt =
+        """
+        You are a personal finance transaction categorizer. Classify each transaction into the most appropriate category.
+
+        You MUST respond with valid JSON in exactly this format:
+        {
+          "classifications": [
+            {"index": 0, "categoryName": "Groceries", "confidence": 0.9},
+            {"index": 1, "categoryName": "Utilities", "confidence": 0.85}
+          ]
+        }
+
+        Rules:
+        - "index" must be the transaction number (integer) from the list below
+        - "categoryName" must be one of the exact category names listed below
+        - "confidence" must be a number between 0 and 1
+        - Include one entry for every transaction
+
+        Available categories:
+        """;
+
+    internal const string DefaultUserPromptTemplate = "Classify these uncategorized transactions:";
+
    private static readonly JsonSerializerOptions JsonOptions = new()
    {
        PropertyNameCaseInsensitive = true,
@@ -39,12 +62,13 @@ public class AiCategorizationService : IAiCategorizationService
        if (settings == null)
        {
            return new AiCategorizationSettingsResponse(
-                false, "http://localhost:11434", "llama3.1:8b", 0.7, 30, true);
+                false, "http://localhost:11434", "llama3.1:8b", 0.7, 30, true, null, null);
        }

        return new AiCategorizationSettingsResponse(
            settings.IsEnabled, settings.OllamaUrl, settings.ModelName,
-            settings.ConfidenceThreshold, settings.TimeoutSeconds, settings.UpdatePayeeDefaults);
+            settings.ConfidenceThreshold, settings.TimeoutSeconds, settings.UpdatePayeeDefaults,
+            settings.SystemPrompt, settings.UserPromptTemplate);
    }

    public async Task<AiCategorizationSettingsResponse> UpdateSettingsAsync(Guid userId, UpdateAiCategorizationSettingsRequest request)
@@ -67,13 +91,16 @@ public class AiCategorizationService : IAiCategorizationService
        settings.ConfidenceThreshold = request.ConfidenceThreshold;
        settings.TimeoutSeconds = request.TimeoutSeconds;
        settings.UpdatePayeeDefaults = request.UpdatePayeeDefaults;
+        settings.SystemPrompt = string.IsNullOrWhiteSpace(request.SystemPrompt) ? null : request.SystemPrompt;
+        settings.UserPromptTemplate = string.IsNullOrWhiteSpace(request.UserPromptTemplate) ? null : request.UserPromptTemplate;
        settings.UpdatedAt = DateTime.UtcNow;

        await _db.SaveChangesAsync();

        return new AiCategorizationSettingsResponse(
            settings.IsEnabled, settings.OllamaUrl, settings.ModelName,
-            settings.ConfidenceThreshold, settings.TimeoutSeconds, settings.UpdatePayeeDefaults);
+            settings.ConfidenceThreshold, settings.TimeoutSeconds, settings.UpdatePayeeDefaults,
+            settings.SystemPrompt, settings.UserPromptTemplate);
    }

    public async Task<OllamaConnectionTestResponse> TestConnectionAsync(Guid userId)
@@ -158,8 +185,8 @@ public class AiCategorizationService : IAiCategorizationService
        List<Transaction> batch,
        int globalOffset)
    {
-        var systemMessage = BuildSystemMessage(categories);
-        var userMessage = BuildUserMessage(recentCategorized, batch);
+        var systemMessage = BuildSystemMessage(settings, categories);
+        var userMessage = BuildUserMessage(settings, recentCategorized, batch);

        var chatRequest = new OllamaChatRequest(
            settings.ModelName,
@@ -179,6 +206,8 @@ public class AiCategorizationService : IAiCategorizationService
            var requestJson = JsonSerializer.Serialize(chatRequest, JsonOptions);
            var content = new StringContent(requestJson, Encoding.UTF8, "application/json");

+            _logger.LogInformation("Sending {Count} transactions to Ollama for classification (batch offset {Offset})", batch.Count, globalOffset);
+
            var response = await client.PostAsync($"{settings.OllamaUrl.TrimEnd('/')}/api/chat", content);
            response.EnsureSuccessStatusCode();

@@ -186,7 +215,12 @@ public class AiCategorizationService : IAiCategorizationService
            var chatResponse = JsonSerializer.Deserialize<OllamaChatResponse>(responseJson, JsonOptions);

            if (chatResponse?.Message?.Content == null)
+            {
+                _logger.LogWarning("Ollama returned null message content for batch at offset {Offset}", globalOffset);
                return new List<TransactionClassification>();
+            }
+
+            _logger.LogDebug("Ollama raw response for batch at offset {Offset}: {Response}", globalOffset, chatResponse.Message.Content);

            return ParseClassifications(chatResponse.Message.Content, categories, batch.Count, globalOffset, settings.ConfidenceThreshold);
        }
@@ -197,24 +231,21 @@ public class AiCategorizationService : IAiCategorizationService
        }
    }

-    private static string BuildSystemMessage(List<Category> categories)
+    private static string BuildSystemMessage(AiCategorizationSettings settings, List<Category> categories)
    {
        var sb = new StringBuilder();
-        sb.AppendLine("You are a personal finance transaction categorizer. Classify each transaction into the most appropriate category.");
-        sb.AppendLine("Respond with JSON: {\"classifications\": [{\"index\": 0, \"categoryId\": \"...\", \"categoryName\": \"...\", \"confidence\": 0.9}, ...]}");
-        sb.AppendLine();
-        sb.AppendLine("Available categories:");
+        sb.AppendLine(settings.SystemPrompt ?? DefaultSystemPrompt);

        foreach (var category in categories)
        {
-            var parentInfo = category.Parent != null ? $", under \"{category.Parent.Name}\"" : "";
-            sb.AppendLine($"- {category.Id} \"{category.Name}\" ({category.Type}{parentInfo})");
+            var parentInfo = category.Parent != null ? $" (under \"{category.Parent.Name}\")" : "";
+            sb.AppendLine($"- \"{category.Name}\"{parentInfo}");
        }

        return sb.ToString();
    }

-    private static string BuildUserMessage(List<Transaction> recentCategorized, List<Transaction> batch)
+    private static string BuildUserMessage(AiCategorizationSettings settings, List<Transaction> recentCategorized, List<Transaction> batch)
    {
        var sb = new StringBuilder();

@@ -232,7 +263,7 @@ public class AiCategorizationService : IAiCategorizationService
            sb.AppendLine();
        }

-        sb.AppendLine("Classify these uncategorized transactions:");
+        sb.AppendLine(settings.UserPromptTemplate ?? DefaultUserPromptTemplate);
        for (int i = 0; i < batch.Count; i++)
        {
            var t = batch[i];
@@ -243,7 +274,7 @@ public class AiCategorizationService : IAiCategorizationService
        return sb.ToString();
    }

-    private static List<TransactionClassification> ParseClassifications(
+    private List<TransactionClassification> ParseClassifications(
        string responseContent,
        List<Category> categories,
        int batchSize,
@@ -255,55 +286,130 @@ public class AiCategorizationService : IAiCategorizationService
            using var doc = JsonDocument.Parse(responseContent);
            var root = doc.RootElement;

-            if (!root.TryGetProperty("classifications", out var classificationsArray))
-                return new List<TransactionClassification>();
+            // Try "classifications" wrapper first, then bare array
+            JsonElement classificationsArray;
+            if (root.TryGetProperty("classifications", out classificationsArray))
+            {
+                // good
+            }
+            else if (root.ValueKind == JsonValueKind.Array)
+            {
+                classificationsArray = root;
+            }
+            else
+            {
+                // Try common alternative wrapper names
+                if (!root.TryGetProperty("results", out classificationsArray) &&
+                    !root.TryGetProperty("data", out classificationsArray))
+                {
+                    _logger.LogWarning("Ollama response has no 'classifications' array. Root properties: {Props}",
+                        string.Join(", ", root.EnumerateObject().Select(p => p.Name)));
+                    return new List<TransactionClassification>();
+                }
+            }

            var categoryLookup = categories.ToDictionary(c => c.Id);
            var categoryNameLookup = categories
-                .GroupBy(c => c.Name.ToLowerInvariant())
+                .GroupBy(c => c.Name.Trim().ToLowerInvariant())
                .ToDictionary(g => g.Key, g => g.First());

            var results = new List<TransactionClassification>();
+            int skippedNoIndex = 0, skippedBadIndex = 0, skippedLowConf = 0, skippedNoMatch = 0;

            foreach (var item in classificationsArray.EnumerateArray())
            {
-                if (!item.TryGetProperty("index", out var indexProp))
-                    continue;
+                // Parse index — handle both number and string
+                int localIndex;
+                if (item.TryGetProperty("index", out var indexProp))
+                {
+                    if (indexProp.ValueKind == JsonValueKind.Number)
+                        localIndex = indexProp.GetInt32();
+                    else if (indexProp.ValueKind == JsonValueKind.String && int.TryParse(indexProp.GetString(), out var parsed))
+                        localIndex = parsed;
+                    else { skippedNoIndex++; continue; }
+                }
+                else { skippedNoIndex++; continue; }

-                var localIndex = indexProp.GetInt32();
                if (localIndex < 0 || localIndex >= batchSize)
-                    continue;
+                { skippedBadIndex++; continue; }

                var globalIndex = globalOffset + localIndex;

-                double confidence = 0;
+                // Parse confidence — handle number, string, or missing (default 1.0)
+                double confidence = 1.0;
                if (item.TryGetProperty("confidence", out var confProp))
-                    confidence = confProp.GetDouble();
+                {
+                    if (confProp.ValueKind == JsonValueKind.Number)
+                        confidence = confProp.GetDouble();
+                    else if (confProp.ValueKind == JsonValueKind.String && double.TryParse(confProp.GetString(), out var parsedConf))
+                        confidence = parsedConf;
+                }

                if (confidence < confidenceThreshold)
-                    continue;
+                { skippedLowConf++; continue; }

                Guid? categoryId = null;
                string? categoryName = null;

-                // Try to resolve by categoryId first
-                if (item.TryGetProperty("categoryId", out var catIdProp) && catIdProp.ValueKind == JsonValueKind.String)
+                // Try to resolve by categoryId (also check snake_case variant)
+                foreach (var propName in new[] { "categoryId", "category_id" })
                {
-                    if (Guid.TryParse(catIdProp.GetString(), out var parsedId) && categoryLookup.ContainsKey(parsedId))
+                    if (categoryId.HasValue) break;
+                    if (item.TryGetProperty(propName, out var catIdProp) && catIdProp.ValueKind == JsonValueKind.String)
                    {
-                        categoryId = parsedId;
-                        categoryName = categoryLookup[parsedId].Name;
+                        if (Guid.TryParse(catIdProp.GetString(), out var parsedId) && categoryLookup.ContainsKey(parsedId))
+                        {
+                            categoryId = parsedId;
+                            categoryName = categoryLookup[parsedId].Name;
+                        }
                    }
                }

-                // Fall back to name matching
-                if (!categoryId.HasValue && item.TryGetProperty("categoryName", out var catNameProp) && catNameProp.ValueKind == JsonValueKind.String)
+                // Fall back to name matching (also check snake_case variant)
+                if (!categoryId.HasValue)
                {
-                    var name = catNameProp.GetString()?.ToLowerInvariant();
-                    if (name != null && categoryNameLookup.TryGetValue(name, out var matched))
+                    string? rawName = null;
+                    foreach (var propName in new[] { "categoryName", "category_name", "category" })
                    {
-                        categoryId = matched.Id;
-                        categoryName = matched.Name;
+                        if (item.TryGetProperty(propName, out var catNameProp) && catNameProp.ValueKind == JsonValueKind.String)
+                        {
+                            rawName = catNameProp.GetString();
+                            break;
+                        }
+                    }
+
+                    if (rawName != null)
+                    {
+                        var name = rawName.Trim().ToLowerInvariant();
+
+                        // Exact match
+                        if (categoryNameLookup.TryGetValue(name, out var matched))
+                        {
+                            categoryId = matched.Id;
+                            categoryName = matched.Name;
+                        }
+                        else
+                        {
+                            // Handle "Parent > Child" format — try the last segment
+                            var lastSegment = name.Contains('>') ? name.Split('>').Last().Trim() : null;
+                            if (lastSegment != null && categoryNameLookup.TryGetValue(lastSegment, out var segmentMatch))
+                            {
+                                categoryId = segmentMatch.Id;
+                                categoryName = segmentMatch.Name;
+                            }
+                            else
+                            {
+                                // Contains-based fuzzy match as last resort
+                                var fuzzy = categories.FirstOrDefault(c =>
+                                    c.Name.Trim().ToLowerInvariant().Contains(name) ||
+                                    name.Contains(c.Name.Trim().ToLowerInvariant()));
+                                if (fuzzy != null)
+                                {
+                                    categoryId = fuzzy.Id;
+                                    categoryName = fuzzy.Name;
+                                }
+                            }
+                        }
                    }
                }

@@ -311,12 +417,21 @@ public class AiCategorizationService : IAiCategorizationService
                {
                    results.Add(new TransactionClassification(globalIndex, categoryId, categoryName, confidence));
                }
+                else
+                {
+                    skippedNoMatch++;
+                }
            }

+            _logger.LogInformation(
+                "Parsed {Matched} classifications from {Total} items. Skipped: {NoIndex} no index, {BadIndex} bad index, {LowConf} low confidence, {NoMatch} no category match",
+                results.Count, classificationsArray.GetArrayLength(), skippedNoIndex, skippedBadIndex, skippedLowConf, skippedNoMatch);
+
            return results;
        }
-        catch (JsonException)
+        catch (JsonException ex)
        {
+            _logger.LogWarning(ex, "Failed to parse Ollama JSON response: {Content}", responseContent.Length > 500 ? responseContent[..500] : responseContent);
            return new List<TransactionClassification>();
        }
    }