Private
Public Access
1
0

Make AI categorization prompts user-configurable

Add SystemPrompt and UserPromptTemplate fields to AI categorization
settings so users can customize the prompts sent to Ollama without
code changes. Null/empty values fall back to built-in defaults.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Catherine Renelle
2026-02-08 19:30:26 -05:00
parent 1c7abb7ffc
commit e742b0fbe7
9 changed files with 1750 additions and 40 deletions
+2
View File
@@ -493,6 +493,8 @@ export interface AiCategorizationSettings {
confidenceThreshold: number confidenceThreshold: number
timeoutSeconds: number timeoutSeconds: number
updatePayeeDefaults: boolean updatePayeeDefaults: boolean
systemPrompt: string | null
userPromptTemplate: string | null
} }
export interface OllamaConnectionTestResult { export interface OllamaConnectionTestResult {
+66 -1
View File
@@ -466,6 +466,44 @@
</v-card-text> </v-card-text>
</v-card> </v-card>
<v-card class="mt-4">
<v-card-title>Prompt Templates</v-card-title>
<v-card-text>
<p class="text-body-2 text-medium-emphasis mb-4">
Customize the prompts sent to the AI model. Leave empty to use built-in defaults.
The category list is always appended to the system prompt, and the transaction list is always appended to the user prompt.
</p>
<v-textarea
v-model="aiSettings.systemPrompt"
label="System Prompt"
variant="outlined"
rows="8"
:placeholder="defaultSystemPrompt"
persistent-placeholder
class="mb-4"
/>
<v-textarea
v-model="aiSettings.userPromptTemplate"
label="User Prompt Template"
variant="outlined"
rows="3"
:placeholder="defaultUserPromptTemplate"
persistent-placeholder
class="mb-4"
/>
<v-btn
variant="tonal"
color="secondary"
@click="resetPromptsToDefaults"
>
Reset to Defaults
</v-btn>
</v-card-text>
</v-card>
<v-snackbar v-model="showAiSuccess" :timeout="4000" color="success"> <v-snackbar v-model="showAiSuccess" :timeout="4000" color="success">
AI categorization settings saved AI categorization settings saved
</v-snackbar> </v-snackbar>
@@ -801,7 +839,9 @@ const aiSettings = reactive<AiCategorizationSettings>({
modelName: 'llama3.1:8b', modelName: 'llama3.1:8b',
confidenceThreshold: 0.7, confidenceThreshold: 0.7,
timeoutSeconds: 30, timeoutSeconds: 30,
updatePayeeDefaults: true updatePayeeDefaults: true,
systemPrompt: null,
userPromptTemplate: null
}) })
const aiSaving = ref(false) const aiSaving = ref(false)
const aiTesting = ref(false) const aiTesting = ref(false)
@@ -811,6 +851,31 @@ const showAiSuccess = ref(false)
const showAiError = ref(false) const showAiError = ref(false)
const aiErrorMessage = ref('') const aiErrorMessage = ref('')
const defaultSystemPrompt = `You are a personal finance transaction categorizer. Classify each transaction into the most appropriate category.
You MUST respond with valid JSON in exactly this format:
{
"classifications": [
{"index": 0, "categoryName": "Groceries", "confidence": 0.9},
{"index": 1, "categoryName": "Utilities", "confidence": 0.85}
]
}
Rules:
- "index" must be the transaction number (integer) from the list below
- "categoryName" must be one of the exact category names listed below
- "confidence" must be a number between 0 and 1
- Include one entry for every transaction
Available categories:`
const defaultUserPromptTemplate = 'Classify these uncategorized transactions:'
function resetPromptsToDefaults() {
aiSettings.systemPrompt = null
aiSettings.userPromptTemplate = null
}
async function loadAiSettings() { async function loadAiSettings() {
try { try {
const { data } = await aiCategorizationApi.getSettings() const { data } = await aiCategorizationApi.getSettings()
@@ -16,6 +16,29 @@ public class AiCategorizationService : IAiCategorizationService
private readonly IHttpClientFactory _httpClientFactory; private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<AiCategorizationService> _logger; private readonly ILogger<AiCategorizationService> _logger;
internal const string DefaultSystemPrompt =
"""
You are a personal finance transaction categorizer. Classify each transaction into the most appropriate category.
You MUST respond with valid JSON in exactly this format:
{
"classifications": [
{"index": 0, "categoryName": "Groceries", "confidence": 0.9},
{"index": 1, "categoryName": "Utilities", "confidence": 0.85}
]
}
Rules:
- "index" must be the transaction number (integer) from the list below
- "categoryName" must be one of the exact category names listed below
- "confidence" must be a number between 0 and 1
- Include one entry for every transaction
Available categories:
""";
internal const string DefaultUserPromptTemplate = "Classify these uncategorized transactions:";
private static readonly JsonSerializerOptions JsonOptions = new() private static readonly JsonSerializerOptions JsonOptions = new()
{ {
PropertyNameCaseInsensitive = true, PropertyNameCaseInsensitive = true,
@@ -39,12 +62,13 @@ public class AiCategorizationService : IAiCategorizationService
if (settings == null) if (settings == null)
{ {
return new AiCategorizationSettingsResponse( return new AiCategorizationSettingsResponse(
false, "http://localhost:11434", "llama3.1:8b", 0.7, 30, true); false, "http://localhost:11434", "llama3.1:8b", 0.7, 30, true, null, null);
} }
return new AiCategorizationSettingsResponse( return new AiCategorizationSettingsResponse(
settings.IsEnabled, settings.OllamaUrl, settings.ModelName, settings.IsEnabled, settings.OllamaUrl, settings.ModelName,
settings.ConfidenceThreshold, settings.TimeoutSeconds, settings.UpdatePayeeDefaults); settings.ConfidenceThreshold, settings.TimeoutSeconds, settings.UpdatePayeeDefaults,
settings.SystemPrompt, settings.UserPromptTemplate);
} }
public async Task<AiCategorizationSettingsResponse> UpdateSettingsAsync(Guid userId, UpdateAiCategorizationSettingsRequest request) public async Task<AiCategorizationSettingsResponse> UpdateSettingsAsync(Guid userId, UpdateAiCategorizationSettingsRequest request)
@@ -67,13 +91,16 @@ public class AiCategorizationService : IAiCategorizationService
settings.ConfidenceThreshold = request.ConfidenceThreshold; settings.ConfidenceThreshold = request.ConfidenceThreshold;
settings.TimeoutSeconds = request.TimeoutSeconds; settings.TimeoutSeconds = request.TimeoutSeconds;
settings.UpdatePayeeDefaults = request.UpdatePayeeDefaults; settings.UpdatePayeeDefaults = request.UpdatePayeeDefaults;
settings.SystemPrompt = string.IsNullOrWhiteSpace(request.SystemPrompt) ? null : request.SystemPrompt;
settings.UserPromptTemplate = string.IsNullOrWhiteSpace(request.UserPromptTemplate) ? null : request.UserPromptTemplate;
settings.UpdatedAt = DateTime.UtcNow; settings.UpdatedAt = DateTime.UtcNow;
await _db.SaveChangesAsync(); await _db.SaveChangesAsync();
return new AiCategorizationSettingsResponse( return new AiCategorizationSettingsResponse(
settings.IsEnabled, settings.OllamaUrl, settings.ModelName, settings.IsEnabled, settings.OllamaUrl, settings.ModelName,
settings.ConfidenceThreshold, settings.TimeoutSeconds, settings.UpdatePayeeDefaults); settings.ConfidenceThreshold, settings.TimeoutSeconds, settings.UpdatePayeeDefaults,
settings.SystemPrompt, settings.UserPromptTemplate);
} }
public async Task<OllamaConnectionTestResponse> TestConnectionAsync(Guid userId) public async Task<OllamaConnectionTestResponse> TestConnectionAsync(Guid userId)
@@ -158,8 +185,8 @@ public class AiCategorizationService : IAiCategorizationService
List<Transaction> batch, List<Transaction> batch,
int globalOffset) int globalOffset)
{ {
var systemMessage = BuildSystemMessage(categories); var systemMessage = BuildSystemMessage(settings, categories);
var userMessage = BuildUserMessage(recentCategorized, batch); var userMessage = BuildUserMessage(settings, recentCategorized, batch);
var chatRequest = new OllamaChatRequest( var chatRequest = new OllamaChatRequest(
settings.ModelName, settings.ModelName,
@@ -179,6 +206,8 @@ public class AiCategorizationService : IAiCategorizationService
var requestJson = JsonSerializer.Serialize(chatRequest, JsonOptions); var requestJson = JsonSerializer.Serialize(chatRequest, JsonOptions);
var content = new StringContent(requestJson, Encoding.UTF8, "application/json"); var content = new StringContent(requestJson, Encoding.UTF8, "application/json");
_logger.LogInformation("Sending {Count} transactions to Ollama for classification (batch offset {Offset})", batch.Count, globalOffset);
var response = await client.PostAsync($"{settings.OllamaUrl.TrimEnd('/')}/api/chat", content); var response = await client.PostAsync($"{settings.OllamaUrl.TrimEnd('/')}/api/chat", content);
response.EnsureSuccessStatusCode(); response.EnsureSuccessStatusCode();
@@ -186,7 +215,12 @@ public class AiCategorizationService : IAiCategorizationService
var chatResponse = JsonSerializer.Deserialize<OllamaChatResponse>(responseJson, JsonOptions); var chatResponse = JsonSerializer.Deserialize<OllamaChatResponse>(responseJson, JsonOptions);
if (chatResponse?.Message?.Content == null) if (chatResponse?.Message?.Content == null)
{
_logger.LogWarning("Ollama returned null message content for batch at offset {Offset}", globalOffset);
return new List<TransactionClassification>(); return new List<TransactionClassification>();
}
_logger.LogDebug("Ollama raw response for batch at offset {Offset}: {Response}", globalOffset, chatResponse.Message.Content);
return ParseClassifications(chatResponse.Message.Content, categories, batch.Count, globalOffset, settings.ConfidenceThreshold); return ParseClassifications(chatResponse.Message.Content, categories, batch.Count, globalOffset, settings.ConfidenceThreshold);
} }
@@ -197,24 +231,21 @@ public class AiCategorizationService : IAiCategorizationService
} }
} }
private static string BuildSystemMessage(List<Category> categories) private static string BuildSystemMessage(AiCategorizationSettings settings, List<Category> categories)
{ {
var sb = new StringBuilder(); var sb = new StringBuilder();
sb.AppendLine("You are a personal finance transaction categorizer. Classify each transaction into the most appropriate category."); sb.AppendLine(settings.SystemPrompt ?? DefaultSystemPrompt);
sb.AppendLine("Respond with JSON: {\"classifications\": [{\"index\": 0, \"categoryId\": \"...\", \"categoryName\": \"...\", \"confidence\": 0.9}, ...]}");
sb.AppendLine();
sb.AppendLine("Available categories:");
foreach (var category in categories) foreach (var category in categories)
{ {
var parentInfo = category.Parent != null ? $", under \"{category.Parent.Name}\"" : ""; var parentInfo = category.Parent != null ? $" (under \"{category.Parent.Name}\")" : "";
sb.AppendLine($"- {category.Id} \"{category.Name}\" ({category.Type}{parentInfo})"); sb.AppendLine($"- \"{category.Name}\"{parentInfo}");
} }
return sb.ToString(); return sb.ToString();
} }
private static string BuildUserMessage(List<Transaction> recentCategorized, List<Transaction> batch) private static string BuildUserMessage(AiCategorizationSettings settings, List<Transaction> recentCategorized, List<Transaction> batch)
{ {
var sb = new StringBuilder(); var sb = new StringBuilder();
@@ -232,7 +263,7 @@ public class AiCategorizationService : IAiCategorizationService
sb.AppendLine(); sb.AppendLine();
} }
sb.AppendLine("Classify these uncategorized transactions:"); sb.AppendLine(settings.UserPromptTemplate ?? DefaultUserPromptTemplate);
for (int i = 0; i < batch.Count; i++) for (int i = 0; i < batch.Count; i++)
{ {
var t = batch[i]; var t = batch[i];
@@ -243,7 +274,7 @@ public class AiCategorizationService : IAiCategorizationService
return sb.ToString(); return sb.ToString();
} }
private static List<TransactionClassification> ParseClassifications( private List<TransactionClassification> ParseClassifications(
string responseContent, string responseContent,
List<Category> categories, List<Category> categories,
int batchSize, int batchSize,
@@ -255,55 +286,130 @@ public class AiCategorizationService : IAiCategorizationService
using var doc = JsonDocument.Parse(responseContent); using var doc = JsonDocument.Parse(responseContent);
var root = doc.RootElement; var root = doc.RootElement;
if (!root.TryGetProperty("classifications", out var classificationsArray)) // Try "classifications" wrapper first, then bare array
return new List<TransactionClassification>(); JsonElement classificationsArray;
if (root.TryGetProperty("classifications", out classificationsArray))
{
// good
}
else if (root.ValueKind == JsonValueKind.Array)
{
classificationsArray = root;
}
else
{
// Try common alternative wrapper names
if (!root.TryGetProperty("results", out classificationsArray) &&
!root.TryGetProperty("data", out classificationsArray))
{
_logger.LogWarning("Ollama response has no 'classifications' array. Root properties: {Props}",
string.Join(", ", root.EnumerateObject().Select(p => p.Name)));
return new List<TransactionClassification>();
}
}
var categoryLookup = categories.ToDictionary(c => c.Id); var categoryLookup = categories.ToDictionary(c => c.Id);
var categoryNameLookup = categories var categoryNameLookup = categories
.GroupBy(c => c.Name.ToLowerInvariant()) .GroupBy(c => c.Name.Trim().ToLowerInvariant())
.ToDictionary(g => g.Key, g => g.First()); .ToDictionary(g => g.Key, g => g.First());
var results = new List<TransactionClassification>(); var results = new List<TransactionClassification>();
int skippedNoIndex = 0, skippedBadIndex = 0, skippedLowConf = 0, skippedNoMatch = 0;
foreach (var item in classificationsArray.EnumerateArray()) foreach (var item in classificationsArray.EnumerateArray())
{ {
if (!item.TryGetProperty("index", out var indexProp)) // Parse index — handle both number and string
continue; int localIndex;
if (item.TryGetProperty("index", out var indexProp))
{
if (indexProp.ValueKind == JsonValueKind.Number)
localIndex = indexProp.GetInt32();
else if (indexProp.ValueKind == JsonValueKind.String && int.TryParse(indexProp.GetString(), out var parsed))
localIndex = parsed;
else { skippedNoIndex++; continue; }
}
else { skippedNoIndex++; continue; }
var localIndex = indexProp.GetInt32();
if (localIndex < 0 || localIndex >= batchSize) if (localIndex < 0 || localIndex >= batchSize)
continue; { skippedBadIndex++; continue; }
var globalIndex = globalOffset + localIndex; var globalIndex = globalOffset + localIndex;
double confidence = 0; // Parse confidence — handle number, string, or missing (default 1.0)
double confidence = 1.0;
if (item.TryGetProperty("confidence", out var confProp)) if (item.TryGetProperty("confidence", out var confProp))
confidence = confProp.GetDouble(); {
if (confProp.ValueKind == JsonValueKind.Number)
confidence = confProp.GetDouble();
else if (confProp.ValueKind == JsonValueKind.String && double.TryParse(confProp.GetString(), out var parsedConf))
confidence = parsedConf;
}
if (confidence < confidenceThreshold) if (confidence < confidenceThreshold)
continue; { skippedLowConf++; continue; }
Guid? categoryId = null; Guid? categoryId = null;
string? categoryName = null; string? categoryName = null;
// Try to resolve by categoryId first // Try to resolve by categoryId (also check snake_case variant)
if (item.TryGetProperty("categoryId", out var catIdProp) && catIdProp.ValueKind == JsonValueKind.String) foreach (var propName in new[] { "categoryId", "category_id" })
{ {
if (Guid.TryParse(catIdProp.GetString(), out var parsedId) && categoryLookup.ContainsKey(parsedId)) if (categoryId.HasValue) break;
if (item.TryGetProperty(propName, out var catIdProp) && catIdProp.ValueKind == JsonValueKind.String)
{ {
categoryId = parsedId; if (Guid.TryParse(catIdProp.GetString(), out var parsedId) && categoryLookup.ContainsKey(parsedId))
categoryName = categoryLookup[parsedId].Name; {
categoryId = parsedId;
categoryName = categoryLookup[parsedId].Name;
}
} }
} }
// Fall back to name matching // Fall back to name matching (also check snake_case variant)
if (!categoryId.HasValue && item.TryGetProperty("categoryName", out var catNameProp) && catNameProp.ValueKind == JsonValueKind.String) if (!categoryId.HasValue)
{ {
var name = catNameProp.GetString()?.ToLowerInvariant(); string? rawName = null;
if (name != null && categoryNameLookup.TryGetValue(name, out var matched)) foreach (var propName in new[] { "categoryName", "category_name", "category" })
{ {
categoryId = matched.Id; if (item.TryGetProperty(propName, out var catNameProp) && catNameProp.ValueKind == JsonValueKind.String)
categoryName = matched.Name; {
rawName = catNameProp.GetString();
break;
}
}
if (rawName != null)
{
var name = rawName.Trim().ToLowerInvariant();
// Exact match
if (categoryNameLookup.TryGetValue(name, out var matched))
{
categoryId = matched.Id;
categoryName = matched.Name;
}
else
{
// Handle "Parent > Child" format — try the last segment
var lastSegment = name.Contains('>') ? name.Split('>').Last().Trim() : null;
if (lastSegment != null && categoryNameLookup.TryGetValue(lastSegment, out var segmentMatch))
{
categoryId = segmentMatch.Id;
categoryName = segmentMatch.Name;
}
else
{
// Contains-based fuzzy match as last resort
var fuzzy = categories.FirstOrDefault(c =>
c.Name.Trim().ToLowerInvariant().Contains(name) ||
name.Contains(c.Name.Trim().ToLowerInvariant()));
if (fuzzy != null)
{
categoryId = fuzzy.Id;
categoryName = fuzzy.Name;
}
}
}
} }
} }
@@ -311,12 +417,21 @@ public class AiCategorizationService : IAiCategorizationService
{ {
results.Add(new TransactionClassification(globalIndex, categoryId, categoryName, confidence)); results.Add(new TransactionClassification(globalIndex, categoryId, categoryName, confidence));
} }
else
{
skippedNoMatch++;
}
} }
_logger.LogInformation(
"Parsed {Matched} classifications from {Total} items. Skipped: {NoIndex} no index, {BadIndex} bad index, {LowConf} low confidence, {NoMatch} no category match",
results.Count, classificationsArray.GetArrayLength(), skippedNoIndex, skippedBadIndex, skippedLowConf, skippedNoMatch);
return results; return results;
} }
catch (JsonException) catch (JsonException ex)
{ {
_logger.LogWarning(ex, "Failed to parse Ollama JSON response: {Content}", responseContent.Length > 500 ? responseContent[..500] : responseContent);
return new List<TransactionClassification>(); return new List<TransactionClassification>();
} }
} }
+6 -2
View File
@@ -9,7 +9,9 @@ public record AiCategorizationSettingsResponse(
string ModelName, string ModelName,
double ConfidenceThreshold, double ConfidenceThreshold,
int TimeoutSeconds, int TimeoutSeconds,
bool UpdatePayeeDefaults); bool UpdatePayeeDefaults,
string? SystemPrompt,
string? UserPromptTemplate);
public record UpdateAiCategorizationSettingsRequest( public record UpdateAiCategorizationSettingsRequest(
bool IsEnabled, bool IsEnabled,
@@ -17,7 +19,9 @@ public record UpdateAiCategorizationSettingsRequest(
string ModelName, string ModelName,
double ConfidenceThreshold, double ConfidenceThreshold,
int TimeoutSeconds, int TimeoutSeconds,
bool UpdatePayeeDefaults); bool UpdatePayeeDefaults,
string? SystemPrompt,
string? UserPromptTemplate);
// Connection test // Connection test
public record OllamaConnectionTestResponse( public record OllamaConnectionTestResponse(
@@ -10,6 +10,8 @@ public class AiCategorizationSettings
public double ConfidenceThreshold { get; set; } = 0.7; public double ConfidenceThreshold { get; set; } = 0.7;
public int TimeoutSeconds { get; set; } = 30; public int TimeoutSeconds { get; set; } = 30;
public bool UpdatePayeeDefaults { get; set; } = true; public bool UpdatePayeeDefaults { get; set; } = true;
public string? SystemPrompt { get; set; }
public string? UserPromptTemplate { get; set; }
public DateTime UpdatedAt { get; set; } = DateTime.UtcNow; public DateTime UpdatedAt { get; set; } = DateTime.UtcNow;
public User User { get; set; } = null!; public User User { get; set; } = null!;
@@ -12,6 +12,8 @@ public class AiCategorizationSettingsConfiguration : IEntityTypeConfiguration<Ai
builder.HasKey(s => s.Id); builder.HasKey(s => s.Id);
builder.Property(s => s.OllamaUrl).HasMaxLength(500); builder.Property(s => s.OllamaUrl).HasMaxLength(500);
builder.Property(s => s.ModelName).HasMaxLength(100); builder.Property(s => s.ModelName).HasMaxLength(100);
builder.Property(s => s.SystemPrompt).HasMaxLength(4000);
builder.Property(s => s.UserPromptTemplate).HasMaxLength(4000);
builder.HasOne(s => s.User) builder.HasOne(s => s.User)
.WithOne(u => u.AiCategorizationSettings) .WithOne(u => u.AiCategorizationSettings)
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,40 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Purrse.Data.Migrations
{
/// <inheritdoc />
public partial class AddAiPromptTemplates : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "SystemPrompt",
table: "ai_categorization_settings",
type: "character varying(4000)",
maxLength: 4000,
nullable: true);
migrationBuilder.AddColumn<string>(
name: "UserPromptTemplate",
table: "ai_categorization_settings",
type: "character varying(4000)",
maxLength: 4000,
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "SystemPrompt",
table: "ai_categorization_settings");
migrationBuilder.DropColumn(
name: "UserPromptTemplate",
table: "ai_categorization_settings");
}
}
}
@@ -107,6 +107,10 @@ namespace Purrse.Data.Migrations
.HasMaxLength(500) .HasMaxLength(500)
.HasColumnType("character varying(500)"); .HasColumnType("character varying(500)");
b.Property<string>("SystemPrompt")
.HasMaxLength(4000)
.HasColumnType("character varying(4000)");
b.Property<int>("TimeoutSeconds") b.Property<int>("TimeoutSeconds")
.HasColumnType("integer"); .HasColumnType("integer");
@@ -119,6 +123,10 @@ namespace Purrse.Data.Migrations
b.Property<Guid>("UserId") b.Property<Guid>("UserId")
.HasColumnType("uuid"); .HasColumnType("uuid");
b.Property<string>("UserPromptTemplate")
.HasMaxLength(4000)
.HasColumnType("character varying(4000)");
b.HasKey("Id"); b.HasKey("Id");
b.HasIndex("UserId") b.HasIndex("UserId")