Add AI categorization with Ollama: settings, auto-classify on sync/import, and manual classify from Transactions
Adds local LLM-based transaction categorization using Ollama: - Settings UI to configure Ollama URL, model, confidence threshold - Auto-categorization during bank sync and file import - Manual "Classify Uncategorized" button on Transactions screen that respects active filters - Payee default category learning loop for classified transactions Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Catherine Renelle
@@ -0,0 +1,380 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Purrse.Core.DTOs;
|
||||
using Purrse.Core.Enums;
|
||||
using Purrse.Core.Interfaces.Services;
|
||||
using Purrse.Core.Models;
|
||||
using Purrse.Data;
|
||||
|
||||
namespace Purrse.Api.Services;
|
||||
|
||||
public class AiCategorizationService : IAiCategorizationService
|
||||
{
|
||||
private readonly PurrseDbContext _db;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly ILogger<AiCategorizationService> _logger;
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
public AiCategorizationService(
|
||||
PurrseDbContext db,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
ILogger<AiCategorizationService> logger)
|
||||
{
|
||||
_db = db;
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<AiCategorizationSettingsResponse> GetSettingsAsync(Guid userId)
|
||||
{
|
||||
var settings = await _db.AiCategorizationSettings.FirstOrDefaultAsync(s => s.UserId == userId);
|
||||
|
||||
if (settings == null)
|
||||
{
|
||||
return new AiCategorizationSettingsResponse(
|
||||
false, "http://localhost:11434", "llama3.1:8b", 0.7, 30, true);
|
||||
}
|
||||
|
||||
return new AiCategorizationSettingsResponse(
|
||||
settings.IsEnabled, settings.OllamaUrl, settings.ModelName,
|
||||
settings.ConfidenceThreshold, settings.TimeoutSeconds, settings.UpdatePayeeDefaults);
|
||||
}
|
||||
|
||||
public async Task<AiCategorizationSettingsResponse> UpdateSettingsAsync(Guid userId, UpdateAiCategorizationSettingsRequest request)
|
||||
{
|
||||
var settings = await _db.AiCategorizationSettings.FirstOrDefaultAsync(s => s.UserId == userId);
|
||||
|
||||
if (settings == null)
|
||||
{
|
||||
settings = new AiCategorizationSettings
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
UserId = userId
|
||||
};
|
||||
_db.AiCategorizationSettings.Add(settings);
|
||||
}
|
||||
|
||||
settings.IsEnabled = request.IsEnabled;
|
||||
settings.OllamaUrl = request.OllamaUrl;
|
||||
settings.ModelName = request.ModelName;
|
||||
settings.ConfidenceThreshold = request.ConfidenceThreshold;
|
||||
settings.TimeoutSeconds = request.TimeoutSeconds;
|
||||
settings.UpdatePayeeDefaults = request.UpdatePayeeDefaults;
|
||||
settings.UpdatedAt = DateTime.UtcNow;
|
||||
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
return new AiCategorizationSettingsResponse(
|
||||
settings.IsEnabled, settings.OllamaUrl, settings.ModelName,
|
||||
settings.ConfidenceThreshold, settings.TimeoutSeconds, settings.UpdatePayeeDefaults);
|
||||
}
|
||||
|
||||
public async Task<OllamaConnectionTestResponse> TestConnectionAsync(Guid userId)
|
||||
{
|
||||
var settings = await _db.AiCategorizationSettings.FirstOrDefaultAsync(s => s.UserId == userId);
|
||||
var ollamaUrl = settings?.OllamaUrl ?? "http://localhost:11434";
|
||||
|
||||
try
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient("Ollama");
|
||||
client.Timeout = TimeSpan.FromSeconds(10);
|
||||
|
||||
var response = await client.GetAsync($"{ollamaUrl.TrimEnd('/')}/api/tags");
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var json = await response.Content.ReadAsStringAsync();
|
||||
var tagsResponse = JsonSerializer.Deserialize<OllamaTagsResponse>(json, JsonOptions);
|
||||
|
||||
var modelNames = tagsResponse?.Models?.Select(m => m.Name).ToList() ?? new List<string>();
|
||||
|
||||
return new OllamaConnectionTestResponse(true, null, modelNames);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Ollama connection test failed for user {UserId}", userId);
|
||||
return new OllamaConnectionTestResponse(false, ex.Message, null);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<List<TransactionClassification>> ClassifyTransactionsAsync(Guid userId, List<Transaction> uncategorizedTransactions)
|
||||
{
|
||||
if (uncategorizedTransactions.Count == 0)
|
||||
return new List<TransactionClassification>();
|
||||
|
||||
try
|
||||
{
|
||||
var settings = await _db.AiCategorizationSettings.FirstOrDefaultAsync(s => s.UserId == userId);
|
||||
if (settings == null || !settings.IsEnabled)
|
||||
return new List<TransactionClassification>();
|
||||
|
||||
// Load all active categories with parent names
|
||||
var categories = await _db.Categories
|
||||
.Include(c => c.Parent)
|
||||
.Where(c => c.UserId == userId && c.IsActive)
|
||||
.ToListAsync();
|
||||
|
||||
if (categories.Count == 0)
|
||||
return new List<TransactionClassification>();
|
||||
|
||||
// Load 25 recent categorized transactions as few-shot examples
|
||||
var recentCategorized = await _db.Transactions
|
||||
.Include(t => t.Category).ThenInclude(c => c!.Parent)
|
||||
.Include(t => t.Account)
|
||||
.Where(t => t.Account.UserId == userId && t.CategoryId != null)
|
||||
.OrderByDescending(t => t.Date)
|
||||
.Take(25)
|
||||
.ToListAsync();
|
||||
|
||||
var allClassifications = new List<TransactionClassification>();
|
||||
|
||||
// Process in batches of 50
|
||||
for (int batchStart = 0; batchStart < uncategorizedTransactions.Count; batchStart += 50)
|
||||
{
|
||||
var batch = uncategorizedTransactions.Skip(batchStart).Take(50).ToList();
|
||||
var batchClassifications = await ClassifyBatchAsync(settings, categories, recentCategorized, batch, batchStart);
|
||||
allClassifications.AddRange(batchClassifications);
|
||||
}
|
||||
|
||||
return allClassifications;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "AI categorization failed for user {UserId}", userId);
|
||||
return new List<TransactionClassification>();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<List<TransactionClassification>> ClassifyBatchAsync(
|
||||
AiCategorizationSettings settings,
|
||||
List<Category> categories,
|
||||
List<Transaction> recentCategorized,
|
||||
List<Transaction> batch,
|
||||
int globalOffset)
|
||||
{
|
||||
var systemMessage = BuildSystemMessage(categories);
|
||||
var userMessage = BuildUserMessage(recentCategorized, batch);
|
||||
|
||||
var chatRequest = new OllamaChatRequest(
|
||||
settings.ModelName,
|
||||
new List<OllamaChatMessage>
|
||||
{
|
||||
new("system", systemMessage),
|
||||
new("user", userMessage)
|
||||
},
|
||||
false,
|
||||
"json");
|
||||
|
||||
try
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient("Ollama");
|
||||
client.Timeout = TimeSpan.FromSeconds(settings.TimeoutSeconds);
|
||||
|
||||
var requestJson = JsonSerializer.Serialize(chatRequest, JsonOptions);
|
||||
var content = new StringContent(requestJson, Encoding.UTF8, "application/json");
|
||||
|
||||
var response = await client.PostAsync($"{settings.OllamaUrl.TrimEnd('/')}/api/chat", content);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var responseJson = await response.Content.ReadAsStringAsync();
|
||||
var chatResponse = JsonSerializer.Deserialize<OllamaChatResponse>(responseJson, JsonOptions);
|
||||
|
||||
if (chatResponse?.Message?.Content == null)
|
||||
return new List<TransactionClassification>();
|
||||
|
||||
return ParseClassifications(chatResponse.Message.Content, categories, batch.Count, globalOffset, settings.ConfidenceThreshold);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Ollama API call failed for batch starting at {Offset}", globalOffset);
|
||||
return new List<TransactionClassification>();
|
||||
}
|
||||
}
|
||||
|
||||
private static string BuildSystemMessage(List<Category> categories)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
sb.AppendLine("You are a personal finance transaction categorizer. Classify each transaction into the most appropriate category.");
|
||||
sb.AppendLine("Respond with JSON: {\"classifications\": [{\"index\": 0, \"categoryId\": \"...\", \"categoryName\": \"...\", \"confidence\": 0.9}, ...]}");
|
||||
sb.AppendLine();
|
||||
sb.AppendLine("Available categories:");
|
||||
|
||||
foreach (var category in categories)
|
||||
{
|
||||
var parentInfo = category.Parent != null ? $", under \"{category.Parent.Name}\"" : "";
|
||||
sb.AppendLine($"- {category.Id} \"{category.Name}\" ({category.Type}{parentInfo})");
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static string BuildUserMessage(List<Transaction> recentCategorized, List<Transaction> batch)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
|
||||
if (recentCategorized.Count > 0)
|
||||
{
|
||||
sb.AppendLine("Recent categorized transactions (for reference):");
|
||||
for (int i = 0; i < recentCategorized.Count; i++)
|
||||
{
|
||||
var t = recentCategorized[i];
|
||||
var categoryName = t.Category?.Parent != null
|
||||
? $"{t.Category.Parent.Name} > {t.Category.Name}"
|
||||
: t.Category?.Name ?? "Unknown";
|
||||
sb.AppendLine($"{i + 1}. \"{t.PayeeName}\" (${t.Amount:F2}) -> \"{categoryName}\"");
|
||||
}
|
||||
sb.AppendLine();
|
||||
}
|
||||
|
||||
sb.AppendLine("Classify these uncategorized transactions:");
|
||||
for (int i = 0; i < batch.Count; i++)
|
||||
{
|
||||
var t = batch[i];
|
||||
var type = t.Amount >= 0 ? "credit" : "debit";
|
||||
sb.AppendLine($"{i}. \"{t.PayeeName}\" (${t.Amount:F2}, {type}, {t.Date:yyyy-MM-dd}, memo: \"{t.Memo}\")");
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static List<TransactionClassification> ParseClassifications(
|
||||
string responseContent,
|
||||
List<Category> categories,
|
||||
int batchSize,
|
||||
int globalOffset,
|
||||
double confidenceThreshold)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var doc = JsonDocument.Parse(responseContent);
|
||||
var root = doc.RootElement;
|
||||
|
||||
if (!root.TryGetProperty("classifications", out var classificationsArray))
|
||||
return new List<TransactionClassification>();
|
||||
|
||||
var categoryLookup = categories.ToDictionary(c => c.Id);
|
||||
var categoryNameLookup = categories
|
||||
.GroupBy(c => c.Name.ToLowerInvariant())
|
||||
.ToDictionary(g => g.Key, g => g.First());
|
||||
|
||||
var results = new List<TransactionClassification>();
|
||||
|
||||
foreach (var item in classificationsArray.EnumerateArray())
|
||||
{
|
||||
if (!item.TryGetProperty("index", out var indexProp))
|
||||
continue;
|
||||
|
||||
var localIndex = indexProp.GetInt32();
|
||||
if (localIndex < 0 || localIndex >= batchSize)
|
||||
continue;
|
||||
|
||||
var globalIndex = globalOffset + localIndex;
|
||||
|
||||
double confidence = 0;
|
||||
if (item.TryGetProperty("confidence", out var confProp))
|
||||
confidence = confProp.GetDouble();
|
||||
|
||||
if (confidence < confidenceThreshold)
|
||||
continue;
|
||||
|
||||
Guid? categoryId = null;
|
||||
string? categoryName = null;
|
||||
|
||||
// Try to resolve by categoryId first
|
||||
if (item.TryGetProperty("categoryId", out var catIdProp) && catIdProp.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
if (Guid.TryParse(catIdProp.GetString(), out var parsedId) && categoryLookup.ContainsKey(parsedId))
|
||||
{
|
||||
categoryId = parsedId;
|
||||
categoryName = categoryLookup[parsedId].Name;
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to name matching
|
||||
if (!categoryId.HasValue && item.TryGetProperty("categoryName", out var catNameProp) && catNameProp.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
var name = catNameProp.GetString()?.ToLowerInvariant();
|
||||
if (name != null && categoryNameLookup.TryGetValue(name, out var matched))
|
||||
{
|
||||
categoryId = matched.Id;
|
||||
categoryName = matched.Name;
|
||||
}
|
||||
}
|
||||
|
||||
if (categoryId.HasValue)
|
||||
{
|
||||
results.Add(new TransactionClassification(globalIndex, categoryId, categoryName, confidence));
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return new List<TransactionClassification>();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<ClassifyUncategorizedResponse> ClassifyUncategorizedAsync(Guid userId, ClassifyUncategorizedRequest request)
|
||||
{
|
||||
var settings = await _db.AiCategorizationSettings.FirstOrDefaultAsync(s => s.UserId == userId);
|
||||
if (settings == null || !settings.IsEnabled)
|
||||
return new ClassifyUncategorizedResponse(0, 0, 0);
|
||||
|
||||
var query = _db.Transactions
|
||||
.Include(t => t.Account)
|
||||
.Where(t => t.Account.UserId == userId && t.CategoryId == null);
|
||||
|
||||
if (request.AccountId.HasValue)
|
||||
query = query.Where(t => t.AccountId == request.AccountId.Value);
|
||||
if (request.StartDate.HasValue)
|
||||
query = query.Where(t => t.Date >= request.StartDate.Value);
|
||||
if (request.EndDate.HasValue)
|
||||
query = query.Where(t => t.Date <= request.EndDate.Value);
|
||||
if (request.Status.HasValue)
|
||||
query = query.Where(t => t.Status == request.Status.Value);
|
||||
if (!string.IsNullOrEmpty(request.SearchText))
|
||||
query = query.Where(t =>
|
||||
(t.PayeeName != null && t.PayeeName.Contains(request.SearchText)) ||
|
||||
(t.Memo != null && t.Memo.Contains(request.SearchText)) ||
|
||||
(t.ReferenceNumber != null && t.ReferenceNumber.Contains(request.SearchText)));
|
||||
|
||||
var uncategorized = await query.OrderByDescending(t => t.Date).ToListAsync();
|
||||
|
||||
if (uncategorized.Count == 0)
|
||||
return new ClassifyUncategorizedResponse(0, 0, 0);
|
||||
|
||||
var classifications = await ClassifyTransactionsAsync(userId, uncategorized);
|
||||
|
||||
int classified = 0;
|
||||
foreach (var c in classifications)
|
||||
{
|
||||
if (c.CategoryId.HasValue && c.TransactionIndex >= 0 && c.TransactionIndex < uncategorized.Count)
|
||||
{
|
||||
uncategorized[c.TransactionIndex].CategoryId = c.CategoryId.Value;
|
||||
classified++;
|
||||
|
||||
if (settings.UpdatePayeeDefaults)
|
||||
{
|
||||
var txn = uncategorized[c.TransactionIndex];
|
||||
if (txn.PayeeId.HasValue)
|
||||
{
|
||||
var payee = await _db.Payees.FindAsync(txn.PayeeId.Value);
|
||||
if (payee != null && payee.DefaultCategoryId == null)
|
||||
payee.DefaultCategoryId = c.CategoryId.Value;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
return new ClassifyUncategorizedResponse(uncategorized.Count, classified, uncategorized.Count - classified);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user