Private
Public Access
1
0

Implement Phase 2: end-to-end import system

Complete the import flow from file upload through duplicate resolution
to transaction creation. Previously, ResolveDuplicatesAsync was a stub
that only incremented counters, FileWatcherService detected files but
didn't process them, and ImportsView.vue ignored the API response.

Backend:
- Add ParsedDataJson column to ImportBatch to persist parsed data
  between upload and confirm steps
- Rewrite ImportService.ResolveDuplicatesAsync to deserialize parsed
  transactions and create/merge/skip based on user decisions, with
  payee matching, fingerprinting, and account balance updates
- Add GetBatchesAsync for import history
- Complete FileWatcherService: wait for file ready, resolve account by
  filename, call UploadAsync, send SignalR notification, archive file
- Add GET /api/imports/batches endpoint

Frontend:
- Create imports API service (frontend/src/services/imports.ts)
- Add ImportBatch, DuplicateResolution types
- Rewrite ImportsView.vue with 3-step Vuetify stepper wizard:
  upload, review with duplicate handling, completion summary,
  plus import history table

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Catherine Renelle
2026-02-08 11:16:39 -05:00
parent 0b6ff8faf1
commit 2ace39c544
12 changed files with 1844 additions and 39 deletions
@@ -32,9 +32,13 @@ public class ImportsController : ControllerBase
=> Ok(await _importService.ConfirmBatchAsync(UserId, batchId));
[HttpPost("batches/{batchId}/resolve-duplicates")]
public async Task<ActionResult> ResolveDuplicates(Guid batchId, ResolveDuplicatesRequest request)
public async Task<ActionResult<ImportBatchResponse>> ResolveDuplicates(Guid batchId, ResolveDuplicatesRequest request)
{
await _importService.ResolveDuplicatesAsync(UserId, batchId, request);
return NoContent();
var result = await _importService.ResolveDuplicatesAsync(UserId, batchId, request);
return Ok(result);
}
[HttpGet("batches")]
public async Task<ActionResult<List<ImportBatchResponse>>> GetBatches()
=> Ok(await _importService.GetBatchesAsync(UserId));
}
+124 -6
View File
@@ -1,16 +1,30 @@
using Microsoft.AspNetCore.SignalR;
using Microsoft.EntityFrameworkCore;
using Purrse.Api.Hubs;
using Purrse.Core.Interfaces.Services;
using Purrse.Data;
namespace Purrse.Api.Services;
public class FileWatcherService : BackgroundService
{
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<FileWatcherService> _logger;
private readonly IHubContext<NotificationHub> _hubContext;
private readonly string _importsPath;
private FileSystemWatcher? _watcher;
public FileWatcherService(IServiceScopeFactory scopeFactory, ILogger<FileWatcherService> logger, IConfiguration config)
private static readonly string[] SupportedExtensions = { ".ofx", ".qfx", ".csv", ".qif" };
public FileWatcherService(
IServiceScopeFactory scopeFactory,
ILogger<FileWatcherService> logger,
IConfiguration config,
IHubContext<NotificationHub> hubContext)
{
_scopeFactory = scopeFactory;
_logger = logger;
_hubContext = hubContext;
_importsPath = config["Imports:WatchPath"] ?? Path.Combine(AppContext.BaseDirectory, "imports");
}
@@ -19,10 +33,15 @@ public class FileWatcherService : BackgroundService
if (!Directory.Exists(_importsPath))
Directory.CreateDirectory(_importsPath);
var archivePath = Path.Combine(_importsPath, "archive");
if (!Directory.Exists(archivePath))
Directory.CreateDirectory(archivePath);
_watcher = new FileSystemWatcher(_importsPath)
{
NotifyFilter = NotifyFilters.FileName | NotifyFilters.CreationTime,
EnableRaisingEvents = true
EnableRaisingEvents = true,
IncludeSubdirectories = false
};
_watcher.Created += OnFileCreated;
@@ -38,10 +57,109 @@ public class FileWatcherService : BackgroundService
return Task.CompletedTask;
}
private void OnFileCreated(object sender, FileSystemEventArgs e)
private async void OnFileCreated(object sender, FileSystemEventArgs e)
{
_logger.LogInformation("New file detected: {FileName}", e.Name);
// Files dropped here can be processed by the import service
// A SignalR notification would be sent to connected clients
try
{
var extension = Path.GetExtension(e.FullPath).ToLowerInvariant();
if (!SupportedExtensions.Contains(extension))
{
_logger.LogDebug("Ignoring non-supported file: {FileName}", e.Name);
return;
}
_logger.LogInformation("New import file detected: {FileName}", e.Name);
// Wait for file to finish writing
await WaitForFileReady(e.FullPath);
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<PurrseDbContext>();
var importService = scope.ServiceProvider.GetRequiredService<IImportService>();
// Find the target account by matching filename to account name
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(e.Name);
var account = await db.Accounts
.Where(a => a.IsActive && !a.IsClosed)
.FirstOrDefaultAsync(a => a.Name.ToLower() == fileNameWithoutExt!.ToLower());
// Fall back to first active account
account ??= await db.Accounts
.Where(a => a.IsActive && !a.IsClosed)
.OrderBy(a => a.SortOrder)
.FirstOrDefaultAsync();
if (account == null)
{
_logger.LogWarning("No active account found for file import: {FileName}", e.Name);
return;
}
_logger.LogInformation("Importing {FileName} into account {AccountName} ({AccountId})",
e.Name, account.Name, account.Id);
await using var fileStream = new FileStream(e.FullPath, FileMode.Open, FileAccess.Read, FileShare.Read);
var result = await importService.UploadAsync(account.UserId, account.Id, e.Name!, fileStream);
_logger.LogInformation(
"File import complete: {FileName} - {Total} transactions ({New} new, {Duplicates} duplicates)",
e.Name, result.TotalTransactions, result.NewTransactions, result.PossibleDuplicates);
// Send SignalR notification to the account owner
await _hubContext.Clients.Group(account.UserId.ToString()).SendAsync("FileImportReady", new
{
batchId = result.BatchId,
fileName = e.Name,
accountId = account.Id,
accountName = account.Name,
totalTransactions = result.TotalTransactions,
newTransactions = result.NewTransactions,
possibleDuplicates = result.PossibleDuplicates
});
// Archive the processed file
ArchiveFile(e.FullPath);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error processing imported file: {FileName}", e.Name);
}
}
private static async Task WaitForFileReady(string filePath, int maxRetries = 30, int delayMs = 500)
{
for (int i = 0; i < maxRetries; i++)
{
try
{
using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.None);
return; // File is ready
}
catch (IOException)
{
await Task.Delay(delayMs);
}
}
throw new IOException($"File {filePath} was not ready after {maxRetries * delayMs}ms");
}
private void ArchiveFile(string filePath)
{
try
{
var archivePath = Path.Combine(_importsPath, "archive");
var timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss");
var fileName = Path.GetFileName(filePath);
var archiveFileName = $"{timestamp}_{fileName}";
var destination = Path.Combine(archivePath, archiveFileName);
File.Move(filePath, destination);
_logger.LogInformation("Archived import file to: {ArchivePath}", destination);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to archive file: {FilePath}", filePath);
}
}
}
+157 -13
View File
@@ -1,3 +1,4 @@
using System.Text.Json;
using Microsoft.EntityFrameworkCore;
using Purrse.Core.DTOs;
using Purrse.Core.Enums;
@@ -5,6 +6,7 @@ using Purrse.Core.Helpers;
using Purrse.Core.Interfaces.Services;
using Purrse.Core.Models;
using Purrse.Plugins.Abstractions;
using Purrse.Plugins.Abstractions.Models;
using Purrse.Data;
namespace Purrse.Api.Services;
@@ -46,7 +48,8 @@ public class ImportService : IImportService
FileName = fileName,
FileType = Path.GetExtension(fileName).TrimStart('.').ToUpperInvariant(),
TotalCount = parseResult.Transactions.Count,
Status = ImportStatus.Pending
Status = ImportStatus.Pending,
ParsedDataJson = JsonSerializer.Serialize(parseResult.Transactions)
};
var previews = new List<ImportedTransactionPreview>();
@@ -72,37 +75,178 @@ public class ImportService : IImportService
public async Task<ImportBatchResponse> ConfirmBatchAsync(Guid userId, Guid batchId)
{
var batch = await _db.ImportBatches.FirstOrDefaultAsync(b => b.Id == batchId && b.UserId == userId)
var batch = await _db.ImportBatches
.Include(b => b.Account)
.FirstOrDefaultAsync(b => b.Id == batchId && b.UserId == userId)
?? throw new KeyNotFoundException("Import batch not found");
batch.Status = ImportStatus.Confirmed;
await _db.SaveChangesAsync();
return new ImportBatchResponse(batch.Id, batch.FileName, batch.FileType, batch.TotalCount,
batch.ImportedCount, batch.DuplicateCount, batch.SkippedCount, batch.Status.ToString(), batch.ImportedAt);
return MapBatchResponse(batch);
}
public async Task ResolveDuplicatesAsync(Guid userId, Guid batchId, ResolveDuplicatesRequest request)
public async Task<ImportBatchResponse> ResolveDuplicatesAsync(Guid userId, Guid batchId, ResolveDuplicatesRequest request)
{
var batch = await _db.ImportBatches.FirstOrDefaultAsync(b => b.Id == batchId && b.UserId == userId)
var batch = await _db.ImportBatches
.Include(b => b.Account)
.FirstOrDefaultAsync(b => b.Id == batchId && b.UserId == userId)
?? throw new KeyNotFoundException("Import batch not found");
foreach (var resolution in request.Resolutions)
if (string.IsNullOrEmpty(batch.ParsedDataJson))
throw new InvalidOperationException("No parsed data available for this batch. It may have already been processed.");
var parsedTransactions = JsonSerializer.Deserialize<List<ParsedTransaction>>(batch.ParsedDataJson)
?? throw new InvalidOperationException("Failed to deserialize parsed data");
var resolutionMap = request.Resolutions.ToDictionary(r => r.Index, r => r.Action);
int importedCount = 0;
int skippedCount = 0;
int mergedCount = 0;
for (int i = 0; i < parsedTransactions.Count; i++)
{
switch (resolution.Action)
var parsed = parsedTransactions[i];
var action = resolutionMap.GetValueOrDefault(i, DuplicateAction.Import);
switch (action)
{
case DuplicateAction.Skip:
batch.SkippedCount++;
break;
case DuplicateAction.Import:
batch.ImportedCount++;
await CreateTransaction(userId, batch, parsed);
importedCount++;
break;
case DuplicateAction.Skip:
skippedCount++;
break;
case DuplicateAction.Merge:
batch.DuplicateCount++;
var merged = await MergeTransaction(batch.AccountId, parsed);
if (merged)
mergedCount++;
else
{
// If no existing transaction found to merge with, import instead
await CreateTransaction(userId, batch, parsed);
importedCount++;
}
break;
}
}
batch.ImportedCount = importedCount;
batch.SkippedCount = skippedCount;
batch.DuplicateCount = mergedCount;
batch.ParsedDataJson = null;
batch.Status = ImportStatus.Confirmed;
await _db.SaveChangesAsync();
return MapBatchResponse(batch);
}
public async Task<List<ImportBatchResponse>> GetBatchesAsync(Guid userId)
{
return await _db.ImportBatches
.Include(b => b.Account)
.Where(b => b.UserId == userId)
.OrderByDescending(b => b.ImportedAt)
.Select(b => new ImportBatchResponse(
b.Id,
b.AccountId,
b.Account.Name,
b.FileName,
b.FileType,
b.TotalCount,
b.ImportedCount,
b.DuplicateCount,
b.SkippedCount,
0,
b.Status.ToString(),
b.ImportedAt))
.ToListAsync();
}
private async Task CreateTransaction(Guid userId, ImportBatch batch, ParsedTransaction parsed)
{
var fingerprint = TransactionFingerprint.Generate(batch.AccountId, parsed.Date, parsed.Amount, parsed.FitId);
var matchedPayee = await _payeeService.MatchPayeeAsync(userId, parsed.PayeeName ?? string.Empty);
var transaction = new Transaction
{
Id = Guid.NewGuid(),
AccountId = batch.AccountId,
Date = parsed.Date,
Amount = parsed.Amount,
PayeeId = matchedPayee?.Id,
PayeeName = parsed.PayeeName,
CategoryId = matchedPayee?.DefaultCategoryId,
Memo = parsed.Memo,
ReferenceNumber = parsed.ReferenceNumber,
CheckNumber = parsed.CheckNumber,
FitId = parsed.FitId,
Fingerprint = fingerprint,
Type = parsed.Amount >= 0 ? TransactionType.Credit : TransactionType.Debit,
Status = TransactionStatus.Uncleared,
ImportBatchId = batch.Id
};
_db.Transactions.Add(transaction);
// Update account balance
batch.Account.Balance += parsed.Amount;
batch.Account.UpdatedAt = DateTime.UtcNow;
}
private async Task<bool> MergeTransaction(Guid accountId, ParsedTransaction parsed)
{
var duplicate = await _duplicateService.FindDuplicateAsync(
accountId, parsed.Date, parsed.Amount, parsed.PayeeName, parsed.FitId, parsed.CheckNumber);
if (duplicate == null)
return false;
var existing = await _db.Transactions.FindAsync(duplicate.ExistingTransactionId);
if (existing == null)
return false;
// Fill in blank fields without overwriting user-edited data
if (string.IsNullOrEmpty(existing.Memo) && !string.IsNullOrEmpty(parsed.Memo))
existing.Memo = parsed.Memo;
if (string.IsNullOrEmpty(existing.FitId) && !string.IsNullOrEmpty(parsed.FitId))
existing.FitId = parsed.FitId;
if (string.IsNullOrEmpty(existing.CheckNumber) && !string.IsNullOrEmpty(parsed.CheckNumber))
existing.CheckNumber = parsed.CheckNumber;
if (string.IsNullOrEmpty(existing.ReferenceNumber) && !string.IsNullOrEmpty(parsed.ReferenceNumber))
existing.ReferenceNumber = parsed.ReferenceNumber;
if (string.IsNullOrEmpty(existing.Fingerprint))
existing.Fingerprint = TransactionFingerprint.Generate(accountId, parsed.Date, parsed.Amount, parsed.FitId);
existing.UpdatedAt = DateTime.UtcNow;
return true;
}
private static ImportBatchResponse MapBatchResponse(ImportBatch batch)
{
return new ImportBatchResponse(
batch.Id,
batch.AccountId,
batch.Account?.Name ?? string.Empty,
batch.FileName,
batch.FileType,
batch.TotalCount,
batch.ImportedCount,
batch.DuplicateCount,
batch.SkippedCount,
batch.DuplicateCount,
batch.Status.ToString(),
batch.ImportedAt);
}
}
@@ -6,17 +6,21 @@ public interface IImportService
{
Task<ImportUploadResponse> UploadAsync(Guid userId, Guid accountId, string fileName, Stream fileStream);
Task<ImportBatchResponse> ConfirmBatchAsync(Guid userId, Guid batchId);
Task ResolveDuplicatesAsync(Guid userId, Guid batchId, ResolveDuplicatesRequest request);
Task<ImportBatchResponse> ResolveDuplicatesAsync(Guid userId, Guid batchId, ResolveDuplicatesRequest request);
Task<List<ImportBatchResponse>> GetBatchesAsync(Guid userId);
}
public record ImportBatchResponse(
Guid Id,
Guid AccountId,
string AccountName,
string FileName,
string FileType,
int TotalCount,
int ImportedCount,
int DuplicateCount,
int SkippedCount,
int MergedCount,
string Status,
DateTime ImportedAt
);
+1
View File
@@ -14,6 +14,7 @@ public class ImportBatch
public int DuplicateCount { get; set; }
public int SkippedCount { get; set; }
public ImportStatus Status { get; set; } = ImportStatus.Pending;
public string? ParsedDataJson { get; set; }
public DateTime ImportedAt { get; set; } = DateTime.UtcNow;
public User User { get; set; } = null!;
@@ -12,6 +12,7 @@ public class ImportBatchConfiguration : IEntityTypeConfiguration<ImportBatch>
builder.HasKey(b => b.Id);
builder.Property(b => b.FileName).HasMaxLength(500).IsRequired();
builder.Property(b => b.FileType).HasMaxLength(50).IsRequired();
builder.Property(b => b.ParsedDataJson).HasColumnType("text");
builder.HasOne(b => b.User)
.WithMany()
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,28 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Purrse.Data.Migrations
{
/// <inheritdoc />
public partial class AddParsedDataToImportBatch : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "ParsedDataJson",
table: "import_batches",
type: "text",
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "ParsedDataJson",
table: "import_batches");
}
}
}
@@ -249,6 +249,9 @@ namespace Purrse.Data.Migrations
b.Property<int>("ImportedCount")
.HasColumnType("integer");
b.Property<string>("ParsedDataJson")
.HasColumnType("text");
b.Property<int>("SkippedCount")
.HasColumnType("integer");