Implement Phase 2: end-to-end import system
Complete the import flow from file upload through duplicate resolution to transaction creation. Previously, ResolveDuplicatesAsync was a stub that only incremented counters, FileWatcherService detected files but didn't process them, and ImportsView.vue ignored the API response. Backend: - Add ParsedDataJson column to ImportBatch to persist parsed data between upload and confirm steps - Rewrite ImportService.ResolveDuplicatesAsync to deserialize parsed transactions and create/merge/skip based on user decisions, with payee matching, fingerprinting, and account balance updates - Add GetBatchesAsync for import history - Complete FileWatcherService: wait for file ready, resolve account by filename, call UploadAsync, send SignalR notification, archive file - Add GET /api/imports/batches endpoint Frontend: - Create imports API service (frontend/src/services/imports.ts) - Add ImportBatch, DuplicateResolution types - Rewrite ImportsView.vue with 3-step Vuetify stepper wizard: upload, review with duplicate handling, completion summary, plus import history table Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -32,9 +32,13 @@ public class ImportsController : ControllerBase
|
||||
=> Ok(await _importService.ConfirmBatchAsync(UserId, batchId));
|
||||
|
||||
[HttpPost("batches/{batchId}/resolve-duplicates")]
|
||||
public async Task<ActionResult> ResolveDuplicates(Guid batchId, ResolveDuplicatesRequest request)
|
||||
public async Task<ActionResult<ImportBatchResponse>> ResolveDuplicates(Guid batchId, ResolveDuplicatesRequest request)
|
||||
{
|
||||
await _importService.ResolveDuplicatesAsync(UserId, batchId, request);
|
||||
return NoContent();
|
||||
var result = await _importService.ResolveDuplicatesAsync(UserId, batchId, request);
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
[HttpGet("batches")]
|
||||
public async Task<ActionResult<List<ImportBatchResponse>>> GetBatches()
|
||||
=> Ok(await _importService.GetBatchesAsync(UserId));
|
||||
}
|
||||
|
||||
@@ -1,16 +1,30 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Purrse.Api.Hubs;
|
||||
using Purrse.Core.Interfaces.Services;
|
||||
using Purrse.Data;
|
||||
|
||||
namespace Purrse.Api.Services;
|
||||
|
||||
public class FileWatcherService : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly ILogger<FileWatcherService> _logger;
|
||||
private readonly IHubContext<NotificationHub> _hubContext;
|
||||
private readonly string _importsPath;
|
||||
private FileSystemWatcher? _watcher;
|
||||
|
||||
public FileWatcherService(IServiceScopeFactory scopeFactory, ILogger<FileWatcherService> logger, IConfiguration config)
|
||||
private static readonly string[] SupportedExtensions = { ".ofx", ".qfx", ".csv", ".qif" };
|
||||
|
||||
public FileWatcherService(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
ILogger<FileWatcherService> logger,
|
||||
IConfiguration config,
|
||||
IHubContext<NotificationHub> hubContext)
|
||||
{
|
||||
_scopeFactory = scopeFactory;
|
||||
_logger = logger;
|
||||
_hubContext = hubContext;
|
||||
_importsPath = config["Imports:WatchPath"] ?? Path.Combine(AppContext.BaseDirectory, "imports");
|
||||
}
|
||||
|
||||
@@ -19,10 +33,15 @@ public class FileWatcherService : BackgroundService
|
||||
if (!Directory.Exists(_importsPath))
|
||||
Directory.CreateDirectory(_importsPath);
|
||||
|
||||
var archivePath = Path.Combine(_importsPath, "archive");
|
||||
if (!Directory.Exists(archivePath))
|
||||
Directory.CreateDirectory(archivePath);
|
||||
|
||||
_watcher = new FileSystemWatcher(_importsPath)
|
||||
{
|
||||
NotifyFilter = NotifyFilters.FileName | NotifyFilters.CreationTime,
|
||||
EnableRaisingEvents = true
|
||||
EnableRaisingEvents = true,
|
||||
IncludeSubdirectories = false
|
||||
};
|
||||
|
||||
_watcher.Created += OnFileCreated;
|
||||
@@ -38,10 +57,109 @@ public class FileWatcherService : BackgroundService
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private void OnFileCreated(object sender, FileSystemEventArgs e)
|
||||
private async void OnFileCreated(object sender, FileSystemEventArgs e)
|
||||
{
|
||||
_logger.LogInformation("New file detected: {FileName}", e.Name);
|
||||
// Files dropped here can be processed by the import service
|
||||
// A SignalR notification would be sent to connected clients
|
||||
try
|
||||
{
|
||||
var extension = Path.GetExtension(e.FullPath).ToLowerInvariant();
|
||||
if (!SupportedExtensions.Contains(extension))
|
||||
{
|
||||
_logger.LogDebug("Ignoring non-supported file: {FileName}", e.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("New import file detected: {FileName}", e.Name);
|
||||
|
||||
// Wait for file to finish writing
|
||||
await WaitForFileReady(e.FullPath);
|
||||
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<PurrseDbContext>();
|
||||
var importService = scope.ServiceProvider.GetRequiredService<IImportService>();
|
||||
|
||||
// Find the target account by matching filename to account name
|
||||
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(e.Name);
|
||||
var account = await db.Accounts
|
||||
.Where(a => a.IsActive && !a.IsClosed)
|
||||
.FirstOrDefaultAsync(a => a.Name.ToLower() == fileNameWithoutExt!.ToLower());
|
||||
|
||||
// Fall back to first active account
|
||||
account ??= await db.Accounts
|
||||
.Where(a => a.IsActive && !a.IsClosed)
|
||||
.OrderBy(a => a.SortOrder)
|
||||
.FirstOrDefaultAsync();
|
||||
|
||||
if (account == null)
|
||||
{
|
||||
_logger.LogWarning("No active account found for file import: {FileName}", e.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Importing {FileName} into account {AccountName} ({AccountId})",
|
||||
e.Name, account.Name, account.Id);
|
||||
|
||||
await using var fileStream = new FileStream(e.FullPath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
var result = await importService.UploadAsync(account.UserId, account.Id, e.Name!, fileStream);
|
||||
|
||||
_logger.LogInformation(
|
||||
"File import complete: {FileName} - {Total} transactions ({New} new, {Duplicates} duplicates)",
|
||||
e.Name, result.TotalTransactions, result.NewTransactions, result.PossibleDuplicates);
|
||||
|
||||
// Send SignalR notification to the account owner
|
||||
await _hubContext.Clients.Group(account.UserId.ToString()).SendAsync("FileImportReady", new
|
||||
{
|
||||
batchId = result.BatchId,
|
||||
fileName = e.Name,
|
||||
accountId = account.Id,
|
||||
accountName = account.Name,
|
||||
totalTransactions = result.TotalTransactions,
|
||||
newTransactions = result.NewTransactions,
|
||||
possibleDuplicates = result.PossibleDuplicates
|
||||
});
|
||||
|
||||
// Archive the processed file
|
||||
ArchiveFile(e.FullPath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error processing imported file: {FileName}", e.Name);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task WaitForFileReady(string filePath, int maxRetries = 30, int delayMs = 500)
|
||||
{
|
||||
for (int i = 0; i < maxRetries; i++)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.None);
|
||||
return; // File is ready
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
await Task.Delay(delayMs);
|
||||
}
|
||||
}
|
||||
|
||||
throw new IOException($"File {filePath} was not ready after {maxRetries * delayMs}ms");
|
||||
}
|
||||
|
||||
private void ArchiveFile(string filePath)
|
||||
{
|
||||
try
|
||||
{
|
||||
var archivePath = Path.Combine(_importsPath, "archive");
|
||||
var timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss");
|
||||
var fileName = Path.GetFileName(filePath);
|
||||
var archiveFileName = $"{timestamp}_{fileName}";
|
||||
var destination = Path.Combine(archivePath, archiveFileName);
|
||||
|
||||
File.Move(filePath, destination);
|
||||
_logger.LogInformation("Archived import file to: {ArchivePath}", destination);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to archive file: {FilePath}", filePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Purrse.Core.DTOs;
|
||||
using Purrse.Core.Enums;
|
||||
@@ -5,6 +6,7 @@ using Purrse.Core.Helpers;
|
||||
using Purrse.Core.Interfaces.Services;
|
||||
using Purrse.Core.Models;
|
||||
using Purrse.Plugins.Abstractions;
|
||||
using Purrse.Plugins.Abstractions.Models;
|
||||
using Purrse.Data;
|
||||
|
||||
namespace Purrse.Api.Services;
|
||||
@@ -46,7 +48,8 @@ public class ImportService : IImportService
|
||||
FileName = fileName,
|
||||
FileType = Path.GetExtension(fileName).TrimStart('.').ToUpperInvariant(),
|
||||
TotalCount = parseResult.Transactions.Count,
|
||||
Status = ImportStatus.Pending
|
||||
Status = ImportStatus.Pending,
|
||||
ParsedDataJson = JsonSerializer.Serialize(parseResult.Transactions)
|
||||
};
|
||||
|
||||
var previews = new List<ImportedTransactionPreview>();
|
||||
@@ -72,37 +75,178 @@ public class ImportService : IImportService
|
||||
|
||||
public async Task<ImportBatchResponse> ConfirmBatchAsync(Guid userId, Guid batchId)
|
||||
{
|
||||
var batch = await _db.ImportBatches.FirstOrDefaultAsync(b => b.Id == batchId && b.UserId == userId)
|
||||
var batch = await _db.ImportBatches
|
||||
.Include(b => b.Account)
|
||||
.FirstOrDefaultAsync(b => b.Id == batchId && b.UserId == userId)
|
||||
?? throw new KeyNotFoundException("Import batch not found");
|
||||
|
||||
batch.Status = ImportStatus.Confirmed;
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
return new ImportBatchResponse(batch.Id, batch.FileName, batch.FileType, batch.TotalCount,
|
||||
batch.ImportedCount, batch.DuplicateCount, batch.SkippedCount, batch.Status.ToString(), batch.ImportedAt);
|
||||
return MapBatchResponse(batch);
|
||||
}
|
||||
|
||||
public async Task ResolveDuplicatesAsync(Guid userId, Guid batchId, ResolveDuplicatesRequest request)
|
||||
public async Task<ImportBatchResponse> ResolveDuplicatesAsync(Guid userId, Guid batchId, ResolveDuplicatesRequest request)
|
||||
{
|
||||
var batch = await _db.ImportBatches.FirstOrDefaultAsync(b => b.Id == batchId && b.UserId == userId)
|
||||
var batch = await _db.ImportBatches
|
||||
.Include(b => b.Account)
|
||||
.FirstOrDefaultAsync(b => b.Id == batchId && b.UserId == userId)
|
||||
?? throw new KeyNotFoundException("Import batch not found");
|
||||
|
||||
foreach (var resolution in request.Resolutions)
|
||||
if (string.IsNullOrEmpty(batch.ParsedDataJson))
|
||||
throw new InvalidOperationException("No parsed data available for this batch. It may have already been processed.");
|
||||
|
||||
var parsedTransactions = JsonSerializer.Deserialize<List<ParsedTransaction>>(batch.ParsedDataJson)
|
||||
?? throw new InvalidOperationException("Failed to deserialize parsed data");
|
||||
|
||||
var resolutionMap = request.Resolutions.ToDictionary(r => r.Index, r => r.Action);
|
||||
|
||||
int importedCount = 0;
|
||||
int skippedCount = 0;
|
||||
int mergedCount = 0;
|
||||
|
||||
for (int i = 0; i < parsedTransactions.Count; i++)
|
||||
{
|
||||
switch (resolution.Action)
|
||||
var parsed = parsedTransactions[i];
|
||||
var action = resolutionMap.GetValueOrDefault(i, DuplicateAction.Import);
|
||||
|
||||
switch (action)
|
||||
{
|
||||
case DuplicateAction.Skip:
|
||||
batch.SkippedCount++;
|
||||
break;
|
||||
case DuplicateAction.Import:
|
||||
batch.ImportedCount++;
|
||||
await CreateTransaction(userId, batch, parsed);
|
||||
importedCount++;
|
||||
break;
|
||||
|
||||
case DuplicateAction.Skip:
|
||||
skippedCount++;
|
||||
break;
|
||||
|
||||
case DuplicateAction.Merge:
|
||||
batch.DuplicateCount++;
|
||||
var merged = await MergeTransaction(batch.AccountId, parsed);
|
||||
if (merged)
|
||||
mergedCount++;
|
||||
else
|
||||
{
|
||||
// If no existing transaction found to merge with, import instead
|
||||
await CreateTransaction(userId, batch, parsed);
|
||||
importedCount++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
batch.ImportedCount = importedCount;
|
||||
batch.SkippedCount = skippedCount;
|
||||
batch.DuplicateCount = mergedCount;
|
||||
batch.ParsedDataJson = null;
|
||||
batch.Status = ImportStatus.Confirmed;
|
||||
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
return MapBatchResponse(batch);
|
||||
}
|
||||
|
||||
public async Task<List<ImportBatchResponse>> GetBatchesAsync(Guid userId)
|
||||
{
|
||||
return await _db.ImportBatches
|
||||
.Include(b => b.Account)
|
||||
.Where(b => b.UserId == userId)
|
||||
.OrderByDescending(b => b.ImportedAt)
|
||||
.Select(b => new ImportBatchResponse(
|
||||
b.Id,
|
||||
b.AccountId,
|
||||
b.Account.Name,
|
||||
b.FileName,
|
||||
b.FileType,
|
||||
b.TotalCount,
|
||||
b.ImportedCount,
|
||||
b.DuplicateCount,
|
||||
b.SkippedCount,
|
||||
0,
|
||||
b.Status.ToString(),
|
||||
b.ImportedAt))
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
private async Task CreateTransaction(Guid userId, ImportBatch batch, ParsedTransaction parsed)
|
||||
{
|
||||
var fingerprint = TransactionFingerprint.Generate(batch.AccountId, parsed.Date, parsed.Amount, parsed.FitId);
|
||||
|
||||
var matchedPayee = await _payeeService.MatchPayeeAsync(userId, parsed.PayeeName ?? string.Empty);
|
||||
|
||||
var transaction = new Transaction
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AccountId = batch.AccountId,
|
||||
Date = parsed.Date,
|
||||
Amount = parsed.Amount,
|
||||
PayeeId = matchedPayee?.Id,
|
||||
PayeeName = parsed.PayeeName,
|
||||
CategoryId = matchedPayee?.DefaultCategoryId,
|
||||
Memo = parsed.Memo,
|
||||
ReferenceNumber = parsed.ReferenceNumber,
|
||||
CheckNumber = parsed.CheckNumber,
|
||||
FitId = parsed.FitId,
|
||||
Fingerprint = fingerprint,
|
||||
Type = parsed.Amount >= 0 ? TransactionType.Credit : TransactionType.Debit,
|
||||
Status = TransactionStatus.Uncleared,
|
||||
ImportBatchId = batch.Id
|
||||
};
|
||||
|
||||
_db.Transactions.Add(transaction);
|
||||
|
||||
// Update account balance
|
||||
batch.Account.Balance += parsed.Amount;
|
||||
batch.Account.UpdatedAt = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
private async Task<bool> MergeTransaction(Guid accountId, ParsedTransaction parsed)
|
||||
{
|
||||
var duplicate = await _duplicateService.FindDuplicateAsync(
|
||||
accountId, parsed.Date, parsed.Amount, parsed.PayeeName, parsed.FitId, parsed.CheckNumber);
|
||||
|
||||
if (duplicate == null)
|
||||
return false;
|
||||
|
||||
var existing = await _db.Transactions.FindAsync(duplicate.ExistingTransactionId);
|
||||
if (existing == null)
|
||||
return false;
|
||||
|
||||
// Fill in blank fields without overwriting user-edited data
|
||||
if (string.IsNullOrEmpty(existing.Memo) && !string.IsNullOrEmpty(parsed.Memo))
|
||||
existing.Memo = parsed.Memo;
|
||||
|
||||
if (string.IsNullOrEmpty(existing.FitId) && !string.IsNullOrEmpty(parsed.FitId))
|
||||
existing.FitId = parsed.FitId;
|
||||
|
||||
if (string.IsNullOrEmpty(existing.CheckNumber) && !string.IsNullOrEmpty(parsed.CheckNumber))
|
||||
existing.CheckNumber = parsed.CheckNumber;
|
||||
|
||||
if (string.IsNullOrEmpty(existing.ReferenceNumber) && !string.IsNullOrEmpty(parsed.ReferenceNumber))
|
||||
existing.ReferenceNumber = parsed.ReferenceNumber;
|
||||
|
||||
if (string.IsNullOrEmpty(existing.Fingerprint))
|
||||
existing.Fingerprint = TransactionFingerprint.Generate(accountId, parsed.Date, parsed.Amount, parsed.FitId);
|
||||
|
||||
existing.UpdatedAt = DateTime.UtcNow;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static ImportBatchResponse MapBatchResponse(ImportBatch batch)
|
||||
{
|
||||
return new ImportBatchResponse(
|
||||
batch.Id,
|
||||
batch.AccountId,
|
||||
batch.Account?.Name ?? string.Empty,
|
||||
batch.FileName,
|
||||
batch.FileType,
|
||||
batch.TotalCount,
|
||||
batch.ImportedCount,
|
||||
batch.DuplicateCount,
|
||||
batch.SkippedCount,
|
||||
batch.DuplicateCount,
|
||||
batch.Status.ToString(),
|
||||
batch.ImportedAt);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,17 +6,21 @@ public interface IImportService
|
||||
{
|
||||
Task<ImportUploadResponse> UploadAsync(Guid userId, Guid accountId, string fileName, Stream fileStream);
|
||||
Task<ImportBatchResponse> ConfirmBatchAsync(Guid userId, Guid batchId);
|
||||
Task ResolveDuplicatesAsync(Guid userId, Guid batchId, ResolveDuplicatesRequest request);
|
||||
Task<ImportBatchResponse> ResolveDuplicatesAsync(Guid userId, Guid batchId, ResolveDuplicatesRequest request);
|
||||
Task<List<ImportBatchResponse>> GetBatchesAsync(Guid userId);
|
||||
}
|
||||
|
||||
public record ImportBatchResponse(
|
||||
Guid Id,
|
||||
Guid AccountId,
|
||||
string AccountName,
|
||||
string FileName,
|
||||
string FileType,
|
||||
int TotalCount,
|
||||
int ImportedCount,
|
||||
int DuplicateCount,
|
||||
int SkippedCount,
|
||||
int MergedCount,
|
||||
string Status,
|
||||
DateTime ImportedAt
|
||||
);
|
||||
|
||||
@@ -14,6 +14,7 @@ public class ImportBatch
|
||||
public int DuplicateCount { get; set; }
|
||||
public int SkippedCount { get; set; }
|
||||
public ImportStatus Status { get; set; } = ImportStatus.Pending;
|
||||
public string? ParsedDataJson { get; set; }
|
||||
public DateTime ImportedAt { get; set; } = DateTime.UtcNow;
|
||||
|
||||
public User User { get; set; } = null!;
|
||||
|
||||
@@ -12,6 +12,7 @@ public class ImportBatchConfiguration : IEntityTypeConfiguration<ImportBatch>
|
||||
builder.HasKey(b => b.Id);
|
||||
builder.Property(b => b.FileName).HasMaxLength(500).IsRequired();
|
||||
builder.Property(b => b.FileType).HasMaxLength(50).IsRequired();
|
||||
builder.Property(b => b.ParsedDataJson).HasColumnType("text");
|
||||
|
||||
builder.HasOne(b => b.User)
|
||||
.WithMany()
|
||||
|
||||
+1149
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,28 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace Purrse.Data.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class AddParsedDataToImportBatch : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.AddColumn<string>(
|
||||
name: "ParsedDataJson",
|
||||
table: "import_batches",
|
||||
type: "text",
|
||||
nullable: true);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropColumn(
|
||||
name: "ParsedDataJson",
|
||||
table: "import_batches");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -249,6 +249,9 @@ namespace Purrse.Data.Migrations
|
||||
b.Property<int>("ImportedCount")
|
||||
.HasColumnType("integer");
|
||||
|
||||
b.Property<string>("ParsedDataJson")
|
||||
.HasColumnType("text");
|
||||
|
||||
b.Property<int>("SkippedCount")
|
||||
.HasColumnType("integer");
|
||||
|
||||
|
||||
Reference in New Issue
Block a user