Implement Phase 2: end-to-end import system
Complete the import flow from file upload through duplicate resolution to transaction creation. Previously, ResolveDuplicatesAsync was a stub that only incremented counters, FileWatcherService detected files but didn't process them, and ImportsView.vue ignored the API response. Backend: - Add ParsedDataJson column to ImportBatch to persist parsed data between upload and confirm steps - Rewrite ImportService.ResolveDuplicatesAsync to deserialize parsed transactions and create/merge/skip based on user decisions, with payee matching, fingerprinting, and account balance updates - Add GetBatchesAsync for import history - Complete FileWatcherService: wait for file ready, resolve account by filename, call UploadAsync, send SignalR notification, archive file - Add GET /api/imports/batches endpoint Frontend: - Create imports API service (frontend/src/services/imports.ts) - Add ImportBatch, DuplicateResolution types - Rewrite ImportsView.vue with 3-step Vuetify stepper wizard: upload, review with duplicate handling, completion summary, plus import history table Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
import api from './api'
|
||||
import type { ImportUploadResponse, ImportBatch, ResolveDuplicatesRequest } from '@/types'
|
||||
|
||||
export const importsApi = {
|
||||
upload: (accountId: string, file: File) => {
|
||||
const formData = new FormData()
|
||||
formData.append('file', file)
|
||||
return api.post<ImportUploadResponse>(`/imports/upload?accountId=${accountId}`, formData, {
|
||||
headers: { 'Content-Type': 'multipart/form-data' }
|
||||
})
|
||||
},
|
||||
|
||||
confirmBatch: (batchId: string) =>
|
||||
api.post<ImportBatch>(`/imports/batches/${batchId}/confirm`),
|
||||
|
||||
resolveDuplicates: (batchId: string, request: ResolveDuplicatesRequest) =>
|
||||
api.post<ImportBatch>(`/imports/batches/${batchId}/resolve-duplicates`, request),
|
||||
|
||||
getBatches: () =>
|
||||
api.get<ImportBatch[]>('/imports/batches'),
|
||||
}
|
||||
@@ -235,3 +235,29 @@ export interface DuplicateMatch {
|
||||
confidence: number
|
||||
matchReason: string
|
||||
}
|
||||
|
||||
export interface ImportBatch {
|
||||
id: string
|
||||
accountId: string
|
||||
accountName: string
|
||||
fileName: string
|
||||
fileType: string
|
||||
totalCount: number
|
||||
importedCount: number
|
||||
duplicateCount: number
|
||||
skippedCount: number
|
||||
mergedCount: number
|
||||
status: string
|
||||
importedAt: string
|
||||
}
|
||||
|
||||
export type DuplicateAction = 'Import' | 'Skip' | 'Merge'
|
||||
|
||||
export interface DuplicateResolution {
|
||||
index: number
|
||||
action: DuplicateAction
|
||||
}
|
||||
|
||||
export interface ResolveDuplicatesRequest {
|
||||
resolutions: DuplicateResolution[]
|
||||
}
|
||||
|
||||
@@ -1,42 +1,348 @@
|
||||
<template>
|
||||
<div>
|
||||
<h1 class="text-h4 mb-4">Import Transactions</h1>
|
||||
<v-card>
|
||||
|
||||
<v-card class="mb-6">
|
||||
<v-stepper v-model="step" :items="['Upload', 'Review', 'Complete']" alt-labels>
|
||||
<template v-slot:item.1>
|
||||
<v-card flat>
|
||||
<v-card-text>
|
||||
<v-file-input v-model="file" label="Select file to import" accept=".ofx,.qfx,.csv,.qif" prepend-icon="mdi-upload" />
|
||||
<v-select v-model="selectedAccount" label="Target Account" :items="accounts" item-title="name" item-value="id" />
|
||||
<v-btn color="primary" :disabled="!file || !selectedAccount" @click="uploadFile" :loading="uploading">Import</v-btn>
|
||||
<v-row>
|
||||
<v-col cols="12" md="6">
|
||||
<v-select
|
||||
v-model="selectedAccount"
|
||||
label="Target Account"
|
||||
:items="accounts"
|
||||
item-title="name"
|
||||
item-value="id"
|
||||
:rules="[v => !!v || 'Account is required']"
|
||||
/>
|
||||
</v-col>
|
||||
<v-col cols="12" md="6">
|
||||
<v-file-input
|
||||
v-model="files"
|
||||
label="Select file to import"
|
||||
accept=".ofx,.qfx,.csv,.qif"
|
||||
prepend-icon="mdi-file-upload"
|
||||
show-size
|
||||
/>
|
||||
</v-col>
|
||||
</v-row>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn
|
||||
color="primary"
|
||||
:disabled="!files?.length || !selectedAccount"
|
||||
:loading="uploading"
|
||||
@click="uploadFile"
|
||||
prepend-icon="mdi-upload"
|
||||
>
|
||||
Upload & Preview
|
||||
</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<template v-slot:item.2>
|
||||
<v-card flat>
|
||||
<v-card-text v-if="uploadResult">
|
||||
<div class="d-flex ga-3 mb-4">
|
||||
<v-chip color="primary" variant="elevated">
|
||||
{{ uploadResult.totalTransactions }} Total
|
||||
</v-chip>
|
||||
<v-chip color="success" variant="elevated">
|
||||
{{ uploadResult.newTransactions }} New
|
||||
</v-chip>
|
||||
<v-chip v-if="uploadResult.possibleDuplicates > 0" color="warning" variant="elevated">
|
||||
{{ uploadResult.possibleDuplicates }} Possible Duplicates
|
||||
</v-chip>
|
||||
</div>
|
||||
|
||||
<v-data-table
|
||||
:headers="previewHeaders"
|
||||
:items="previewItems"
|
||||
density="compact"
|
||||
:items-per-page="25"
|
||||
>
|
||||
<template #item.date="{ item }">
|
||||
{{ formatDate(item.date) }}
|
||||
</template>
|
||||
<template #item.amount="{ item }">
|
||||
<span :class="item.amount >= 0 ? 'text-success' : 'text-error'" class="font-weight-medium">
|
||||
{{ formatCurrency(item.amount) }}
|
||||
</span>
|
||||
</template>
|
||||
<template #item.status="{ item }">
|
||||
<v-chip
|
||||
v-if="!item.duplicateMatch"
|
||||
color="success"
|
||||
size="small"
|
||||
variant="flat"
|
||||
>
|
||||
New
|
||||
</v-chip>
|
||||
<v-chip
|
||||
v-else
|
||||
:color="getConfidenceColor(item.duplicateMatch.confidence)"
|
||||
size="small"
|
||||
variant="flat"
|
||||
>
|
||||
Duplicate ({{ Math.round(item.duplicateMatch.confidence * 100) }}%)
|
||||
</v-chip>
|
||||
</template>
|
||||
<template #item.matchReason="{ item }">
|
||||
<span v-if="item.duplicateMatch" class="text-caption text-medium-emphasis">
|
||||
{{ item.duplicateMatch.matchReason }}
|
||||
</span>
|
||||
</template>
|
||||
<template #item.action="{ item }">
|
||||
<v-select
|
||||
v-if="item.duplicateMatch"
|
||||
v-model="resolutions[item.index]"
|
||||
:items="actionOptions"
|
||||
density="compact"
|
||||
variant="outlined"
|
||||
hide-details
|
||||
style="min-width: 120px"
|
||||
/>
|
||||
<span v-else class="text-caption text-success">Import</span>
|
||||
</template>
|
||||
</v-data-table>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-btn variant="text" @click="resetWizard">Back</v-btn>
|
||||
<v-spacer />
|
||||
<v-btn
|
||||
color="primary"
|
||||
:loading="confirming"
|
||||
@click="confirmImport"
|
||||
prepend-icon="mdi-check"
|
||||
>
|
||||
Confirm Import
|
||||
</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<template v-slot:item.3>
|
||||
<v-card flat>
|
||||
<v-card-text v-if="confirmResult" class="text-center py-8">
|
||||
<v-icon color="success" size="64" class="mb-4">mdi-check-circle</v-icon>
|
||||
<h2 class="text-h5 mb-4">Import Complete</h2>
|
||||
<div class="d-flex justify-center ga-3 mb-4">
|
||||
<v-chip color="success" variant="elevated" size="large">
|
||||
{{ confirmResult.importedCount }} Imported
|
||||
</v-chip>
|
||||
<v-chip v-if="confirmResult.duplicateCount > 0" color="info" variant="elevated" size="large">
|
||||
{{ confirmResult.duplicateCount }} Merged
|
||||
</v-chip>
|
||||
<v-chip v-if="confirmResult.skippedCount > 0" color="default" variant="elevated" size="large">
|
||||
{{ confirmResult.skippedCount }} Skipped
|
||||
</v-chip>
|
||||
</div>
|
||||
<p class="text-body-1 text-medium-emphasis">
|
||||
File: {{ confirmResult.fileName }} | Account: {{ confirmResult.accountName }}
|
||||
</p>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn color="primary" variant="text" @click="resetWizard" prepend-icon="mdi-upload">
|
||||
Import Another
|
||||
</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</template>
|
||||
</v-stepper>
|
||||
</v-card>
|
||||
|
||||
<h2 class="text-h5 mb-3">Import History</h2>
|
||||
<v-card>
|
||||
<v-data-table
|
||||
:headers="historyHeaders"
|
||||
:items="batches"
|
||||
:loading="loadingBatches"
|
||||
density="compact"
|
||||
:items-per-page="10"
|
||||
no-data-text="No import history yet"
|
||||
>
|
||||
<template #item.importedAt="{ item }">
|
||||
{{ formatDateTime(item.importedAt) }}
|
||||
</template>
|
||||
<template #item.status="{ item }">
|
||||
<v-chip
|
||||
:color="item.status === 'Confirmed' ? 'success' : item.status === 'Pending' ? 'warning' : 'error'"
|
||||
size="small"
|
||||
variant="flat"
|
||||
>
|
||||
{{ item.status }}
|
||||
</v-chip>
|
||||
</template>
|
||||
<template #item.summary="{ item }">
|
||||
{{ item.importedCount }} imported
|
||||
<span v-if="item.duplicateCount > 0">, {{ item.duplicateCount }} merged</span>
|
||||
<span v-if="item.skippedCount > 0">, {{ item.skippedCount }} skipped</span>
|
||||
</template>
|
||||
</v-data-table>
|
||||
</v-card>
|
||||
|
||||
<v-snackbar v-model="snackbar.show" :color="snackbar.color" :timeout="4000">
|
||||
{{ snackbar.text }}
|
||||
</v-snackbar>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, onMounted } from 'vue'
|
||||
import { ref, onMounted, computed } from 'vue'
|
||||
import { useAccountsStore } from '@/stores/accounts'
|
||||
import api from '@/services/api'
|
||||
import { importsApi } from '@/services/imports'
|
||||
import type { ImportUploadResponse, ImportBatch, DuplicateAction } from '@/types'
|
||||
|
||||
const accountsStore = useAccountsStore()
|
||||
const accounts = ref(accountsStore.accounts)
|
||||
const file = ref<File | null>(null)
|
||||
const accounts = computed(() => accountsStore.accounts.filter(a => a.isActive && !a.isClosed))
|
||||
|
||||
const step = ref(1)
|
||||
const files = ref<File[]>([])
|
||||
const selectedAccount = ref('')
|
||||
const uploading = ref(false)
|
||||
const confirming = ref(false)
|
||||
const uploadResult = ref<ImportUploadResponse | null>(null)
|
||||
const confirmResult = ref<ImportBatch | null>(null)
|
||||
const resolutions = ref<Record<number, DuplicateAction>>({})
|
||||
const batches = ref<ImportBatch[]>([])
|
||||
const loadingBatches = ref(false)
|
||||
const snackbar = ref({ show: false, text: '', color: 'success' })
|
||||
|
||||
const actionOptions = [
|
||||
{ title: 'Skip', value: 'Skip' as DuplicateAction },
|
||||
{ title: 'Import', value: 'Import' as DuplicateAction },
|
||||
{ title: 'Merge', value: 'Merge' as DuplicateAction },
|
||||
]
|
||||
|
||||
const previewHeaders = [
|
||||
{ title: 'Date', key: 'date', width: '110px' },
|
||||
{ title: 'Amount', key: 'amount', align: 'end' as const, width: '110px' },
|
||||
{ title: 'Payee', key: 'payeeName' },
|
||||
{ title: 'Memo', key: 'memo' },
|
||||
{ title: 'Status', key: 'status', width: '140px' },
|
||||
{ title: 'Match Reason', key: 'matchReason' },
|
||||
{ title: 'Action', key: 'action', width: '140px' },
|
||||
]
|
||||
|
||||
const historyHeaders = [
|
||||
{ title: 'Date', key: 'importedAt', width: '170px' },
|
||||
{ title: 'File', key: 'fileName' },
|
||||
{ title: 'Type', key: 'fileType', width: '70px' },
|
||||
{ title: 'Account', key: 'accountName' },
|
||||
{ title: 'Total', key: 'totalCount', align: 'end' as const, width: '80px' },
|
||||
{ title: 'Summary', key: 'summary' },
|
||||
{ title: 'Status', key: 'status', width: '110px' },
|
||||
]
|
||||
|
||||
const previewItems = computed(() => {
|
||||
return uploadResult.value?.previews ?? []
|
||||
})
|
||||
|
||||
onMounted(async () => {
|
||||
await accountsStore.fetchAccounts()
|
||||
accounts.value = accountsStore.accounts
|
||||
await loadBatches()
|
||||
})
|
||||
|
||||
async function uploadFile() {
|
||||
if (!file.value || !selectedAccount.value) return
|
||||
if (!files.value?.length || !selectedAccount.value) return
|
||||
uploading.value = true
|
||||
try {
|
||||
const formData = new FormData()
|
||||
formData.append('file', file.value)
|
||||
await api.post(`/imports/upload?accountId=${selectedAccount.value}`, formData, { headers: { 'Content-Type': 'multipart/form-data' } })
|
||||
file.value = null
|
||||
const { data } = await importsApi.upload(selectedAccount.value, files.value[0])
|
||||
uploadResult.value = data
|
||||
|
||||
// Set default resolutions for duplicates
|
||||
for (const preview of data.previews) {
|
||||
if (preview.duplicateMatch) {
|
||||
resolutions.value[preview.index] = preview.duplicateMatch.confidence >= 0.85 ? 'Skip' : 'Import'
|
||||
}
|
||||
}
|
||||
|
||||
step.value = 2
|
||||
} catch (err: any) {
|
||||
snackbar.value = {
|
||||
show: true,
|
||||
text: err.response?.data?.error || err.response?.data?.message || 'Upload failed',
|
||||
color: 'error'
|
||||
}
|
||||
} finally {
|
||||
uploading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function confirmImport() {
|
||||
if (!uploadResult.value) return
|
||||
confirming.value = true
|
||||
try {
|
||||
// Build resolution list for all transactions
|
||||
const allResolutions = uploadResult.value.previews.map(p => ({
|
||||
index: p.index,
|
||||
action: (p.duplicateMatch ? resolutions.value[p.index] ?? 'Skip' : 'Import') as DuplicateAction
|
||||
}))
|
||||
|
||||
const { data } = await importsApi.resolveDuplicates(uploadResult.value.batchId, {
|
||||
resolutions: allResolutions
|
||||
})
|
||||
|
||||
confirmResult.value = data
|
||||
step.value = 3
|
||||
await loadBatches()
|
||||
|
||||
snackbar.value = {
|
||||
show: true,
|
||||
text: `Successfully imported ${data.importedCount} transactions`,
|
||||
color: 'success'
|
||||
}
|
||||
} catch (err: any) {
|
||||
snackbar.value = {
|
||||
show: true,
|
||||
text: err.response?.data?.error || err.response?.data?.message || 'Import confirmation failed',
|
||||
color: 'error'
|
||||
}
|
||||
} finally {
|
||||
confirming.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function loadBatches() {
|
||||
loadingBatches.value = true
|
||||
try {
|
||||
const { data } = await importsApi.getBatches()
|
||||
batches.value = data
|
||||
} catch {
|
||||
// Silently fail - history is non-critical
|
||||
} finally {
|
||||
loadingBatches.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function resetWizard() {
|
||||
step.value = 1
|
||||
files.value = []
|
||||
uploadResult.value = null
|
||||
confirmResult.value = null
|
||||
resolutions.value = {}
|
||||
}
|
||||
|
||||
function getConfidenceColor(confidence: number): string {
|
||||
if (confidence >= 0.85) return 'error'
|
||||
if (confidence >= 0.50) return 'warning'
|
||||
return 'success'
|
||||
}
|
||||
|
||||
function formatDate(dateStr: string): string {
|
||||
return new Date(dateStr).toLocaleDateString()
|
||||
}
|
||||
|
||||
function formatDateTime(dateStr: string): string {
|
||||
return new Date(dateStr).toLocaleString()
|
||||
}
|
||||
|
||||
function formatCurrency(amount: number): string {
|
||||
return new Intl.NumberFormat('en-US', { style: 'currency', currency: 'USD' }).format(amount)
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -32,9 +32,13 @@ public class ImportsController : ControllerBase
|
||||
=> Ok(await _importService.ConfirmBatchAsync(UserId, batchId));
|
||||
|
||||
[HttpPost("batches/{batchId}/resolve-duplicates")]
|
||||
public async Task<ActionResult> ResolveDuplicates(Guid batchId, ResolveDuplicatesRequest request)
|
||||
public async Task<ActionResult<ImportBatchResponse>> ResolveDuplicates(Guid batchId, ResolveDuplicatesRequest request)
|
||||
{
|
||||
await _importService.ResolveDuplicatesAsync(UserId, batchId, request);
|
||||
return NoContent();
|
||||
var result = await _importService.ResolveDuplicatesAsync(UserId, batchId, request);
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
[HttpGet("batches")]
|
||||
public async Task<ActionResult<List<ImportBatchResponse>>> GetBatches()
|
||||
=> Ok(await _importService.GetBatchesAsync(UserId));
|
||||
}
|
||||
|
||||
@@ -1,16 +1,30 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Purrse.Api.Hubs;
|
||||
using Purrse.Core.Interfaces.Services;
|
||||
using Purrse.Data;
|
||||
|
||||
namespace Purrse.Api.Services;
|
||||
|
||||
public class FileWatcherService : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly ILogger<FileWatcherService> _logger;
|
||||
private readonly IHubContext<NotificationHub> _hubContext;
|
||||
private readonly string _importsPath;
|
||||
private FileSystemWatcher? _watcher;
|
||||
|
||||
public FileWatcherService(IServiceScopeFactory scopeFactory, ILogger<FileWatcherService> logger, IConfiguration config)
|
||||
private static readonly string[] SupportedExtensions = { ".ofx", ".qfx", ".csv", ".qif" };
|
||||
|
||||
public FileWatcherService(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
ILogger<FileWatcherService> logger,
|
||||
IConfiguration config,
|
||||
IHubContext<NotificationHub> hubContext)
|
||||
{
|
||||
_scopeFactory = scopeFactory;
|
||||
_logger = logger;
|
||||
_hubContext = hubContext;
|
||||
_importsPath = config["Imports:WatchPath"] ?? Path.Combine(AppContext.BaseDirectory, "imports");
|
||||
}
|
||||
|
||||
@@ -19,10 +33,15 @@ public class FileWatcherService : BackgroundService
|
||||
if (!Directory.Exists(_importsPath))
|
||||
Directory.CreateDirectory(_importsPath);
|
||||
|
||||
var archivePath = Path.Combine(_importsPath, "archive");
|
||||
if (!Directory.Exists(archivePath))
|
||||
Directory.CreateDirectory(archivePath);
|
||||
|
||||
_watcher = new FileSystemWatcher(_importsPath)
|
||||
{
|
||||
NotifyFilter = NotifyFilters.FileName | NotifyFilters.CreationTime,
|
||||
EnableRaisingEvents = true
|
||||
EnableRaisingEvents = true,
|
||||
IncludeSubdirectories = false
|
||||
};
|
||||
|
||||
_watcher.Created += OnFileCreated;
|
||||
@@ -38,10 +57,109 @@ public class FileWatcherService : BackgroundService
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private void OnFileCreated(object sender, FileSystemEventArgs e)
|
||||
private async void OnFileCreated(object sender, FileSystemEventArgs e)
|
||||
{
|
||||
_logger.LogInformation("New file detected: {FileName}", e.Name);
|
||||
// Files dropped here can be processed by the import service
|
||||
// A SignalR notification would be sent to connected clients
|
||||
try
|
||||
{
|
||||
var extension = Path.GetExtension(e.FullPath).ToLowerInvariant();
|
||||
if (!SupportedExtensions.Contains(extension))
|
||||
{
|
||||
_logger.LogDebug("Ignoring non-supported file: {FileName}", e.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("New import file detected: {FileName}", e.Name);
|
||||
|
||||
// Wait for file to finish writing
|
||||
await WaitForFileReady(e.FullPath);
|
||||
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<PurrseDbContext>();
|
||||
var importService = scope.ServiceProvider.GetRequiredService<IImportService>();
|
||||
|
||||
// Find the target account by matching filename to account name
|
||||
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(e.Name);
|
||||
var account = await db.Accounts
|
||||
.Where(a => a.IsActive && !a.IsClosed)
|
||||
.FirstOrDefaultAsync(a => a.Name.ToLower() == fileNameWithoutExt!.ToLower());
|
||||
|
||||
// Fall back to first active account
|
||||
account ??= await db.Accounts
|
||||
.Where(a => a.IsActive && !a.IsClosed)
|
||||
.OrderBy(a => a.SortOrder)
|
||||
.FirstOrDefaultAsync();
|
||||
|
||||
if (account == null)
|
||||
{
|
||||
_logger.LogWarning("No active account found for file import: {FileName}", e.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Importing {FileName} into account {AccountName} ({AccountId})",
|
||||
e.Name, account.Name, account.Id);
|
||||
|
||||
await using var fileStream = new FileStream(e.FullPath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
var result = await importService.UploadAsync(account.UserId, account.Id, e.Name!, fileStream);
|
||||
|
||||
_logger.LogInformation(
|
||||
"File import complete: {FileName} - {Total} transactions ({New} new, {Duplicates} duplicates)",
|
||||
e.Name, result.TotalTransactions, result.NewTransactions, result.PossibleDuplicates);
|
||||
|
||||
// Send SignalR notification to the account owner
|
||||
await _hubContext.Clients.Group(account.UserId.ToString()).SendAsync("FileImportReady", new
|
||||
{
|
||||
batchId = result.BatchId,
|
||||
fileName = e.Name,
|
||||
accountId = account.Id,
|
||||
accountName = account.Name,
|
||||
totalTransactions = result.TotalTransactions,
|
||||
newTransactions = result.NewTransactions,
|
||||
possibleDuplicates = result.PossibleDuplicates
|
||||
});
|
||||
|
||||
// Archive the processed file
|
||||
ArchiveFile(e.FullPath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error processing imported file: {FileName}", e.Name);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task WaitForFileReady(string filePath, int maxRetries = 30, int delayMs = 500)
|
||||
{
|
||||
for (int i = 0; i < maxRetries; i++)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.None);
|
||||
return; // File is ready
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
await Task.Delay(delayMs);
|
||||
}
|
||||
}
|
||||
|
||||
throw new IOException($"File {filePath} was not ready after {maxRetries * delayMs}ms");
|
||||
}
|
||||
|
||||
private void ArchiveFile(string filePath)
|
||||
{
|
||||
try
|
||||
{
|
||||
var archivePath = Path.Combine(_importsPath, "archive");
|
||||
var timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss");
|
||||
var fileName = Path.GetFileName(filePath);
|
||||
var archiveFileName = $"{timestamp}_{fileName}";
|
||||
var destination = Path.Combine(archivePath, archiveFileName);
|
||||
|
||||
File.Move(filePath, destination);
|
||||
_logger.LogInformation("Archived import file to: {ArchivePath}", destination);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to archive file: {FilePath}", filePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Purrse.Core.DTOs;
|
||||
using Purrse.Core.Enums;
|
||||
@@ -5,6 +6,7 @@ using Purrse.Core.Helpers;
|
||||
using Purrse.Core.Interfaces.Services;
|
||||
using Purrse.Core.Models;
|
||||
using Purrse.Plugins.Abstractions;
|
||||
using Purrse.Plugins.Abstractions.Models;
|
||||
using Purrse.Data;
|
||||
|
||||
namespace Purrse.Api.Services;
|
||||
@@ -46,7 +48,8 @@ public class ImportService : IImportService
|
||||
FileName = fileName,
|
||||
FileType = Path.GetExtension(fileName).TrimStart('.').ToUpperInvariant(),
|
||||
TotalCount = parseResult.Transactions.Count,
|
||||
Status = ImportStatus.Pending
|
||||
Status = ImportStatus.Pending,
|
||||
ParsedDataJson = JsonSerializer.Serialize(parseResult.Transactions)
|
||||
};
|
||||
|
||||
var previews = new List<ImportedTransactionPreview>();
|
||||
@@ -72,37 +75,178 @@ public class ImportService : IImportService
|
||||
|
||||
public async Task<ImportBatchResponse> ConfirmBatchAsync(Guid userId, Guid batchId)
|
||||
{
|
||||
var batch = await _db.ImportBatches.FirstOrDefaultAsync(b => b.Id == batchId && b.UserId == userId)
|
||||
var batch = await _db.ImportBatches
|
||||
.Include(b => b.Account)
|
||||
.FirstOrDefaultAsync(b => b.Id == batchId && b.UserId == userId)
|
||||
?? throw new KeyNotFoundException("Import batch not found");
|
||||
|
||||
batch.Status = ImportStatus.Confirmed;
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
return new ImportBatchResponse(batch.Id, batch.FileName, batch.FileType, batch.TotalCount,
|
||||
batch.ImportedCount, batch.DuplicateCount, batch.SkippedCount, batch.Status.ToString(), batch.ImportedAt);
|
||||
return MapBatchResponse(batch);
|
||||
}
|
||||
|
||||
public async Task ResolveDuplicatesAsync(Guid userId, Guid batchId, ResolveDuplicatesRequest request)
|
||||
public async Task<ImportBatchResponse> ResolveDuplicatesAsync(Guid userId, Guid batchId, ResolveDuplicatesRequest request)
|
||||
{
|
||||
var batch = await _db.ImportBatches.FirstOrDefaultAsync(b => b.Id == batchId && b.UserId == userId)
|
||||
var batch = await _db.ImportBatches
|
||||
.Include(b => b.Account)
|
||||
.FirstOrDefaultAsync(b => b.Id == batchId && b.UserId == userId)
|
||||
?? throw new KeyNotFoundException("Import batch not found");
|
||||
|
||||
foreach (var resolution in request.Resolutions)
|
||||
if (string.IsNullOrEmpty(batch.ParsedDataJson))
|
||||
throw new InvalidOperationException("No parsed data available for this batch. It may have already been processed.");
|
||||
|
||||
var parsedTransactions = JsonSerializer.Deserialize<List<ParsedTransaction>>(batch.ParsedDataJson)
|
||||
?? throw new InvalidOperationException("Failed to deserialize parsed data");
|
||||
|
||||
var resolutionMap = request.Resolutions.ToDictionary(r => r.Index, r => r.Action);
|
||||
|
||||
int importedCount = 0;
|
||||
int skippedCount = 0;
|
||||
int mergedCount = 0;
|
||||
|
||||
for (int i = 0; i < parsedTransactions.Count; i++)
|
||||
{
|
||||
switch (resolution.Action)
|
||||
var parsed = parsedTransactions[i];
|
||||
var action = resolutionMap.GetValueOrDefault(i, DuplicateAction.Import);
|
||||
|
||||
switch (action)
|
||||
{
|
||||
case DuplicateAction.Skip:
|
||||
batch.SkippedCount++;
|
||||
break;
|
||||
case DuplicateAction.Import:
|
||||
batch.ImportedCount++;
|
||||
await CreateTransaction(userId, batch, parsed);
|
||||
importedCount++;
|
||||
break;
|
||||
|
||||
case DuplicateAction.Skip:
|
||||
skippedCount++;
|
||||
break;
|
||||
|
||||
case DuplicateAction.Merge:
|
||||
batch.DuplicateCount++;
|
||||
var merged = await MergeTransaction(batch.AccountId, parsed);
|
||||
if (merged)
|
||||
mergedCount++;
|
||||
else
|
||||
{
|
||||
// If no existing transaction found to merge with, import instead
|
||||
await CreateTransaction(userId, batch, parsed);
|
||||
importedCount++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
batch.ImportedCount = importedCount;
|
||||
batch.SkippedCount = skippedCount;
|
||||
batch.DuplicateCount = mergedCount;
|
||||
batch.ParsedDataJson = null;
|
||||
batch.Status = ImportStatus.Confirmed;
|
||||
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
return MapBatchResponse(batch);
|
||||
}
|
||||
|
||||
public async Task<List<ImportBatchResponse>> GetBatchesAsync(Guid userId)
|
||||
{
|
||||
return await _db.ImportBatches
|
||||
.Include(b => b.Account)
|
||||
.Where(b => b.UserId == userId)
|
||||
.OrderByDescending(b => b.ImportedAt)
|
||||
.Select(b => new ImportBatchResponse(
|
||||
b.Id,
|
||||
b.AccountId,
|
||||
b.Account.Name,
|
||||
b.FileName,
|
||||
b.FileType,
|
||||
b.TotalCount,
|
||||
b.ImportedCount,
|
||||
b.DuplicateCount,
|
||||
b.SkippedCount,
|
||||
0,
|
||||
b.Status.ToString(),
|
||||
b.ImportedAt))
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
private async Task CreateTransaction(Guid userId, ImportBatch batch, ParsedTransaction parsed)
|
||||
{
|
||||
var fingerprint = TransactionFingerprint.Generate(batch.AccountId, parsed.Date, parsed.Amount, parsed.FitId);
|
||||
|
||||
var matchedPayee = await _payeeService.MatchPayeeAsync(userId, parsed.PayeeName ?? string.Empty);
|
||||
|
||||
var transaction = new Transaction
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AccountId = batch.AccountId,
|
||||
Date = parsed.Date,
|
||||
Amount = parsed.Amount,
|
||||
PayeeId = matchedPayee?.Id,
|
||||
PayeeName = parsed.PayeeName,
|
||||
CategoryId = matchedPayee?.DefaultCategoryId,
|
||||
Memo = parsed.Memo,
|
||||
ReferenceNumber = parsed.ReferenceNumber,
|
||||
CheckNumber = parsed.CheckNumber,
|
||||
FitId = parsed.FitId,
|
||||
Fingerprint = fingerprint,
|
||||
Type = parsed.Amount >= 0 ? TransactionType.Credit : TransactionType.Debit,
|
||||
Status = TransactionStatus.Uncleared,
|
||||
ImportBatchId = batch.Id
|
||||
};
|
||||
|
||||
_db.Transactions.Add(transaction);
|
||||
|
||||
// Update account balance
|
||||
batch.Account.Balance += parsed.Amount;
|
||||
batch.Account.UpdatedAt = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
private async Task<bool> MergeTransaction(Guid accountId, ParsedTransaction parsed)
|
||||
{
|
||||
var duplicate = await _duplicateService.FindDuplicateAsync(
|
||||
accountId, parsed.Date, parsed.Amount, parsed.PayeeName, parsed.FitId, parsed.CheckNumber);
|
||||
|
||||
if (duplicate == null)
|
||||
return false;
|
||||
|
||||
var existing = await _db.Transactions.FindAsync(duplicate.ExistingTransactionId);
|
||||
if (existing == null)
|
||||
return false;
|
||||
|
||||
// Fill in blank fields without overwriting user-edited data
|
||||
if (string.IsNullOrEmpty(existing.Memo) && !string.IsNullOrEmpty(parsed.Memo))
|
||||
existing.Memo = parsed.Memo;
|
||||
|
||||
if (string.IsNullOrEmpty(existing.FitId) && !string.IsNullOrEmpty(parsed.FitId))
|
||||
existing.FitId = parsed.FitId;
|
||||
|
||||
if (string.IsNullOrEmpty(existing.CheckNumber) && !string.IsNullOrEmpty(parsed.CheckNumber))
|
||||
existing.CheckNumber = parsed.CheckNumber;
|
||||
|
||||
if (string.IsNullOrEmpty(existing.ReferenceNumber) && !string.IsNullOrEmpty(parsed.ReferenceNumber))
|
||||
existing.ReferenceNumber = parsed.ReferenceNumber;
|
||||
|
||||
if (string.IsNullOrEmpty(existing.Fingerprint))
|
||||
existing.Fingerprint = TransactionFingerprint.Generate(accountId, parsed.Date, parsed.Amount, parsed.FitId);
|
||||
|
||||
existing.UpdatedAt = DateTime.UtcNow;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static ImportBatchResponse MapBatchResponse(ImportBatch batch)
|
||||
{
|
||||
return new ImportBatchResponse(
|
||||
batch.Id,
|
||||
batch.AccountId,
|
||||
batch.Account?.Name ?? string.Empty,
|
||||
batch.FileName,
|
||||
batch.FileType,
|
||||
batch.TotalCount,
|
||||
batch.ImportedCount,
|
||||
batch.DuplicateCount,
|
||||
batch.SkippedCount,
|
||||
batch.DuplicateCount,
|
||||
batch.Status.ToString(),
|
||||
batch.ImportedAt);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,17 +6,21 @@ public interface IImportService
|
||||
{
|
||||
Task<ImportUploadResponse> UploadAsync(Guid userId, Guid accountId, string fileName, Stream fileStream);
|
||||
Task<ImportBatchResponse> ConfirmBatchAsync(Guid userId, Guid batchId);
|
||||
Task ResolveDuplicatesAsync(Guid userId, Guid batchId, ResolveDuplicatesRequest request);
|
||||
Task<ImportBatchResponse> ResolveDuplicatesAsync(Guid userId, Guid batchId, ResolveDuplicatesRequest request);
|
||||
Task<List<ImportBatchResponse>> GetBatchesAsync(Guid userId);
|
||||
}
|
||||
|
||||
public record ImportBatchResponse(
|
||||
Guid Id,
|
||||
Guid AccountId,
|
||||
string AccountName,
|
||||
string FileName,
|
||||
string FileType,
|
||||
int TotalCount,
|
||||
int ImportedCount,
|
||||
int DuplicateCount,
|
||||
int SkippedCount,
|
||||
int MergedCount,
|
||||
string Status,
|
||||
DateTime ImportedAt
|
||||
);
|
||||
|
||||
@@ -14,6 +14,7 @@ public class ImportBatch
|
||||
public int DuplicateCount { get; set; }
|
||||
public int SkippedCount { get; set; }
|
||||
public ImportStatus Status { get; set; } = ImportStatus.Pending;
|
||||
public string? ParsedDataJson { get; set; }
|
||||
public DateTime ImportedAt { get; set; } = DateTime.UtcNow;
|
||||
|
||||
public User User { get; set; } = null!;
|
||||
|
||||
@@ -12,6 +12,7 @@ public class ImportBatchConfiguration : IEntityTypeConfiguration<ImportBatch>
|
||||
builder.HasKey(b => b.Id);
|
||||
builder.Property(b => b.FileName).HasMaxLength(500).IsRequired();
|
||||
builder.Property(b => b.FileType).HasMaxLength(50).IsRequired();
|
||||
builder.Property(b => b.ParsedDataJson).HasColumnType("text");
|
||||
|
||||
builder.HasOne(b => b.User)
|
||||
.WithMany()
|
||||
|
||||
+1149
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,28 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace Purrse.Data.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class AddParsedDataToImportBatch : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.AddColumn<string>(
|
||||
name: "ParsedDataJson",
|
||||
table: "import_batches",
|
||||
type: "text",
|
||||
nullable: true);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropColumn(
|
||||
name: "ParsedDataJson",
|
||||
table: "import_batches");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -249,6 +249,9 @@ namespace Purrse.Data.Migrations
|
||||
b.Property<int>("ImportedCount")
|
||||
.HasColumnType("integer");
|
||||
|
||||
b.Property<string>("ParsedDataJson")
|
||||
.HasColumnType("text");
|
||||
|
||||
b.Property<int>("SkippedCount")
|
||||
.HasColumnType("integer");
|
||||
|
||||
|
||||
Reference in New Issue
Block a user