Fix SimpleFIN re-sync timeout and 403 rate limiting
Cache SimpleFIN API responses for 60 seconds so multiple accounts in the same sync cycle reuse a single HTTP call instead of hitting the API per-account. Pre-load existing transactions by FitId in one batch query to skip per-duplicate FindAsync calls during memo backfill. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -413,6 +413,17 @@ public class BankSyncService : IBankSyncService
|
|||||||
var accountResults = new List<SyncAccountResult>();
|
var accountResults = new List<SyncAccountResult>();
|
||||||
int totalFetched = 0, totalImported = 0, totalSkipped = 0;
|
int totalFetched = 0, totalImported = 0, totalSkipped = 0;
|
||||||
|
|
||||||
|
// For SimpleFIN, use the earliest start date across all accounts so the
|
||||||
|
// single API call (cached) returns transactions covering every account.
|
||||||
|
var endDate = DateTime.UtcNow;
|
||||||
|
DateTime? simpleFinStartDate = null;
|
||||||
|
if (connection.Provider == SyncProvider.SimpleFIN)
|
||||||
|
{
|
||||||
|
simpleFinStartDate = enabledAccounts
|
||||||
|
.Select(la => la.LastSyncAt ?? endDate.AddDays(-90))
|
||||||
|
.Min();
|
||||||
|
}
|
||||||
|
|
||||||
foreach (var linkedAccount in enabledAccounts)
|
foreach (var linkedAccount in enabledAccounts)
|
||||||
{
|
{
|
||||||
var log = new SyncLog
|
var log = new SyncLog
|
||||||
@@ -427,8 +438,7 @@ public class BankSyncService : IBankSyncService
|
|||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var endDate = DateTime.UtcNow;
|
var startDate = simpleFinStartDate ?? linkedAccount.LastSyncAt ?? endDate.AddDays(-90);
|
||||||
var startDate = linkedAccount.LastSyncAt ?? endDate.AddDays(-90);
|
|
||||||
|
|
||||||
var provider = connection.Provider == SyncProvider.Plaid
|
var provider = connection.Provider == SyncProvider.Plaid
|
||||||
? (Purrse.Plugins.Abstractions.IAccountSyncProvider)_plaid
|
? (Purrse.Plugins.Abstractions.IAccountSyncProvider)_plaid
|
||||||
@@ -454,24 +464,39 @@ public class BankSyncService : IBankSyncService
|
|||||||
int imported = 0;
|
int imported = 0;
|
||||||
int skipped = 0;
|
int skipped = 0;
|
||||||
|
|
||||||
|
// Pre-load existing transactions for this account that have FitIds matching
|
||||||
|
// incoming data, so we can batch-update memos without per-row FindAsync calls
|
||||||
|
var incomingFitIds = parseResult.Transactions
|
||||||
|
.Where(t => !string.IsNullOrEmpty(t.FitId))
|
||||||
|
.Select(t => t.FitId!)
|
||||||
|
.ToHashSet();
|
||||||
|
var existingByFitId = incomingFitIds.Count > 0
|
||||||
|
? await _db.Transactions
|
||||||
|
.Where(t => t.AccountId == linkedAccount.AccountId!.Value
|
||||||
|
&& t.FitId != null && incomingFitIds.Contains(t.FitId))
|
||||||
|
.ToDictionaryAsync(t => t.FitId!, t => t)
|
||||||
|
: new Dictionary<string, Transaction>();
|
||||||
|
|
||||||
foreach (var parsed in parseResult.Transactions)
|
foreach (var parsed in parseResult.Transactions)
|
||||||
{
|
{
|
||||||
|
// Fast-path: if we already have a transaction with this FitId, skip
|
||||||
|
// and backfill memo if needed — no duplicate detection query required
|
||||||
|
if (!string.IsNullOrEmpty(parsed.FitId) && existingByFitId.TryGetValue(parsed.FitId, out var existingTx))
|
||||||
|
{
|
||||||
|
if (!string.IsNullOrEmpty(parsed.Memo) && string.IsNullOrEmpty(existingTx.Memo))
|
||||||
|
{
|
||||||
|
existingTx.Memo = parsed.Memo;
|
||||||
|
}
|
||||||
|
skipped++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
var duplicate = await _duplicateService.FindDuplicateAsync(
|
var duplicate = await _duplicateService.FindDuplicateAsync(
|
||||||
linkedAccount.AccountId!.Value, parsed.Date, parsed.Amount,
|
linkedAccount.AccountId!.Value, parsed.Date, parsed.Amount,
|
||||||
parsed.PayeeName, parsed.FitId, parsed.CheckNumber);
|
parsed.PayeeName, parsed.FitId, parsed.CheckNumber);
|
||||||
|
|
||||||
if (duplicate != null)
|
if (duplicate != null)
|
||||||
{
|
{
|
||||||
// Update memo if the existing transaction has no memo
|
|
||||||
if (!string.IsNullOrEmpty(parsed.Memo))
|
|
||||||
{
|
|
||||||
var existing = await _db.Transactions.FindAsync(duplicate.ExistingTransactionId);
|
|
||||||
if (existing != null && string.IsNullOrEmpty(existing.Memo))
|
|
||||||
{
|
|
||||||
existing.Memo = parsed.Memo;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
skipped++;
|
skipped++;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,11 @@ public class SimpleFinSyncProvider : IAccountSyncProvider
|
|||||||
{
|
{
|
||||||
private readonly HttpClient _httpClient;
|
private readonly HttpClient _httpClient;
|
||||||
|
|
||||||
|
// Short-lived cache to avoid hitting SimpleFIN multiple times in one sync cycle
|
||||||
|
private string? _cachedUrl;
|
||||||
|
private SimpleFinAccountSet? _cachedResponse;
|
||||||
|
private DateTime _cacheExpiry;
|
||||||
|
|
||||||
public string Id => "simplefin-sync";
|
public string Id => "simplefin-sync";
|
||||||
public string Name => "SimpleFIN Bank Sync";
|
public string Name => "SimpleFIN Bank Sync";
|
||||||
public string Version => "1.0.0";
|
public string Version => "1.0.0";
|
||||||
@@ -63,8 +68,8 @@ public class SimpleFinSyncProvider : IAccountSyncProvider
|
|||||||
var startUnix = new DateTimeOffset(startDate).ToUnixTimeSeconds();
|
var startUnix = new DateTimeOffset(startDate).ToUnixTimeSeconds();
|
||||||
var endUnix = new DateTimeOffset(endDate).ToUnixTimeSeconds();
|
var endUnix = new DateTimeOffset(endDate).ToUnixTimeSeconds();
|
||||||
|
|
||||||
var response = await GetWithAuthAsync(
|
var url = $"{accessUrl}/accounts?start-date={startUnix}&end-date={endUnix}";
|
||||||
$"{accessUrl}/accounts?start-date={startUnix}&end-date={endUnix}");
|
var response = await GetWithAuthCachedAsync(url);
|
||||||
|
|
||||||
if (response == null)
|
if (response == null)
|
||||||
return new ParseResult { Success = false, ErrorMessage = "No response from SimpleFIN" };
|
return new ParseResult { Success = false, ErrorMessage = "No response from SimpleFIN" };
|
||||||
@@ -93,6 +98,27 @@ public class SimpleFinSyncProvider : IAccountSyncProvider
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Returns a cached response if the same URL was fetched within the last 60 seconds.
|
||||||
|
/// SimpleFIN returns all accounts in a single response, so caching avoids redundant
|
||||||
|
/// API calls when syncing multiple accounts from the same connection.
|
||||||
|
/// </summary>
|
||||||
|
private async Task<SimpleFinAccountSet?> GetWithAuthCachedAsync(string url)
|
||||||
|
{
|
||||||
|
// Cache by base URL (without query params) so that calls for different
|
||||||
|
// date ranges within the same sync cycle reuse the first response.
|
||||||
|
// The first call uses the widest date window (earliest LastSyncAt).
|
||||||
|
var cacheKey = url.Contains('?') ? url[..url.IndexOf('?')] : url;
|
||||||
|
if (_cachedUrl == cacheKey && _cachedResponse != null && DateTime.UtcNow < _cacheExpiry)
|
||||||
|
return _cachedResponse;
|
||||||
|
|
||||||
|
var response = await GetWithAuthAsync(url);
|
||||||
|
_cachedUrl = cacheKey;
|
||||||
|
_cachedResponse = response;
|
||||||
|
_cacheExpiry = DateTime.UtcNow.AddSeconds(60);
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// HttpClient does not extract Basic Auth credentials from URLs (user:pass@host).
|
/// HttpClient does not extract Basic Auth credentials from URLs (user:pass@host).
|
||||||
/// This method parses the URI, builds the Authorization header, and strips credentials from the request URL.
|
/// This method parses the URI, builds the Authorization header, and strips credentials from the request URL.
|
||||||
|
|||||||
Reference in New Issue
Block a user