Skip to content

fix: recover stalled wallet syncs via idle watchdog and progress reorder #1296

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
reubenyap wants to merge 7 commits into
base: staging
Choose a base branch
from
+183 −84
Open

fix: recover stalled wallet syncs via idle watchdog and progress reorder #1296

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
425e494
refactor: extract _refresh() body into _doRefreshWork helper
reubenyap Apr 20, 2026
ab2823e
fix: fire refresh progress updates after awaited work completes
reubenyap Apr 20, 2026
b91464a
fix: add idle watchdog to _refresh() so hung syncs release the mutex
reubenyap Apr 20, 2026
44339df
fix: isolate onRequestComplete hook from RPC error handling
Apr 22, 2026
8c6e809
fix: suppress uncaught errors from detached refresh work after watchd…
Apr 22, 2026
d064947
refactor: promote refresh watchdog intervals to named constants
Apr 22, 2026
e7ed2ae
docs: correct watchdog comment — Spark is covered by electrum hook, n…
Apr 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions lib/electrumx_rpc/electrumx_client.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,28 @@ class ElectrumXClient {
final Mutex _torConnectingLock = Mutex();
bool _requireMutex = false;

/// Optional hook fired after each successful RPC completes. Used by
/// higher layers (e.g. the wallet refresh idle watchdog) to detect
/// liveness during long sequential RPC loops without having to
/// instrument every call site.
///
/// Single-subscriber. Invoked via [_fireOnRequestComplete] which
/// swallows exceptions so a faulty hook cannot be misattributed as
/// an RPC failure by the surrounding catch block.
void Function()? onRequestComplete;

void _fireOnRequestComplete() {
try {
onRequestComplete?.call();
} catch (e, s) {
Logging.instance.w(
"onRequestComplete hook threw",
error: e,
stackTrace: s,
);
}
}

ElectrumXClient({
required String host,
required int port,
Expand Down Expand Up @@ -368,6 +390,7 @@ class ElectrumXClient {
}

currentFailoverIndex = -1;
_fireOnRequestComplete();

// If the command is a ping, a good return should always be null.
if (command.contains("ping")) {
Expand Down Expand Up @@ -474,6 +497,7 @@ class ElectrumXClient {
// }

currentFailoverIndex = -1;
_fireOnRequestComplete();
return response;
} on WifiOnlyException {
rethrow;
Expand Down
243 changes: 159 additions & 84 deletions lib/wallets/wallet/wallet.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,20 @@ abstract class Wallet<T extends CryptoCurrency> {

// ===== private properties ===========================================

/// Maximum time with no refresh activity before the idle watchdog
/// trips and unblocks _refresh() so refreshMutex can be released.
static const _refreshIdleThreshold = Duration(minutes: 5);

/// How often the idle watchdog checks _lastRefreshProgress.
static const _refreshWatchdogTick = Duration(seconds: 30);

Timer? _periodicRefreshTimer;
Timer? _networkAliveTimer;

/// Timestamp of the last _fireRefreshPercentChange during an active
/// refresh. Consumed by the idle watchdog in _refresh() to detect hangs.
DateTime? _lastRefreshProgress;

bool _shouldAutoSync = false;

bool _isConnected = false;
Expand Down Expand Up @@ -603,6 +614,7 @@ abstract class Wallet<T extends CryptoCurrency> {
}

void _fireRefreshPercentChange(double percent) {
_lastRefreshProgress = DateTime.now();
if (this is ElectrumXInterface) {
(this as ElectrumXInterface?)?.refreshingPercent = percent;
}
Expand Down Expand Up @@ -641,95 +653,66 @@ abstract class Wallet<T extends CryptoCurrency> {
);
}

// add some small buffer before making calls.
// this can probably be removed in the future but was added as a
// debugging feature
await Future<void>.delayed(const Duration(milliseconds: 300));

// TODO: [prio=low] handle this differently. Extra modification of this file for coin specific functionality should be avoided.
final Set<String> codesToCheck = {};
if (this is PaynymInterface && !viewOnly) {
// isSegwit does not matter here at all
final myCode = await (this as PaynymInterface).getPaymentCode(
isSegwit: false,
);

final nym = await PaynymIsApi().nym(myCode.toString());
if (nym.value != null) {
for (final follower in nym.value!.followers) {
codesToCheck.add(follower.code);
}
for (final following in nym.value!.following) {
codesToCheck.add(following.code);
}
}
// Idle watchdog: trips when no refresh activity has been observed
// for _refreshIdleThreshold, signalling that the refresh is wedged.
// Slow-but-active syncs keep the watchdog fed and aren't killed:
// - _fireRefreshPercentChange ticks (coarse phase checkpoints)
// - successful electrum RPCs (via ElectrumXClient.onRequestComplete)
// — this covers Spark anon-set downloads and long updateTransactions
// loops, which use electrumXClient directly and do not call
// _fireRefreshPercentChange between phases.
// Per-call hang detection is still the responsibility of the
// underlying adapters (e.g. electrum's connectionTimeout). This only
// catches what slips through those layers and would otherwise hold
// refreshMutex locked until the app is force-closed.
_lastRefreshProgress = DateTime.now();

// Feed the watchdog from successful electrum RPCs, so long sequential
// fetches (e.g. updateTransactions on a wallet with a large history)
// are classified as active rather than idle.
if (this is ElectrumXInterface) {
(this as ElectrumXInterface).electrumXClient.onRequestComplete = () {
_lastRefreshProgress = DateTime.now();
};
}

_fireRefreshPercentChange(0);
await updateChainHeight();

if (this is BitcoinFrostWallet) {
await (this as BitcoinFrostWallet).lookAhead();
}

_fireRefreshPercentChange(0.1);

// TODO: [prio=low] handle this differently. Extra modification of this file for coin specific functionality should be avoided.
if (this is MultiAddressInterface) {
if (info.otherData[WalletInfoKeys.reuseAddress] != true) {
await (this as MultiAddressInterface)
.checkReceivingAddressForTransactions();
final watchdogCompleter = Completer<void>();
final watchdog = Timer.periodic(_refreshWatchdogTick, (timer) {
if (watchdogCompleter.isCompleted) {
timer.cancel();
return;
}
}

_fireRefreshPercentChange(0.2);

// TODO: [prio=low] handle this differently. Extra modification of this file for coin specific functionality should be avoided.
if (this is MultiAddressInterface) {
if (info.otherData[WalletInfoKeys.reuseAddress] != true) {
await (this as MultiAddressInterface)
.checkChangeAddressForTransactions();
final last = _lastRefreshProgress;
if (last == null) return;
if (DateTime.now().difference(last) >= _refreshIdleThreshold) {
timer.cancel();
watchdogCompleter.completeError(
TimeoutException(
'Wallet refresh for $walletId idle for '
'${_refreshIdleThreshold.inMinutes} min',
_refreshIdleThreshold,
),
);
}
});

final work = _doRefreshWork(viewOnly);
try {
await Future.any([work, watchdogCompleter.future]);
} finally {
watchdog.cancel();
if (this is ElectrumXInterface) {
(this as ElectrumXInterface).electrumXClient.onRequestComplete =
null;
}
_lastRefreshProgress = null;
// If the watchdog won the race, `work` is detached and still
// running; an eventual throw would surface as an uncaught async
// error. Attach a no-op error handler to suppress it. If `work`
// completed first, this future is already resolved and the
// handler is a no-op.
unawaited(work.catchError((_) {}));
}
_fireRefreshPercentChange(0.3);
if (this is SparkInterface && !viewOnly) {
// this should be called before updateTransactions()
await (this as SparkInterface).refreshSparkData((0.3, 0.6));
}

if (this is NamecoinWallet) {
await updateUTXOs();
_fireRefreshPercentChange(0.6);
await (this as NamecoinWallet).checkAutoRegisterNameNewOutputs();
_fireRefreshPercentChange(0.70);
await updateTransactions();
} else {
final fetchFuture = updateTransactions();
_fireRefreshPercentChange(0.6);
final utxosRefreshFuture = updateUTXOs();
_fireRefreshPercentChange(0.65);
await utxosRefreshFuture;
_fireRefreshPercentChange(0.70);
await fetchFuture;
}

// TODO: [prio=low] handle this differently. Extra modification of this file for coin specific functionality should be avoided.
if (!viewOnly && this is PaynymInterface && codesToCheck.isNotEmpty) {
await (this as PaynymInterface).checkForNotificationTransactionsTo(
codesToCheck,
);
// check utxos again for notification outputs
await updateUTXOs();
}
_fireRefreshPercentChange(0.80);

// await getAllTxsToWatch();

_fireRefreshPercentChange(0.90);

await updateBalance();

_fireRefreshPercentChange(1.0);

completer.complete();
} catch (error, strace) {
Expand All @@ -750,6 +733,98 @@ abstract class Wallet<T extends CryptoCurrency> {
}
}

Future<void> _doRefreshWork(bool viewOnly) async {
// add some small buffer before making calls.
// this can probably be removed in the future but was added as a
// debugging feature
await Future<void>.delayed(const Duration(milliseconds: 300));

// TODO: [prio=low] handle this differently. Extra modification of this file for coin specific functionality should be avoided.
final Set<String> codesToCheck = {};
if (this is PaynymInterface && !viewOnly) {
// isSegwit does not matter here at all
final myCode = await (this as PaynymInterface).getPaymentCode(
isSegwit: false,
);

final nym = await PaynymIsApi().nym(myCode.toString());
if (nym.value != null) {
for (final follower in nym.value!.followers) {
codesToCheck.add(follower.code);
}
for (final following in nym.value!.following) {
codesToCheck.add(following.code);
}
}
}

_fireRefreshPercentChange(0);
await updateChainHeight();

if (this is BitcoinFrostWallet) {
await (this as BitcoinFrostWallet).lookAhead();
}

_fireRefreshPercentChange(0.1);

// TODO: [prio=low] handle this differently. Extra modification of this file for coin specific functionality should be avoided.
if (this is MultiAddressInterface) {
if (info.otherData[WalletInfoKeys.reuseAddress] != true) {
await (this as MultiAddressInterface)
.checkReceivingAddressForTransactions();
}
}

_fireRefreshPercentChange(0.2);

// TODO: [prio=low] handle this differently. Extra modification of this file for coin specific functionality should be avoided.
if (this is MultiAddressInterface) {
if (info.otherData[WalletInfoKeys.reuseAddress] != true) {
await (this as MultiAddressInterface)
.checkChangeAddressForTransactions();
}
}
_fireRefreshPercentChange(0.3);
if (this is SparkInterface && !viewOnly) {
// this should be called before updateTransactions()
await (this as SparkInterface).refreshSparkData((0.3, 0.6));
}

if (this is NamecoinWallet) {
await updateUTXOs();
_fireRefreshPercentChange(0.6);
await (this as NamecoinWallet).checkAutoRegisterNameNewOutputs();
_fireRefreshPercentChange(0.70);
await updateTransactions();
} else {
final fetchFuture = updateTransactions();
_fireRefreshPercentChange(0.6);
final utxosRefreshFuture = updateUTXOs();
await utxosRefreshFuture;
_fireRefreshPercentChange(0.65);
await fetchFuture;
_fireRefreshPercentChange(0.70);
}

// TODO: [prio=low] handle this differently. Extra modification of this file for coin specific functionality should be avoided.
if (!viewOnly && this is PaynymInterface && codesToCheck.isNotEmpty) {
await (this as PaynymInterface).checkForNotificationTransactionsTo(
codesToCheck,
);
// check utxos again for notification outputs
await updateUTXOs();
}
_fireRefreshPercentChange(0.80);

// await getAllTxsToWatch();

_fireRefreshPercentChange(0.90);

await updateBalance();

_fireRefreshPercentChange(1.0);
}

Future<void> exit() async {
Logging.instance.i("exit called on $walletId");
_periodicRefreshTimer?.cancel();
Expand Down