forked from bkinnightskytw/code_work_spawner
fix: keep polling after network failures (#8)
This commit is contained in:
parent
c3a1bf961a
commit
c02fb0e62a
|
|
@ -177,9 +177,11 @@ class IssueAssistantApp {
|
||||||
'poll tick source=$source projects=${projects.map((p) => p.key).join(",")} '
|
'poll tick source=$source projects=${projects.map((p) => p.key).join(",")} '
|
||||||
'duration_ms=${stopwatch.elapsedMilliseconds} error=$error',
|
'duration_ms=${stopwatch.elapsedMilliseconds} error=$error',
|
||||||
);
|
);
|
||||||
_cancelScheduler();
|
if (_runCompleter != null) {
|
||||||
if (!(_runCompleter?.isCompleted ?? true)) {
|
_logError(
|
||||||
_runCompleter!.completeError(error, stackTrace);
|
'poll tick source=$source will_retry=true '
|
||||||
|
'next_run=scheduled stack_trace=$stackTrace',
|
||||||
|
);
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
for (final state in dueStates) {
|
for (final state in dueStates) {
|
||||||
|
|
|
||||||
|
|
@ -1580,4 +1580,122 @@ projects:
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/// ```gherkin
|
||||||
|
/// Feature: Continuous issue polling
|
||||||
|
///
|
||||||
|
/// As a maintainer running the long-lived poller
|
||||||
|
/// I want transient tracker network failures to be logged without stopping the process
|
||||||
|
/// So that the assistant can recover automatically when connectivity returns
|
||||||
|
/// ```
|
||||||
|
group('IssueAssistantApp.run', () {
|
||||||
|
/// ```gherkin
|
||||||
|
/// Scenario: Continue polling after a transient GitHub search failure
|
||||||
|
/// Given a temporary project checkout that can be used as the local repo path
|
||||||
|
/// And a gh script that fails the first search request and succeeds afterwards
|
||||||
|
/// And an orchestrator-style config with a short poll interval
|
||||||
|
/// When the long-running app starts and enough time passes for a retry
|
||||||
|
/// Then the app logs the failed poll without stopping entirely
|
||||||
|
/// And it performs another search request after the transient failure
|
||||||
|
/// ```
|
||||||
|
test('Continue polling after a transient GitHub search failure', () async {
|
||||||
|
// Given a temporary project checkout that can be used as the local repo path.
|
||||||
|
final sandbox = await Directory.systemTemp.createTemp('cws-run-retry-');
|
||||||
|
final repoDir = await Directory(p.join(sandbox.path, 'repo')).create();
|
||||||
|
await initGitRepo(repoDir);
|
||||||
|
|
||||||
|
// And a gh script that fails the first search request and succeeds afterwards.
|
||||||
|
final ghScript = File(p.join(sandbox.path, 'gh'));
|
||||||
|
final ghLog = File(p.join(sandbox.path, 'gh-log.jsonl'));
|
||||||
|
final failureState = File(p.join(sandbox.path, 'search.failed.once'));
|
||||||
|
await writeFlakySearchGhScript(
|
||||||
|
ghScript: ghScript,
|
||||||
|
failureStateFile: failureState,
|
||||||
|
issueListResponse: [
|
||||||
|
{
|
||||||
|
'number': 81,
|
||||||
|
'title': 'Recovered issue polling',
|
||||||
|
'body': 'Retry should succeed.',
|
||||||
|
'state': 'open',
|
||||||
|
'html_url': 'https://example.test/issues/81',
|
||||||
|
'updated_at': '2026-04-05T13:45:00Z',
|
||||||
|
'user': {'login': 'reporter'},
|
||||||
|
'labels': const [],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
commentResponses: {
|
||||||
|
81: [
|
||||||
|
{
|
||||||
|
'id': 810,
|
||||||
|
'body': 'connectivity recovered',
|
||||||
|
'html_url': 'https://example.test/issues/81#issuecomment-810',
|
||||||
|
'created_at': '2026-04-05T13:45:00Z',
|
||||||
|
'updated_at': '2026-04-05T13:45:00Z',
|
||||||
|
'user': {'login': 'reporter'},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
ghLog: ghLog,
|
||||||
|
);
|
||||||
|
|
||||||
|
// And an orchestrator-style config with a short poll interval.
|
||||||
|
final configFile = File(p.join(sandbox.path, 'agent-orchestrator.yaml'));
|
||||||
|
await configFile.writeAsString('''
|
||||||
|
defaults:
|
||||||
|
issueAssistant:
|
||||||
|
enabled: true
|
||||||
|
pollInterval: 1s
|
||||||
|
projects:
|
||||||
|
sample:
|
||||||
|
repo: owner/sample
|
||||||
|
path: ${repoDir.path}
|
||||||
|
''');
|
||||||
|
final app = await IssueAssistantApp.open(
|
||||||
|
config: await AppConfig.load(configFile.path),
|
||||||
|
databasePath: p.join(sandbox.path, 'state.sqlite3'),
|
||||||
|
ghCommand: ghScript.path,
|
||||||
|
);
|
||||||
|
final logMessages = <String>[];
|
||||||
|
final previousLevel = Logger.root.level;
|
||||||
|
Logger.root.level = Level.INFO;
|
||||||
|
final subscription = Logger.root.onRecord.listen((record) {
|
||||||
|
logMessages.add(record.message);
|
||||||
|
});
|
||||||
|
|
||||||
|
// When the long-running app starts and enough time passes for a retry.
|
||||||
|
final runFuture = app.run();
|
||||||
|
await Future<void>.delayed(const Duration(milliseconds: 2600));
|
||||||
|
await app.close();
|
||||||
|
await runFuture;
|
||||||
|
await subscription.cancel();
|
||||||
|
Logger.root.level = previousLevel;
|
||||||
|
|
||||||
|
// Then the app logs the failed poll without stopping entirely.
|
||||||
|
expect(
|
||||||
|
logMessages.any(
|
||||||
|
(message) =>
|
||||||
|
message.contains('poll tick source=startup') &&
|
||||||
|
message.contains('error=ProcessException'),
|
||||||
|
),
|
||||||
|
isTrue,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
logMessages.any((message) => message.contains('will_retry=true')),
|
||||||
|
isTrue,
|
||||||
|
);
|
||||||
|
|
||||||
|
// And it performs another search request after the transient failure.
|
||||||
|
final logLines = await ghLog.readAsLines();
|
||||||
|
expect(
|
||||||
|
logLines
|
||||||
|
.where(
|
||||||
|
(line) =>
|
||||||
|
line ==
|
||||||
|
'api -X GET search/issues -f q=repo:owner/sample is:issue state:open -f sort=updated -f order=asc -f per_page=100 -f page=1',
|
||||||
|
)
|
||||||
|
.length,
|
||||||
|
greaterThanOrEqualTo(2),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -182,6 +182,83 @@ Future<void> writeFakeGhScript({
|
||||||
await Process.run('chmod', ['+x', ghScript.path]);
|
await Process.run('chmod', ['+x', ghScript.path]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Future<void> writeFlakySearchGhScript({
|
||||||
|
required File ghScript,
|
||||||
|
required File failureStateFile,
|
||||||
|
required List<Map<String, Object?>> issueListResponse,
|
||||||
|
required Map<int, List<Map<String, Object?>>> commentResponses,
|
||||||
|
File? ghLog,
|
||||||
|
}) async {
|
||||||
|
final searchItems = issueListResponse
|
||||||
|
.map(
|
||||||
|
(issue) => <String, Object?>{
|
||||||
|
...issue,
|
||||||
|
'repository_url': 'https://api.github.com/repos/owner/sample',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.toList(growable: false);
|
||||||
|
final searchResponse = jsonEncode({
|
||||||
|
'total_count': searchItems.length,
|
||||||
|
'incomplete_results': false,
|
||||||
|
'items': searchItems,
|
||||||
|
});
|
||||||
|
|
||||||
|
final buffer = StringBuffer()
|
||||||
|
..writeln('#!/usr/bin/env bash')
|
||||||
|
..writeln('set -euo pipefail');
|
||||||
|
|
||||||
|
if (ghLog != null) {
|
||||||
|
buffer.writeln(
|
||||||
|
r'''printf '%s\n' "$*" >> '''
|
||||||
|
'"${ghLog.path}"',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
buffer
|
||||||
|
..writeln(r'''if [[ "$1" == "api" && "$2" == "user" ]]; then''')
|
||||||
|
..writeln(" cat <<'JSON'")
|
||||||
|
..writeln(jsonEncode({'login': 'test-user'}))
|
||||||
|
..writeln('JSON')
|
||||||
|
..writeln(' exit 0')
|
||||||
|
..writeln('fi');
|
||||||
|
|
||||||
|
buffer
|
||||||
|
..writeln(r'''if [[ "$1" == "api" && "$4" == "search/issues" ]]; then''')
|
||||||
|
..writeln(' if [[ ! -f "${failureStateFile.path}" ]]; then')
|
||||||
|
..writeln(' touch "${failureStateFile.path}"')
|
||||||
|
..writeln(r''' echo "error connecting to api.github.com" >&2''')
|
||||||
|
..writeln(
|
||||||
|
r''' echo "check your internet connection or https://githubstatus.com" >&2''',
|
||||||
|
)
|
||||||
|
..writeln(' exit 1')
|
||||||
|
..writeln(r''' fi''')
|
||||||
|
..writeln(" cat <<'JSON'")
|
||||||
|
..writeln(searchResponse)
|
||||||
|
..writeln('JSON')
|
||||||
|
..writeln(' exit 0')
|
||||||
|
..writeln('fi');
|
||||||
|
|
||||||
|
for (final entry in commentResponses.entries) {
|
||||||
|
buffer
|
||||||
|
..writeln(
|
||||||
|
'if [[ "\$1" == "api" && "\$4" == '
|
||||||
|
'"repos/owner/sample/issues/${entry.key}/comments?per_page=100" ]]; then',
|
||||||
|
)
|
||||||
|
..writeln(" cat <<'JSON'")
|
||||||
|
..writeln(jsonEncode(entry.value))
|
||||||
|
..writeln('JSON')
|
||||||
|
..writeln(' exit 0')
|
||||||
|
..writeln('fi');
|
||||||
|
}
|
||||||
|
|
||||||
|
buffer
|
||||||
|
..writeln(r'''echo "unexpected gh args: $*" >&2''')
|
||||||
|
..writeln('exit 1');
|
||||||
|
|
||||||
|
await ghScript.writeAsString(buffer.toString());
|
||||||
|
await Process.run('chmod', ['+x', ghScript.path]);
|
||||||
|
}
|
||||||
|
|
||||||
Future<void> writeFakeTeaScript({
|
Future<void> writeFakeTeaScript({
|
||||||
required File teaScript,
|
required File teaScript,
|
||||||
required Map<String, List<Map<String, Object?>>> issueListResponsesByRepo,
|
required Map<String, List<Map<String, Object?>>> issueListResponsesByRepo,
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue