feat: enable aggregate adjacent github polling
This commit is contained in:
parent
893649b716
commit
200992d67b
|
|
@ -12,6 +12,6 @@ account. Those markers support template variables such as `{{gh_login}}`,
|
||||||
`{{comment_tag}}`, and `{{fingerprint}}`. Set either template to an empty
|
`{{comment_tag}}`, and `{{fingerprint}}`. Set either template to an empty
|
||||||
string to disable that section.
|
string to disable that section.
|
||||||
|
|
||||||
Start from [`examples/simple-github.yaml`](examples/simple-github.yaml). This
|
Start from [`examples/simple-github.yaml`](simple-github.yaml). This
|
||||||
repo uses `dataDir`, `worktreeDir`, and `projects`, and uses `worktreeDir` for
|
repo uses `dataDir`, `worktreeDir`, and `projects`, and uses `worktreeDir` for
|
||||||
bug-verification worktrees.
|
bug-verification worktrees.
|
||||||
|
|
|
||||||
|
|
@ -8,31 +8,56 @@ class GitHubClient {
|
||||||
|
|
||||||
final String ghCommand;
|
final String ghCommand;
|
||||||
Future<String?>? _authenticatedLoginFuture;
|
Future<String?>? _authenticatedLoginFuture;
|
||||||
|
static const int _pageSize = 100;
|
||||||
|
|
||||||
Future<List<GitHubIssueSummary>> fetchUpdatedIssues({
|
Future<List<GitHubIssueSummary>> fetchUpdatedIssuesForRepos({
|
||||||
required RepositorySlug repo,
|
required Iterable<RepositorySlug> repos,
|
||||||
required DateTime? since,
|
required DateTime? since,
|
||||||
}) async {
|
}) async {
|
||||||
final query = <String, String>{
|
final repoList = repos.toList(growable: false);
|
||||||
'state': 'all',
|
if (repoList.isEmpty) {
|
||||||
'sort': 'updated',
|
return const <GitHubIssueSummary>[];
|
||||||
'direction': 'asc',
|
}
|
||||||
'per_page': '100',
|
|
||||||
if (since != null) 'since': since.toUtc().toIso8601String(),
|
|
||||||
};
|
|
||||||
|
|
||||||
final json = await _runJson([
|
final queryParts = <String>[
|
||||||
'api',
|
for (final repo in repoList) 'repo:${repo.fullName}',
|
||||||
'-X',
|
'is:issue',
|
||||||
'GET',
|
if (since != null) 'updated:>=${since.toUtc().toIso8601String()}',
|
||||||
_buildApiPath(repo, ['issues'], query),
|
];
|
||||||
]);
|
|
||||||
|
|
||||||
return (json as List<dynamic>)
|
final issues = <GitHubIssueSummary>[];
|
||||||
.cast<Map<String, dynamic>>()
|
for (var page = 1; true; page += 1) {
|
||||||
.where((item) => !item.containsKey('pull_request'))
|
final json = await _runJson([
|
||||||
.map(GitHubIssueSummary.fromJson)
|
'api',
|
||||||
.toList(growable: false);
|
'-X',
|
||||||
|
'GET',
|
||||||
|
'search/issues',
|
||||||
|
'-f',
|
||||||
|
'q=${queryParts.join(' ')}',
|
||||||
|
'-f',
|
||||||
|
'sort=updated',
|
||||||
|
'-f',
|
||||||
|
'order=asc',
|
||||||
|
'-f',
|
||||||
|
'per_page=$_pageSize',
|
||||||
|
'-f',
|
||||||
|
'page=$page',
|
||||||
|
]);
|
||||||
|
final items =
|
||||||
|
((json as Map<String, dynamic>)['items'] as List<dynamic>? ??
|
||||||
|
const [])
|
||||||
|
.cast<Map<String, dynamic>>();
|
||||||
|
issues.addAll(
|
||||||
|
items
|
||||||
|
.where((item) => !item.containsKey('pull_request'))
|
||||||
|
.map(GitHubIssueSummary.fromSearchJson),
|
||||||
|
);
|
||||||
|
if (items.length < _pageSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return issues;
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<IssueThread> fetchThread({
|
Future<IssueThread> fetchThread({
|
||||||
|
|
@ -129,6 +154,7 @@ class GitHubClient {
|
||||||
|
|
||||||
class GitHubIssueSummary {
|
class GitHubIssueSummary {
|
||||||
GitHubIssueSummary({
|
GitHubIssueSummary({
|
||||||
|
required this.repoSlug,
|
||||||
required this.number,
|
required this.number,
|
||||||
required this.title,
|
required this.title,
|
||||||
required this.body,
|
required this.body,
|
||||||
|
|
@ -139,6 +165,7 @@ class GitHubIssueSummary {
|
||||||
required this.labels,
|
required this.labels,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
final RepositorySlug repoSlug;
|
||||||
final int number;
|
final int number;
|
||||||
final String title;
|
final String title;
|
||||||
final String body;
|
final String body;
|
||||||
|
|
@ -148,9 +175,58 @@ class GitHubIssueSummary {
|
||||||
final String userLogin;
|
final String userLogin;
|
||||||
final List<String> labels;
|
final List<String> labels;
|
||||||
|
|
||||||
|
factory GitHubIssueSummary.fromRepositoryJson(
|
||||||
|
RepositorySlug repo,
|
||||||
|
Map<String, dynamic> json,
|
||||||
|
) {
|
||||||
|
final labelsNode = json['labels'] as List<dynamic>? ?? const [];
|
||||||
|
return GitHubIssueSummary(
|
||||||
|
repoSlug: repo,
|
||||||
|
number: json['number'] as int,
|
||||||
|
title: json['title'] as String? ?? '',
|
||||||
|
body: json['body'] as String? ?? '',
|
||||||
|
state: json['state'] as String? ?? 'open',
|
||||||
|
url: json['html_url'] as String? ?? '',
|
||||||
|
updatedAt: DateTime.parse(json['updated_at'] as String),
|
||||||
|
userLogin:
|
||||||
|
(json['user'] as Map<String, dynamic>? ?? const {})['login']
|
||||||
|
as String? ??
|
||||||
|
'unknown',
|
||||||
|
labels: labelsNode
|
||||||
|
.map((label) => (label as Map<String, dynamic>)['name'].toString())
|
||||||
|
.toList(growable: false),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
factory GitHubIssueSummary.fromSearchJson(Map<String, dynamic> json) {
|
||||||
|
final repositoryUrl = json['repository_url'] as String?;
|
||||||
|
if (repositoryUrl == null || repositoryUrl.isEmpty) {
|
||||||
|
throw const FormatException(
|
||||||
|
'Missing repository_url in GitHub search result item.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return GitHubIssueSummary.fromRepositoryJson(
|
||||||
|
_parseRepositorySlugFromRepositoryUrl(repositoryUrl),
|
||||||
|
json,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
static RepositorySlug _parseRepositorySlugFromRepositoryUrl(String value) {
|
||||||
|
final uri = Uri.parse(value);
|
||||||
|
final segments = uri.pathSegments;
|
||||||
|
final reposIndex = segments.indexOf('repos');
|
||||||
|
if (reposIndex == -1 || reposIndex + 2 >= segments.length) {
|
||||||
|
throw FormatException(
|
||||||
|
'Invalid repository_url in GitHub search result item: "$value"',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return RepositorySlug(segments[reposIndex + 1], segments[reposIndex + 2]);
|
||||||
|
}
|
||||||
|
|
||||||
factory GitHubIssueSummary.fromJson(Map<String, dynamic> json) {
|
factory GitHubIssueSummary.fromJson(Map<String, dynamic> json) {
|
||||||
final labelsNode = json['labels'] as List<dynamic>? ?? const [];
|
final labelsNode = json['labels'] as List<dynamic>? ?? const [];
|
||||||
return GitHubIssueSummary(
|
return GitHubIssueSummary(
|
||||||
|
repoSlug: RepositorySlug('', ''),
|
||||||
number: json['number'] as int,
|
number: json['number'] as int,
|
||||||
title: json['title'] as String? ?? '',
|
title: json['title'] as String? ?? '',
|
||||||
body: json['body'] as String? ?? '',
|
body: json['body'] as String? ?? '',
|
||||||
|
|
@ -168,6 +244,7 @@ class GitHubIssueSummary {
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String, Object?> toJson() => {
|
Map<String, Object?> toJson() => {
|
||||||
|
'repo': repoSlug.fullName,
|
||||||
'number': number,
|
'number': number,
|
||||||
'title': title,
|
'title': title,
|
||||||
'body': body,
|
'body': body,
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ import 'workspace_manager.dart';
|
||||||
|
|
||||||
class IssueAssistantApp {
|
class IssueAssistantApp {
|
||||||
static final Logger _logger = Logger('code_work_spawner.app');
|
static final Logger _logger = Logger('code_work_spawner.app');
|
||||||
|
static const Duration _pollCoalescingWindow = Duration(seconds: 2);
|
||||||
|
|
||||||
IssueAssistantApp._({
|
IssueAssistantApp._({
|
||||||
required this.config,
|
required this.config,
|
||||||
|
|
@ -29,6 +30,7 @@ class IssueAssistantApp {
|
||||||
final WorkspaceManager workspaceManager;
|
final WorkspaceManager workspaceManager;
|
||||||
final Map<String, _ProjectPollState> _projectPollStates = {};
|
final Map<String, _ProjectPollState> _projectPollStates = {};
|
||||||
final Set<Future<void>> _activePolls = <Future<void>>{};
|
final Set<Future<void>> _activePolls = <Future<void>>{};
|
||||||
|
Timer? _schedulerTimer;
|
||||||
Completer<void>? _runCompleter;
|
Completer<void>? _runCompleter;
|
||||||
bool _isClosing = false;
|
bool _isClosing = false;
|
||||||
bool _isClosed = false;
|
bool _isClosed = false;
|
||||||
|
|
@ -67,33 +69,37 @@ class IssueAssistantApp {
|
||||||
final completer = Completer<void>();
|
final completer = Completer<void>();
|
||||||
_runCompleter = completer;
|
_runCompleter = completer;
|
||||||
|
|
||||||
|
final now = DateTime.now();
|
||||||
for (final project in enabledProjects) {
|
for (final project in enabledProjects) {
|
||||||
final state = _projectPollStates.putIfAbsent(
|
_projectPollStates.putIfAbsent(
|
||||||
project.key,
|
project.key,
|
||||||
() => _ProjectPollState(project: project),
|
() => _ProjectPollState(project: project, nextDueAt: now),
|
||||||
);
|
);
|
||||||
unawaited(_triggerProjectPoll(project, source: 'startup'));
|
|
||||||
state.timer = Timer.periodic(project.issueAssistant.pollInterval, (_) {
|
|
||||||
unawaited(_triggerProjectPoll(project, source: 'interval'));
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
unawaited(_triggerDueProjectPolls(source: 'startup'));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await completer.future;
|
await completer.future;
|
||||||
} finally {
|
} finally {
|
||||||
_cancelProjectPollTimers();
|
_cancelScheduler();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<void> runOnce() async {
|
Future<void> runOnce() async {
|
||||||
_log('starting poll cycle projects=${config.projects.length}');
|
_log('starting poll cycle projects=${config.projects.length}');
|
||||||
|
final enabledProjects = <ProjectConfig>[];
|
||||||
for (final project in config.projects.values) {
|
for (final project in config.projects.values) {
|
||||||
if (!project.issueAssistant.enabled) {
|
if (!project.issueAssistant.enabled) {
|
||||||
_log('skip project=${project.key} reason=issue_assistant_disabled');
|
_log('skip project=${project.key} reason=issue_assistant_disabled');
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
await _pollProject(project);
|
enabledProjects.add(project);
|
||||||
}
|
}
|
||||||
|
if (enabledProjects.isEmpty) {
|
||||||
|
_log('no enabled projects configured');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await _pollProjects(enabledProjects);
|
||||||
_log('poll cycle completed');
|
_log('poll cycle completed');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -103,7 +109,7 @@ class IssueAssistantApp {
|
||||||
}
|
}
|
||||||
|
|
||||||
_isClosing = true;
|
_isClosing = true;
|
||||||
_cancelProjectPollTimers();
|
_cancelScheduler();
|
||||||
_completeRunIfPending();
|
_completeRunIfPending();
|
||||||
await Future.wait(_activePolls.toList(growable: false));
|
await Future.wait(_activePolls.toList(growable: false));
|
||||||
await database.close();
|
await database.close();
|
||||||
|
|
@ -111,50 +117,63 @@ class IssueAssistantApp {
|
||||||
_isClosing = false;
|
_isClosing = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<void> _triggerProjectPoll(
|
Future<void> _triggerDueProjectPolls({required String source}) async {
|
||||||
ProjectConfig project, {
|
|
||||||
required String source,
|
|
||||||
}) async {
|
|
||||||
if (_isClosing || _isClosed) {
|
if (_isClosing || _isClosed) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
final state = _projectPollStates.putIfAbsent(
|
final now = DateTime.now();
|
||||||
project.key,
|
final dueStates = _projectPollStates.values
|
||||||
() => _ProjectPollState(project: project),
|
.where((state) => !state.isRunning && !state.nextDueAt.isAfter(now))
|
||||||
);
|
.toList(growable: false);
|
||||||
if (state.isRunning) {
|
if (dueStates.isEmpty) {
|
||||||
_log(
|
_scheduleNextPoll();
|
||||||
'skip poll project=${project.key} reason=already_running source=$source',
|
|
||||||
);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
late final Future<void> pollFuture;
|
late final Future<void> pollFuture;
|
||||||
pollFuture = () async {
|
pollFuture = () async {
|
||||||
state.isRunning = true;
|
for (final state in dueStates) {
|
||||||
|
state.isRunning = true;
|
||||||
|
}
|
||||||
final stopwatch = Stopwatch()..start();
|
final stopwatch = Stopwatch()..start();
|
||||||
_log('poll tick project=${project.key} source=$source');
|
final projects = dueStates
|
||||||
|
.map((state) => state.project)
|
||||||
|
.toList(growable: false);
|
||||||
|
_log(
|
||||||
|
'poll tick source=$source projects=${projects.map((p) => p.key).join(",")}',
|
||||||
|
);
|
||||||
try {
|
try {
|
||||||
await _pollProject(project);
|
await _pollProjects(projects);
|
||||||
stopwatch.stop();
|
stopwatch.stop();
|
||||||
|
final batchCompletedAt = DateTime.now();
|
||||||
|
for (final state in dueStates) {
|
||||||
|
while (!state.nextDueAt.isAfter(batchCompletedAt)) {
|
||||||
|
state.nextDueAt = state.nextDueAt.add(
|
||||||
|
state.project.issueAssistant.pollInterval,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
_log(
|
_log(
|
||||||
'poll tick project=${project.key} source=$source '
|
'poll tick source=$source projects=${projects.map((p) => p.key).join(",")} '
|
||||||
'duration_ms=${stopwatch.elapsedMilliseconds}',
|
'duration_ms=${stopwatch.elapsedMilliseconds}',
|
||||||
);
|
);
|
||||||
} catch (error, stackTrace) {
|
} catch (error, stackTrace) {
|
||||||
stopwatch.stop();
|
stopwatch.stop();
|
||||||
_logError(
|
_logError(
|
||||||
'poll tick project=${project.key} source=$source '
|
'poll tick source=$source projects=${projects.map((p) => p.key).join(",")} '
|
||||||
'duration_ms=${stopwatch.elapsedMilliseconds} error=$error',
|
'duration_ms=${stopwatch.elapsedMilliseconds} error=$error',
|
||||||
);
|
);
|
||||||
_cancelProjectPollTimers();
|
_cancelScheduler();
|
||||||
if (!(_runCompleter?.isCompleted ?? true)) {
|
if (!(_runCompleter?.isCompleted ?? true)) {
|
||||||
_runCompleter!.completeError(error, stackTrace);
|
_runCompleter!.completeError(error, stackTrace);
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
state.isRunning = false;
|
for (final state in dueStates) {
|
||||||
|
state.isRunning = false;
|
||||||
|
}
|
||||||
_activePolls.remove(pollFuture);
|
_activePolls.remove(pollFuture);
|
||||||
|
_scheduleNextPoll();
|
||||||
}
|
}
|
||||||
}();
|
}();
|
||||||
|
|
||||||
|
|
@ -162,11 +181,36 @@ class IssueAssistantApp {
|
||||||
await pollFuture;
|
await pollFuture;
|
||||||
}
|
}
|
||||||
|
|
||||||
void _cancelProjectPollTimers() {
|
void _cancelScheduler() {
|
||||||
for (final state in _projectPollStates.values) {
|
_schedulerTimer?.cancel();
|
||||||
state.timer?.cancel();
|
_schedulerTimer = null;
|
||||||
state.timer = null;
|
}
|
||||||
|
|
||||||
|
void _scheduleNextPoll() {
|
||||||
|
if (_isClosing || _isClosed || _runCompleter == null) {
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
_cancelScheduler();
|
||||||
|
|
||||||
|
DateTime? nextDueAt;
|
||||||
|
for (final state in _projectPollStates.values) {
|
||||||
|
if (state.isRunning) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (nextDueAt == null || state.nextDueAt.isBefore(nextDueAt)) {
|
||||||
|
nextDueAt = state.nextDueAt;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (nextDueAt == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
final scheduledAt = nextDueAt.add(_pollCoalescingWindow);
|
||||||
|
final delay = scheduledAt.difference(DateTime.now());
|
||||||
|
_schedulerTimer = Timer(
|
||||||
|
delay.isNegative ? Duration.zero : delay,
|
||||||
|
() => unawaited(_triggerDueProjectPolls(source: 'interval')),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
void _completeRunIfPending() {
|
void _completeRunIfPending() {
|
||||||
|
|
@ -176,59 +220,95 @@ class IssueAssistantApp {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<void> _pollProject(ProjectConfig project) async {
|
Future<void> _pollProjects(List<ProjectConfig> projects) async {
|
||||||
final pollRunId = await database.startPollRun(project.key);
|
final pollRunIds = <String, int>{};
|
||||||
try {
|
final projectsByRepo = <String, ProjectConfig>{
|
||||||
|
for (final project in projects) project.repoSlug.fullName: project,
|
||||||
|
};
|
||||||
|
final sinceByProject = <String, DateTime?>{};
|
||||||
|
final latestSeenByProject = <String, DateTime?>{};
|
||||||
|
|
||||||
|
for (final project in projects) {
|
||||||
|
pollRunIds[project.key] = await database.startPollRun(project.key);
|
||||||
final repoDirectory = Directory(project.path);
|
final repoDirectory = Directory(project.path);
|
||||||
final repoExists = await repoDirectory.exists();
|
final repoExists = await repoDirectory.exists();
|
||||||
final projectState = await database.findProjectState(project.key);
|
final projectState = await database.findProjectState(project.key);
|
||||||
final since = projectState?.lastSeenUpdatedAt == null
|
final since = projectState?.lastSeenUpdatedAt == null
|
||||||
? null
|
? null
|
||||||
: DateTime.parse(projectState!.lastSeenUpdatedAt!);
|
: DateTime.parse(projectState!.lastSeenUpdatedAt!);
|
||||||
|
sinceByProject[project.key] = since;
|
||||||
|
latestSeenByProject[project.key] = since;
|
||||||
_log(
|
_log(
|
||||||
'poll project=${project.key} repo=${project.repo} '
|
'poll project=${project.key} repo=${project.repo} '
|
||||||
'path=${project.path} path_exists=$repoExists '
|
'path=${project.path} path_exists=$repoExists '
|
||||||
'since=${since?.toUtc().toIso8601String() ?? "initial"}',
|
'since=${since?.toUtc().toIso8601String() ?? "initial"}',
|
||||||
);
|
);
|
||||||
final issues = await githubClient.fetchUpdatedIssues(
|
}
|
||||||
repo: project.repoSlug,
|
|
||||||
since: since,
|
|
||||||
);
|
|
||||||
_log('project=${project.key} fetched_issues=${issues.length}');
|
|
||||||
|
|
||||||
final latestSeen = issues.map((issue) => issue.updatedAt).fold<DateTime?>(
|
try {
|
||||||
since,
|
final batchSince = sinceByProject.values
|
||||||
(current, next) {
|
.whereType<DateTime>()
|
||||||
if (current == null || next.isAfter(current)) {
|
.fold<DateTime?>(null, (current, next) {
|
||||||
return next;
|
if (current == null || next.isBefore(current)) {
|
||||||
}
|
return next;
|
||||||
return current;
|
}
|
||||||
},
|
return current;
|
||||||
|
});
|
||||||
|
final issues = await githubClient.fetchUpdatedIssuesForRepos(
|
||||||
|
repos: projects.map((project) => project.repoSlug),
|
||||||
|
since: batchSince,
|
||||||
);
|
);
|
||||||
|
final issuesByProject = <String, List<GitHubIssueSummary>>{
|
||||||
|
for (final project in projects) project.key: <GitHubIssueSummary>[],
|
||||||
|
};
|
||||||
|
for (final issue in issues) {
|
||||||
|
final project = projectsByRepo[issue.repoSlug.fullName];
|
||||||
|
if (project == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
final since = sinceByProject[project.key];
|
||||||
|
if (since != null && !issue.updatedAt.isAfter(since)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
issuesByProject[project.key]!.add(issue);
|
||||||
|
final latestSeen = latestSeenByProject[project.key];
|
||||||
|
if (latestSeen == null || issue.updatedAt.isAfter(latestSeen)) {
|
||||||
|
latestSeenByProject[project.key] = issue.updatedAt;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
_log(
|
for (final project in projects) {
|
||||||
'project=${project.key} issue_queue=${issues.length} '
|
final projectIssues = issuesByProject[project.key]!;
|
||||||
'max_concurrent=${project.issueAssistant.maxConcurrentIssues}',
|
_log('project=${project.key} fetched_issues=${projectIssues.length}');
|
||||||
);
|
_log(
|
||||||
await _processIssuesConcurrently(project, issues);
|
'project=${project.key} issue_queue=${projectIssues.length} '
|
||||||
|
'max_concurrent=${project.issueAssistant.maxConcurrentIssues}',
|
||||||
|
);
|
||||||
|
await _processIssuesConcurrently(project, projectIssues);
|
||||||
|
|
||||||
await database.upsertProjectState(
|
await database.upsertProjectState(
|
||||||
projectKey: project.key,
|
projectKey: project.key,
|
||||||
repo: project.repo,
|
repo: project.repo,
|
||||||
lastSeenUpdatedAt: latestSeen,
|
lastSeenUpdatedAt: latestSeenByProject[project.key],
|
||||||
);
|
);
|
||||||
await database.finishPollRun(pollRunId: pollRunId, success: true);
|
await database.finishPollRun(
|
||||||
_log(
|
pollRunId: pollRunIds.remove(project.key)!,
|
||||||
'poll project=${project.key} completed latest_seen='
|
success: true,
|
||||||
'${latestSeen?.toUtc().toIso8601String() ?? "unchanged"}',
|
);
|
||||||
);
|
_log(
|
||||||
|
'poll project=${project.key} completed latest_seen='
|
||||||
|
'${latestSeenByProject[project.key]?.toUtc().toIso8601String() ?? "unchanged"}',
|
||||||
|
);
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
await database.finishPollRun(
|
for (final entry in pollRunIds.entries) {
|
||||||
pollRunId: pollRunId,
|
await database.finishPollRun(
|
||||||
success: false,
|
pollRunId: entry.value,
|
||||||
error: error.toString(),
|
success: false,
|
||||||
);
|
error: error.toString(),
|
||||||
_log('poll project=${project.key} failed error=$error');
|
);
|
||||||
|
_log('poll project=${entry.key} failed error=$error');
|
||||||
|
}
|
||||||
rethrow;
|
rethrow;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -624,9 +704,9 @@ class IssueAssistantApp {
|
||||||
}
|
}
|
||||||
|
|
||||||
class _ProjectPollState {
|
class _ProjectPollState {
|
||||||
_ProjectPollState({required this.project});
|
_ProjectPollState({required this.project, required this.nextDueAt});
|
||||||
|
|
||||||
final ProjectConfig project;
|
final ProjectConfig project;
|
||||||
Timer? timer;
|
DateTime nextDueAt;
|
||||||
bool isRunning = false;
|
bool isRunning = false;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1129,5 +1129,163 @@ projects:
|
||||||
expect(jobs, hasLength(1));
|
expect(jobs, hasLength(1));
|
||||||
expect(jobs.single.status, JobStatus.skipped);
|
expect(jobs.single.status, JobStatus.skipped);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/// ```gherkin
|
||||||
|
/// Scenario: Aggregate one search request across multiple enabled projects
|
||||||
|
/// Given two temporary project checkouts that can be used as local repo paths
|
||||||
|
/// And GitHub issue data containing an explicit assistant mention in each repository
|
||||||
|
/// And a responder that emits a planning reply
|
||||||
|
/// And an orchestrator-style config with both repositories enabled
|
||||||
|
/// When the app processes one polling cycle
|
||||||
|
/// Then it fetches issue updates through one aggregate GitHub search request
|
||||||
|
/// And it still posts one reply in each repository
|
||||||
|
/// ```
|
||||||
|
test('Aggregate one search request across multiple enabled projects', () async {
|
||||||
|
// Given two temporary project checkouts that can be used as local repo paths.
|
||||||
|
final sandbox = await Directory.systemTemp.createTemp(
|
||||||
|
'cws-aggregate-search-',
|
||||||
|
);
|
||||||
|
final repoOneDir = await Directory(
|
||||||
|
p.join(sandbox.path, 'repo-one'),
|
||||||
|
).create();
|
||||||
|
final repoTwoDir = await Directory(
|
||||||
|
p.join(sandbox.path, 'repo-two'),
|
||||||
|
).create();
|
||||||
|
await initGitRepo(repoOneDir);
|
||||||
|
await initGitRepo(repoTwoDir);
|
||||||
|
|
||||||
|
// And GitHub issue data containing an explicit assistant mention in each repository.
|
||||||
|
final ghScript = File(p.join(sandbox.path, 'gh'));
|
||||||
|
final ghLog = File(p.join(sandbox.path, 'gh-log.jsonl'));
|
||||||
|
await writeFakeGhScript(
|
||||||
|
ghScript: ghScript,
|
||||||
|
issueListResponse: const [],
|
||||||
|
commentResponses: const {},
|
||||||
|
issueListResponsesByRepo: {
|
||||||
|
'owner/repo-one': [
|
||||||
|
{
|
||||||
|
'number': 11,
|
||||||
|
'title': 'Repo one mention',
|
||||||
|
'body': 'first repo',
|
||||||
|
'state': 'open',
|
||||||
|
'html_url': 'https://example.test/owner/repo-one/issues/11',
|
||||||
|
'updated_at': '2026-04-04T17:00:00Z',
|
||||||
|
'user': {'login': 'reporter-one'},
|
||||||
|
'labels': const [],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
'owner/repo-two': [
|
||||||
|
{
|
||||||
|
'number': 22,
|
||||||
|
'title': 'Repo two mention',
|
||||||
|
'body': 'second repo',
|
||||||
|
'state': 'open',
|
||||||
|
'html_url': 'https://example.test/owner/repo-two/issues/22',
|
||||||
|
'updated_at': '2026-04-04T17:01:00Z',
|
||||||
|
'user': {'login': 'reporter-two'},
|
||||||
|
'labels': const [],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
commentResponsesByRepo: {
|
||||||
|
'owner/repo-one': {
|
||||||
|
11: [
|
||||||
|
{
|
||||||
|
'id': 110,
|
||||||
|
'body': '@helper please review repo one',
|
||||||
|
'html_url':
|
||||||
|
'https://example.test/owner/repo-one/issues/11#issuecomment-110',
|
||||||
|
'created_at': '2026-04-04T17:00:00Z',
|
||||||
|
'updated_at': '2026-04-04T17:00:00Z',
|
||||||
|
'user': {'login': 'reporter-one'},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
'owner/repo-two': {
|
||||||
|
22: [
|
||||||
|
{
|
||||||
|
'id': 220,
|
||||||
|
'body': '@helper please review repo two',
|
||||||
|
'html_url':
|
||||||
|
'https://example.test/owner/repo-two/issues/22#issuecomment-220',
|
||||||
|
'created_at': '2026-04-04T17:01:00Z',
|
||||||
|
'updated_at': '2026-04-04T17:01:00Z',
|
||||||
|
'user': {'login': 'reporter-two'},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
postCommentIssueNumbersByRepo: {
|
||||||
|
'owner/repo-one': {11},
|
||||||
|
'owner/repo-two': {22},
|
||||||
|
},
|
||||||
|
ghLog: ghLog,
|
||||||
|
);
|
||||||
|
|
||||||
|
// And a responder that emits a planning reply.
|
||||||
|
final responderScript = File(p.join(sandbox.path, 'responder.sh'));
|
||||||
|
await writeResponderScript(responderScript, {
|
||||||
|
'decision': 'reply',
|
||||||
|
'mode': 'planning',
|
||||||
|
'markdown': 'aggregate reply',
|
||||||
|
'summary': 'posted planning advice',
|
||||||
|
});
|
||||||
|
|
||||||
|
// And an orchestrator-style config with both repositories enabled.
|
||||||
|
final configFile = File(p.join(sandbox.path, 'agent-orchestrator.yaml'));
|
||||||
|
await configFile.writeAsString('''
|
||||||
|
defaults:
|
||||||
|
issueAssistant:
|
||||||
|
enabled: true
|
||||||
|
pollInterval: 30s
|
||||||
|
mentionTriggers: ["@helper"]
|
||||||
|
responders:
|
||||||
|
- id: primary
|
||||||
|
command: ${responderScript.path}
|
||||||
|
projects:
|
||||||
|
repoOne:
|
||||||
|
repo: owner/repo-one
|
||||||
|
path: ${repoOneDir.path}
|
||||||
|
repoTwo:
|
||||||
|
repo: owner/repo-two
|
||||||
|
path: ${repoTwoDir.path}
|
||||||
|
''');
|
||||||
|
final app = await IssueAssistantApp.open(
|
||||||
|
config: await AppConfig.load(configFile.path),
|
||||||
|
databasePath: p.join(sandbox.path, 'state.sqlite3'),
|
||||||
|
ghCommand: ghScript.path,
|
||||||
|
);
|
||||||
|
|
||||||
|
// When the app processes one polling cycle.
|
||||||
|
await app.runOnce();
|
||||||
|
await app.close();
|
||||||
|
|
||||||
|
// Then it fetches issue updates through one aggregate GitHub search request.
|
||||||
|
final logLines = await ghLog.readAsLines();
|
||||||
|
expect(
|
||||||
|
logLines.where((line) => line.contains('search/issues')).length,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
|
||||||
|
// And it still posts one reply in each repository.
|
||||||
|
expect(
|
||||||
|
logLines
|
||||||
|
.where(
|
||||||
|
(line) =>
|
||||||
|
line.contains('repos/owner/repo-one/issues/11/comments'),
|
||||||
|
)
|
||||||
|
.length,
|
||||||
|
2,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
logLines
|
||||||
|
.where(
|
||||||
|
(line) =>
|
||||||
|
line.contains('repos/owner/repo-two/issues/22/comments'),
|
||||||
|
)
|
||||||
|
.length,
|
||||||
|
2,
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,13 +8,49 @@ Future<void> writeFakeGhScript({
|
||||||
required File ghScript,
|
required File ghScript,
|
||||||
required List<Map<String, Object?>> issueListResponse,
|
required List<Map<String, Object?>> issueListResponse,
|
||||||
required Map<int, List<Map<String, Object?>>> commentResponses,
|
required Map<int, List<Map<String, Object?>>> commentResponses,
|
||||||
|
Map<String, List<Map<String, Object?>>>? issueListResponsesByRepo,
|
||||||
|
Map<String, Map<int, List<Map<String, Object?>>>>? commentResponsesByRepo,
|
||||||
File? ghLog,
|
File? ghLog,
|
||||||
int? postCommentIssueNumber,
|
int? postCommentIssueNumber,
|
||||||
Set<int>? postCommentIssueNumbers,
|
Set<int>? postCommentIssueNumbers,
|
||||||
|
Map<String, Set<int>>? postCommentIssueNumbersByRepo,
|
||||||
File? postedBodyFile,
|
File? postedBodyFile,
|
||||||
String? viewerLogin,
|
String? viewerLogin,
|
||||||
bool failViewerLookup = false,
|
bool failViewerLookup = false,
|
||||||
}) async {
|
}) async {
|
||||||
|
final normalizedIssueListResponsesByRepo =
|
||||||
|
issueListResponsesByRepo ??
|
||||||
|
<String, List<Map<String, Object?>>>{'owner/sample': issueListResponse};
|
||||||
|
final normalizedCommentResponsesByRepo =
|
||||||
|
commentResponsesByRepo ??
|
||||||
|
<String, Map<int, List<Map<String, Object?>>>>{
|
||||||
|
'owner/sample': commentResponses,
|
||||||
|
};
|
||||||
|
final normalizedPostCommentIssueNumbersByRepo =
|
||||||
|
postCommentIssueNumbersByRepo ??
|
||||||
|
<String, Set<int>>{
|
||||||
|
'owner/sample': {
|
||||||
|
...?postCommentIssueNumbers,
|
||||||
|
...?postCommentIssueNumber == null ? null : {postCommentIssueNumber},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
final searchItems =
|
||||||
|
normalizedIssueListResponsesByRepo.entries
|
||||||
|
.expand(
|
||||||
|
(entry) => entry.value.map(
|
||||||
|
(issue) => <String, Object?>{
|
||||||
|
...issue,
|
||||||
|
'repository_url': 'https://api.github.com/repos/${entry.key}',
|
||||||
|
},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.toList(growable: false)
|
||||||
|
..sort((left, right) {
|
||||||
|
final leftUpdatedAt = left['updated_at'] as String? ?? '';
|
||||||
|
final rightUpdatedAt = right['updated_at'] as String? ?? '';
|
||||||
|
return leftUpdatedAt.compareTo(rightUpdatedAt);
|
||||||
|
});
|
||||||
|
|
||||||
final buffer = StringBuffer()
|
final buffer = StringBuffer()
|
||||||
..writeln('#!/usr/bin/env bash')
|
..writeln('#!/usr/bin/env bash')
|
||||||
..writeln('set -euo pipefail');
|
..writeln('set -euo pipefail');
|
||||||
|
|
@ -42,20 +78,24 @@ Future<void> writeFakeGhScript({
|
||||||
}
|
}
|
||||||
|
|
||||||
buffer
|
buffer
|
||||||
..writeln(
|
..writeln(r'''if [[ "$1" == "api" && "$4" == "search/issues" ]]; then''')
|
||||||
r'''if [[ "$1" == "api" && "$4" == repos/owner/sample/issues\?* ]]; then''',
|
|
||||||
)
|
|
||||||
..writeln(" cat <<'JSON'")
|
..writeln(" cat <<'JSON'")
|
||||||
..writeln(jsonEncode(issueListResponse))
|
..writeln(
|
||||||
|
jsonEncode({
|
||||||
|
'total_count': searchItems.length,
|
||||||
|
'incomplete_results': false,
|
||||||
|
'items': searchItems,
|
||||||
|
}),
|
||||||
|
)
|
||||||
..writeln('JSON')
|
..writeln('JSON')
|
||||||
..writeln(' exit 0')
|
..writeln(' exit 0')
|
||||||
..writeln('fi');
|
..writeln('fi');
|
||||||
|
|
||||||
for (final entry in commentResponses.entries) {
|
for (final entry in normalizedIssueListResponsesByRepo.entries) {
|
||||||
|
final repo = entry.key;
|
||||||
buffer
|
buffer
|
||||||
..writeln(
|
..writeln(
|
||||||
'if [[ "\$1" == "api" && "\$4" == '
|
'if [[ "\$1" == "api" && "\$4" == repos/$repo/issues\\?* ]]; then',
|
||||||
'"repos/owner/sample/issues/${entry.key}/comments?per_page=100" ]]; then',
|
|
||||||
)
|
)
|
||||||
..writeln(" cat <<'JSON'")
|
..writeln(" cat <<'JSON'")
|
||||||
..writeln(jsonEncode(entry.value))
|
..writeln(jsonEncode(entry.value))
|
||||||
|
|
@ -64,40 +104,55 @@ Future<void> writeFakeGhScript({
|
||||||
..writeln('fi');
|
..writeln('fi');
|
||||||
}
|
}
|
||||||
|
|
||||||
final commentIssueNumbers = <int>{...?postCommentIssueNumbers};
|
for (final repoEntry in normalizedCommentResponsesByRepo.entries) {
|
||||||
if (postCommentIssueNumber != null) {
|
final repo = repoEntry.key;
|
||||||
commentIssueNumbers.add(postCommentIssueNumber);
|
for (final entry in repoEntry.value.entries) {
|
||||||
}
|
|
||||||
for (final issueNumber in commentIssueNumbers) {
|
|
||||||
buffer
|
|
||||||
..writeln(
|
|
||||||
'if [[ "\$1" == "api" && "\$4" == '
|
|
||||||
'"repos/owner/sample/issues/$issueNumber/comments" ]]; then',
|
|
||||||
)
|
|
||||||
..writeln(r''' for arg in "$@"; do''')
|
|
||||||
..writeln(r''' :''');
|
|
||||||
if (postedBodyFile != null) {
|
|
||||||
buffer
|
buffer
|
||||||
..writeln(r''' if [[ "$arg" == body=* ]]; then''')
|
|
||||||
..writeln(
|
..writeln(
|
||||||
r''' printf '%s' "${arg#body=}" > '''
|
'if [[ "\$1" == "api" && "\$4" == '
|
||||||
'"${postedBodyFile.path}"',
|
'"repos/$repo/issues/${entry.key}/comments?per_page=100" ]]; then',
|
||||||
)
|
)
|
||||||
..writeln(r''' fi''');
|
..writeln(" cat <<'JSON'")
|
||||||
|
..writeln(jsonEncode(entry.value))
|
||||||
|
..writeln('JSON')
|
||||||
|
..writeln(' exit 0')
|
||||||
|
..writeln('fi');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (final repoEntry in normalizedPostCommentIssueNumbersByRepo.entries) {
|
||||||
|
final repo = repoEntry.key;
|
||||||
|
for (final issueNumber in repoEntry.value) {
|
||||||
|
buffer
|
||||||
|
..writeln(
|
||||||
|
'if [[ "\$1" == "api" && "\$4" == '
|
||||||
|
'"repos/$repo/issues/$issueNumber/comments" ]]; then',
|
||||||
|
)
|
||||||
|
..writeln(r''' for arg in "$@"; do''')
|
||||||
|
..writeln(r''' :''');
|
||||||
|
if (postedBodyFile != null) {
|
||||||
|
buffer
|
||||||
|
..writeln(r''' if [[ "$arg" == body=* ]]; then''')
|
||||||
|
..writeln(
|
||||||
|
r''' printf '%s' "${arg#body=}" > '''
|
||||||
|
'"${postedBodyFile.path}"',
|
||||||
|
)
|
||||||
|
..writeln(r''' fi''');
|
||||||
|
}
|
||||||
|
buffer
|
||||||
|
..writeln(r''' done''')
|
||||||
|
..writeln(" cat <<'JSON'")
|
||||||
|
..writeln(
|
||||||
|
jsonEncode({
|
||||||
|
'id': 700,
|
||||||
|
'html_url': 'https://example.test/comment/700',
|
||||||
|
'body': 'posted',
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
..writeln('JSON')
|
||||||
|
..writeln(' exit 0')
|
||||||
|
..writeln('fi');
|
||||||
}
|
}
|
||||||
buffer
|
|
||||||
..writeln(r''' done''')
|
|
||||||
..writeln(" cat <<'JSON'")
|
|
||||||
..writeln(
|
|
||||||
jsonEncode({
|
|
||||||
'id': 700,
|
|
||||||
'html_url': 'https://example.test/comment/700',
|
|
||||||
'body': 'posted',
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
..writeln('JSON')
|
|
||||||
..writeln(' exit 0')
|
|
||||||
..writeln('fi');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
buffer
|
buffer
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue