Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/core/src/store/settings_list.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ pub static SETTINGS: LazyLock<Vec<Entry>> = LazyLock::new(|| {
s.push(e("review.timeout", "Review may run for", Work, Ty::Time, json!("20m"), "how long one review may take, and one comparison of attempts: a time (`45m`, `1h`) or `none`. Read when the review starts. Reached: the reviewer is stopped and the review fails (`review timed out after 20m (review.timeout)`), and what happens next is as for any failed review. Not from the repo's own file, so the code under review cannot change it. Unlike `limit.time`, which stops any run and waits for you. More: [the gate](gate.md)").none().means("how long one review may take. At it the review fails, as any failed review. Read when the review starts"));
s.push(e("review.afterChat", "Review after a chat", Work, Ty::Time, json!("60s"), "how long a chat with the agent of a task whose work was reviewed (under review, waiting for your yes, done) must be quiet before the code it changed is reviewed: no new message from you and the agent at rest for this long, or you left the chat (another task on the board, another space in herdr). Then one review runs on the final state. A chat that changed no code reviews nothing: the task goes back to where it was, with the same review. More: [how work gets done](how-work-gets-done.md)"));
s.push(e("review.evidence", "Reviewer brings evidence", Work, Ty::Switch, json!(true), "the reviewer brings evidence: screenshots, a recording, a link, test output"));
s.push(e("review.prComment", "Review comment on the pull request", Work, Ty::Switch, json!(true), "after each review, the board posts it as a comment on the task's pull request: the verdict, the gate and the findings. Turned off, no comment is posted; the review still runs, is judged at the gate and shows on the review page and in the task panel. Read as each review ends, so a change counts for a review under way. More: [the gate](gate.md)"));
// how much it may take (limits/). Never a fact of the repo's own file: code an agent writes cannot lift a limit.
s.push(e("limit.time", "Longest run", Work, Ty::Time, json!("none"), "the longest one run of an agent may take, the working agent's or the reviewer's: a time (`2h`, `45m`) or `none`. Read when a run starts: set while one is under way, it counts from the next. Reached: the agent is stopped, its work is kept as it is, and the task waits for you under \"needs you\" (`stopped at its 2h limit`). Nothing is merged. More: [limits](limits.md)").none().means("the longest one run of an agent may take, the working agent's or the reviewer's. At it the agent is stopped, its work is kept, nothing merges, and the task waits for you. Read when a run starts"));
s.push(e("limit.tokens", "Most tokens", Work, Ty::Tokens, json!("none"), "the most tokens a task may use, all its agents, attempts and reviews together, what is read again from the provider's cache included: a count (`2m`, `500k`) or `none`. Counted at the end of each turn, so one turn can pass it. Reached: as `limit.time` (`stopped at 2m tokens`). For a program whose tokens cannot be counted it does nothing, and `show <id>` says so. More: [limits](limits.md)").means("the most tokens the task may use, all its agents and reviews together, counted at the end of each turn. At it the task is stopped, its work is kept, nothing merges, and it waits for you"));
Expand Down
5 changes: 3 additions & 2 deletions crates/flow/src/finish/pipeline_end.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,9 @@ pub fn after_review(c: Ctx, g: &mut Guard) -> Res<()> {
return Ok(());
};
let rv = Value::Object(rv);
// The pull request says what the review found, the findings below the gate marked as not required.
if t.b("pr") && to_land { let round = get(id).map(|x| review_round(&x)).unwrap_or(0); flow::git::gh(&["pr", "comment", t.s("pr"), "--body", &format!("{}{}", gate::pr_comment(&rv, round), flow::git::BOARD_COMMENT)], c.root, 60000); }
// The pull request says what the review found, the findings below the gate marked as not required. With
// `review.prComment` off it says nothing: read now, so turning it off during a review already counts.
if t.b("pr") && to_land { let cur = get(id); if work(cur.as_ref()).b("review.prComment") { let round = cur.map(|x| review_round(&x)).unwrap_or(0); flow::git::gh(&["pr", "comment", t.s("pr"), "--body", &format!("{}{}", gate::pr_comment(&rv, round), flow::git::BOARD_COMMENT)], c.root, 60000); } }
review = Some(rv);
}
// A conflict with main that was repaired on the way here: one line in the record, with whether a review followed.
Expand Down
3 changes: 2 additions & 1 deletion docs/gate.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ The severity says what the fault does:
that stopped it (one herdr did not report as a dialog) is there to see, before you retry.

Findings below the gate still show on the review page, in the task panel and in a comment on the pull request, marked
as not required.
as not required. `review.prComment` turned off, the board posts no review comment on the pull request: the review
still runs, is judged at the gate and shows on the review page and in the task panel.

## Merging by itself

Expand Down
1 change: 1 addition & 0 deletions docs/settings.md
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ are changed on that machine.
| `review.timeout` | `20m` | task, repo entry, you | how long one review may take, and one comparison of attempts: a time (`45m`, `1h`) or `none`. Read when the review starts. Reached: the reviewer is stopped and the review fails (`review timed out after 20m (review.timeout)`), and what happens next is as for any failed review. Not from the repo's own file, so the code under review cannot change it. Unlike `limit.time`, which stops any run and waits for you. More: [the gate](gate.md) |
| `review.afterChat` | `60s` | task, repo entry, you | how long a chat with the agent of a task whose work was reviewed (under review, waiting for your yes, done) must be quiet before the code it changed is reviewed: no new message from you and the agent at rest for this long, or you left the chat (another task on the board, another space in herdr). Then one review runs on the final state. A chat that changed no code reviews nothing: the task goes back to where it was, with the same review. More: [how work gets done](how-work-gets-done.md) |
| `review.evidence` | `true` | task, repo entry, you | the reviewer brings evidence: screenshots, a recording, a link, test output |
| `review.prComment` | `true` | task, repo entry, you | after each review, the board posts it as a comment on the task's pull request: the verdict, the gate and the findings. Turned off, no comment is posted; the review still runs, is judged at the gate and shows on the review page and in the task panel. Read as each review ends, so a change counts for a review under way. More: [the gate](gate.md) |
| `limit.time` | `none` | task, repo entry, you | the longest one run of an agent may take, the working agent's or the reviewer's: a time (`2h`, `45m`) or `none`. Read when a run starts: set while one is under way, it counts from the next. Reached: the agent is stopped, its work is kept as it is, and the task waits for you under "needs you" (`stopped at its 2h limit`). Nothing is merged. More: [limits](limits.md) |
| `limit.tokens` | `none` | task, repo entry, you | the most tokens a task may use, all its agents, attempts and reviews together, what is read again from the provider's cache included: a count (`2m`, `500k`) or `none`. Counted at the end of each turn, so one turn can pass it. Reached: as `limit.time` (`stopped at 2m tokens`). For a program whose tokens cannot be counted it does nothing, and `show <id>` says so. More: [limits](limits.md) |
| `merge.by` | `you` | task, repo entry, you | `you`: finished work waits for your yes. `itself`: it merges the moment it may: its review passed, its checks pass, nothing conflicts. `none`: the board never lands it, for a repo whose work lands through code review on GitHub: a task whose review passed waits as `Review passed: ready for code review`, and `approve` is refused, `--anyway` too (a merge approved before it was set and not made yet is called off); its review rounds, its check and its pushes to its pull request are any task's. Read right before the merge, so a change counts for tasks under way. More: [the gate](gate.md#merging-by-itself), [landed through code review](gate.md#landed-through-code-review-on-github) |
Expand Down
2 changes: 1 addition & 1 deletion tests/automations_open.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ fn an_automations_settings_are_the_page_a_task_has_its_own_rows_marked_a_row_cha
let w = world();
let a = three(&w).remove(0);
sw::store::settings::change::set_value("review.rounds", &json!(5), &sw::store::settings::lookup::Scope::you()).unwrap();
let mut s = open(&w.root, 65); s.s.side = false;
let mut s = open(&w.root, 66); s.s.side = false;
let row_of = |s: &mut Board, name: &str| -> String { let re = regex::Regex::new(&format!(" {}( |$)", regex::escape(name))).unwrap(); s.screen(120).into_iter().find(|l| re.is_match(l)).unwrap_or_default() };
let names = |s: &Board| -> Vec<String> { sw::board::update::edit::rows(&s.s).iter().map(|x| x.t("name")).collect() };
let at = |s: &Board| s.s.editing.as_ref().unwrap().at;
Expand Down
2 changes: 1 addition & 1 deletion tests/board_parts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ fn the_settings_page_every_work_setting_plain_and_a_mark_on_this_tasks_own() {
set_value("review.rounds", &json!(5), &Scope::default()).unwrap();
set_value("merge.by", &json!("itself"), &Scope::repo("proj")).unwrap();
set_value("reviewer.agent", &json!("claude"), &Scope::task(&task(&id))).unwrap();
let mut s = open(&w.root, 66); s.s.sel = id.clone(); s.load();
let mut s = open(&w.root, 67); s.s.sel = id.clone(); s.load();
s.press(&["ctrl+e"]);
let text = s.screen(120).join("\n");
assert!(text.starts_with(" Settings PRJ-1 \"One\""), "{text}");
Expand Down
67 changes: 66 additions & 1 deletion tests/gate_more.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! The gate, more: a later round's verdict read against the earlier findings and what the reviewer is told, the
//! review.mustFix and review.rounds settings, and the review page and the pull request showing every finding.
//! review.mustFix and review.rounds settings, the review page and the pull request showing every finding, and
//! review.prComment turning that pull request comment off.
use crate::common;
use common::*;
use serde_json::{json, Value};
Expand Down Expand Up @@ -104,6 +105,70 @@ fn review_must_fix_is_major_by_default_yours_or_one_tasks_and_review_rounds_0_se
assert_eq!(task(&other).at("settings").s("review.mustFix"), "critical");
}

#[test]
fn review_pr_comment_off_posts_no_comment_but_the_review_still_runs() {
let w = world_with(Opts { repo: true, origin: true, ..Default::default() });
w.script(json!({ "worker": [{ "files": { "f.txt": "1\n" }, "commit": "v1", "note": "Did v1." }], "reviewer": [{ "verdict": { "meets": true, "text": "Looked at it.", "findings": [] } }] }));
assert_eq!(setting("review.prComment", None), json!(true));
let id = w.add("Quiet pull request");
assert_eq!(sw::cli::main(vec!["set".into(), "review.prComment".into(), "false".into(), "--task".into(), id.clone()]), 0);
delegate(&id);
let t = settle(&id);
assert_eq!(t.s("stage"), "approval", "{}", t.at("failure"));
assert!(t.at("review").b("meets"), "{}", t.at("review"));
let log = gh_log(&w);
assert!(!log.contains("pr comment"), "{log}");
}

/// Everything the board asked of gh for one task with a pull request, the world's own folder left out, with
/// `review.prComment` on or off (yours): `script` by role, `asked` what the task was delegated with.
fn gh_calls(comment: bool, script: Value, asked: Value) -> Vec<String> {
let w = world_with(Opts { repo: true, origin: true, settings: json!({ "review.prComment": comment }), ..Default::default() });
w.script(script);
let id = w.add("Quiet pull request");
delegate_with(&id, asked);
let mut t = settle_all(&id);
for _ in 0..3 { if t.s("stage") != "approval" { t = settle_all(&id); } }
assert_eq!(t.s("stage"), "approval", "{}", t.at("failure"));
assert!(t.at("review").b("pass"), "{}", t.at("review"));
let base = w.base.display().to_string();
w.gh.log.borrow().iter().map(|l| l.replace(&base, "<world>")).collect()
}
/// The `on` calls without their review comments must be the `off` calls exactly: nothing else on the pull request changes.
fn only_the_comments_go(on: &[String], off: &[String]) -> Vec<String> {
let comments: Vec<String> = on.iter().filter(|l| l.starts_with("pr comment ")).cloned().collect();
assert_eq!(off.to_vec(), on.iter().filter(|l| !l.starts_with("pr comment ")).cloned().collect::<Vec<_>>());
assert!(!off.iter().any(|l| l.contains("**Review ")), "{off:?}");
comments
}

#[test]
fn review_pr_comment_off_no_comment_in_any_round_failed_or_passed_and_nothing_else_on_the_pull_request_changes() {
let work = |n: i64| json!({ "files": { "f.txt": format!("{n}\n") }, "commit": format!("v{n}"), "note": format!("Did v{n}.") });
let fails = json!({ "verdict": { "meets": false, "text": "Not yet.", "findings": [finding("major", "Crashes on empty files", json!({}))] } });
let passes = json!({ "verdict": { "meets": true, "text": "Fixed.", "findings": [] } });
let script = json!({ "worker": [work(1), work(2)], "reviewer": [fails, passes] });
let on = gh_calls(true, script.clone(), json!({}));
let off = gh_calls(false, script, json!({}));
let comments = only_the_comments_go(&on, &off);
assert_eq!(comments.len(), 2, "{on:?}");
assert!(comments[0].contains("--body **Review 1 · Fail · 1 major**"), "{}", comments[0]);
assert!(comments[1].contains("--body **Review 2 · Pass**"), "{}", comments[1]);
assert!(off.iter().any(|l| l.starts_with("pr create ")), "the pull request is still opened: {off:?}");
}

#[test]
fn review_pr_comment_off_the_chosen_attempts_review_posts_no_comment_either() {
let picker = json!([{ "comparison": { "pick": "b", "why": "B is the smallest change.", "attempts": [{ "letter": "A", "note": "Works, but large." }, { "letter": "B", "note": "Small and right." }] } }]);
let script = json!({ "worker": [step(1), step(2)], "picker": picker, "reviewer": [{ "verdict": { "meets": true, "text": "Looked at it.", "findings": [] } }] });
let asked = json!({ "attempts.by": "codex/gpt-5.5, pi", "attempts.pick": "reviewer" });
let on = gh_calls(true, script.clone(), asked.clone());
let off = gh_calls(false, script, asked);
let comments = only_the_comments_go(&on, &off);
assert_eq!(comments.len(), 1, "{on:?}");
assert!(comments[0].contains("--body **Review 1 · Pass**"), "{}", comments[0]);
}

#[test]
fn the_pull_request_and_the_review_page_show_every_finding_those_below_the_gate_marked_not_required() {
let w = world_with(Opts { repo: true, origin: true, ..Default::default() });
Expand Down
Loading