diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/metadata.json new file mode 100644 index 00000000..c7b6b3b9 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 2, + "primary_skill": "rust-debugging", + "secondary_skills": ["cargo-test", "chrono"], + "name": "rust_bugfix_001_chrono_off_by_one", + "difficulty": "easy", + "runtime_stressors": ["fault_localization", "test_interpretation"], + "task_type": "single_file_bugfix", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/prompt.txt new file mode 100644 index 00000000..d0b72ca6 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/prompt.txt @@ -0,0 +1 @@ +The CLI command `report --date today` returns tomorrow's date in some timezones. Fix the bug and verify the behavior. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/repo/Cargo.lock new file mode 100644 index 00000000..6a3c46aa --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/repo/Cargo.lock @@ -0,0 +1,284 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "cc" +version = "1.2.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "date-resolver" +version = "0.1.0" +dependencies = [ + "chrono", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "js-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.185" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasm-bindgen" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/repo/Cargo.toml new file mode 100644 index 00000000..c8ff957d --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/repo/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "date-resolver" +version = "0.1.0" +edition = "2021" + +[dependencies] +chrono = "0.4" + +[[test]] +name = "test_fix" +path = "tests/test_fix.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/repo/src/lib.rs new file mode 100644 index 00000000..693ad140 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/repo/src/lib.rs @@ -0,0 +1,6 @@ +use chrono::{DateTime, Duration, NaiveDate, Utc}; + +pub fn resolve_today(now: DateTime) -> NaiveDate { + let shifted = now + Duration::hours(24); + shifted.date_naive() +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/repo/tests/test_fix.rs new file mode 100644 index 00000000..696c9753 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/repo/tests/test_fix.rs @@ -0,0 +1,24 @@ +use chrono::{TimeZone, Utc}; +use date_resolver::resolve_today; + +#[test] +fn test_resolve_today_returns_correct_date() { + let now = Utc.with_ymd_and_hms(2024, 3, 15, 10, 0, 0).unwrap(); + let result = resolve_today(now); + assert_eq!( + result.to_string(), + "2024-03-15", + "resolve_today should return today's date, not tomorrow's" + ); +} + +#[test] +fn test_resolve_today_near_midnight() { + let now = Utc.with_ymd_and_hms(2024, 6, 1, 23, 30, 0).unwrap(); + let result = resolve_today(now); + assert_eq!( + result.to_string(), + "2024-06-01", + "resolve_today near midnight should still return the current date" + ); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/task.yaml new file mode 100644 index 00000000..c69ea15f --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_001_chrono_off_by_one/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_001_chrono_off_by_one +family: bugfix +difficulty: easy +language: rust +max_minutes: 5 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 3 +task_type: single_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/metadata.json new file mode 100644 index 00000000..98509e42 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 3, + "primary_skill": "rust-debugging", + "secondary_skills": ["cargo-test", "std-env"], + "name": "rust_bugfix_002_env_config_precedence", + "difficulty": "easy", + "runtime_stressors": ["fault_localization", "test_interpretation"], + "task_type": "single_file_bugfix", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/prompt.txt new file mode 100644 index 00000000..5c72ec7f --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/prompt.txt @@ -0,0 +1 @@ +When both an environment variable and a config file value are present, the application should prefer the environment variable. Currently the config file value always wins regardless. Fix the precedence bug and verify it. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/repo/Cargo.lock new file mode 100644 index 00000000..f36cf823 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "config-loader" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/repo/Cargo.toml new file mode 100644 index 00000000..65f33968 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/repo/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "config-loader" +version = "0.1.0" +edition = "2021" + +[dependencies] + +[[test]] +name = "test_fix" +path = "tests/test_fix.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/repo/src/lib.rs new file mode 100644 index 00000000..6819564a --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/repo/src/lib.rs @@ -0,0 +1,9 @@ +use std::collections::HashMap; + +pub fn load_config(key: &str, file_config: &HashMap) -> Option { + if let Some(val) = file_config.get(key) { + return Some(val.clone()); + } + + std::env::var(key).ok() +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/repo/tests/test_fix.rs new file mode 100644 index 00000000..0f9b7a25 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/repo/tests/test_fix.rs @@ -0,0 +1,46 @@ +use std::collections::HashMap; +use config_loader::load_config; + +#[test] +fn test_env_var_wins_over_file_config() { + let key = "CFG_TEST_LOG_LEVEL"; + std::env::set_var(key, "debug"); + + let mut file_config = HashMap::new(); + file_config.insert(key.to_string(), "info".to_string()); + + let result = load_config(key, &file_config); + std::env::remove_var(key); + + assert_eq!( + result, + Some("debug".to_string()), + "environment variable should take precedence over file config" + ); +} + +#[test] +fn test_file_config_used_when_no_env_var() { + let key = "CFG_TEST_OUTPUT_DIR"; + std::env::remove_var(key); + + let mut file_config = HashMap::new(); + file_config.insert(key.to_string(), "/tmp/output".to_string()); + + let result = load_config(key, &file_config); + assert_eq!( + result, + Some("/tmp/output".to_string()), + "file config should be used when no environment variable is set" + ); +} + +#[test] +fn test_returns_none_when_neither_source_has_key() { + let key = "CFG_TEST_NONEXISTENT_KEY_XYZ"; + std::env::remove_var(key); + let file_config = HashMap::new(); + + let result = load_config(key, &file_config); + assert_eq!(result, None, "should return None when key is absent everywhere"); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/task.yaml new file mode 100644 index 00000000..5f654be8 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_002_env_config_precedence/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_002_env_config_precedence +family: bugfix +difficulty: easy +language: rust +max_minutes: 5 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 3 +task_type: single_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/metadata.json new file mode 100644 index 00000000..9617ae33 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 2, + "primary_skill": "rust-debugging", + "secondary_skills": ["cargo-test", "result-error-handling"], + "name": "rust_bugfix_003_exit_code_validation", + "difficulty": "easy", + "runtime_stressors": ["fault_localization", "test_interpretation"], + "task_type": "single_file_bugfix", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/prompt.txt new file mode 100644 index 00000000..7a6d6f97 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/prompt.txt @@ -0,0 +1 @@ +The validation tool reports success even when given invalid input. Fix the validator so it returns an error for invalid data and verify the behavior. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/repo/Cargo.lock new file mode 100644 index 00000000..d2492790 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "input-validator" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/repo/Cargo.toml new file mode 100644 index 00000000..dce2634f --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/repo/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "input-validator" +version = "0.1.0" +edition = "2021" + +[dependencies] + +[[test]] +name = "test_fix" +path = "tests/test_fix.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/repo/src/lib.rs new file mode 100644 index 00000000..3a19e9e6 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/repo/src/lib.rs @@ -0,0 +1,11 @@ +pub fn validate(input: &str) -> Result<(), String> { + if input.is_empty() { + return Ok(()); + } + + if !input.chars().all(|c| c.is_ascii_alphanumeric()) { + return Err(format!("invalid character in input: {:?}", input)); + } + + Ok(()) +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/repo/tests/test_fix.rs new file mode 100644 index 00000000..9cbda7da --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/repo/tests/test_fix.rs @@ -0,0 +1,29 @@ +use input_validator::validate; + +#[test] +fn test_empty_input_is_invalid() { + let result = validate(""); + assert!( + result.is_err(), + "empty input should be invalid, but validate returned Ok" + ); +} + +#[test] +fn test_valid_alphanumeric_input_is_ok() { + let result = validate("hello123"); + assert!( + result.is_ok(), + "alphanumeric input should be valid, but validate returned Err: {:?}", + result + ); +} + +#[test] +fn test_input_with_special_chars_is_invalid() { + let result = validate("bad input!"); + assert!( + result.is_err(), + "input with spaces and punctuation should be invalid" + ); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/task.yaml new file mode 100644 index 00000000..87b84b2b --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_003_exit_code_validation/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_003_exit_code_validation +family: bugfix +difficulty: easy +language: rust +max_minutes: 5 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 3 +task_type: single_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/metadata.json new file mode 100644 index 00000000..db4f2d2e --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 4, + "primary_skill": "rust-debugging", + "secondary_skills": ["cargo-test", "collections"], + "name": "rust_bugfix_004_vec_dedup_unsorted", + "difficulty": "easy", + "runtime_stressors": ["fault_localization", "test_interpretation"], + "task_type": "single_file_bugfix", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/prompt.txt new file mode 100644 index 00000000..9222cb9d --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/prompt.txt @@ -0,0 +1 @@ +The deduplication function leaves duplicate entries in the output when the same value appears non-consecutively. Fix it so all duplicates are removed while preserving the original order of first occurrences, then verify the behavior. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/repo/Cargo.lock new file mode 100644 index 00000000..b03ba1d6 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "dedup-util" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/repo/Cargo.toml new file mode 100644 index 00000000..d11b5895 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/repo/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "dedup-util" +version = "0.1.0" +edition = "2021" + +[dependencies] + +[[test]] +name = "test_fix" +path = "tests/test_fix.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/repo/src/lib.rs new file mode 100644 index 00000000..b8bf8d10 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/repo/src/lib.rs @@ -0,0 +1,4 @@ +pub fn unique_items(mut items: Vec) -> Vec { + items.dedup(); + items +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/repo/tests/test_fix.rs new file mode 100644 index 00000000..55effdc7 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/repo/tests/test_fix.rs @@ -0,0 +1,32 @@ +use dedup_util::unique_items; + +#[test] +fn test_non_consecutive_duplicates_are_removed() { + let input = vec![ + "a".to_string(), + "b".to_string(), + "a".to_string(), + "c".to_string(), + "b".to_string(), + ]; + let result = unique_items(input); + assert_eq!( + result, + vec!["a".to_string(), "b".to_string(), "c".to_string()], + "unique_items should remove all duplicates, preserving first-occurrence order" + ); +} + +#[test] +fn test_already_unique_list_is_unchanged() { + let input = vec!["x".to_string(), "y".to_string(), "z".to_string()]; + let result = unique_items(input.clone()); + assert_eq!(result, input, "a list with no duplicates should be returned unchanged"); +} + +#[test] +fn test_all_same_elements_returns_single() { + let input = vec!["dup".to_string(), "dup".to_string(), "dup".to_string()]; + let result = unique_items(input); + assert_eq!(result, vec!["dup".to_string()], "all identical elements should collapse to one"); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/task.yaml new file mode 100644 index 00000000..53c7e22c --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_004_vec_dedup_unsorted/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_004_vec_dedup_unsorted +family: bugfix +difficulty: easy +language: rust +max_minutes: 5 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 3 +task_type: single_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/metadata.json new file mode 100644 index 00000000..65bf0e1e --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 1, + "primary_skill": "rust-debugging", + "secondary_skills": ["cargo-test", "integer-arithmetic"], + "name": "rust_bugfix_005_modulo_negative", + "difficulty": "easy", + "runtime_stressors": ["fault_localization", "test_interpretation"], + "task_type": "single_file_bugfix", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/prompt.txt new file mode 100644 index 00000000..53f826f7 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/prompt.txt @@ -0,0 +1 @@ +The wrap_index function returns negative indices for some inputs, causing out-of-bounds access when used to index a collection. Fix it so the result is always a valid non-negative index, then verify the behavior. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/repo/Cargo.lock new file mode 100644 index 00000000..938cea99 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "wrap-index" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/repo/Cargo.toml new file mode 100644 index 00000000..fe6a76db --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/repo/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "wrap-index" +version = "0.1.0" +edition = "2021" + +[dependencies] + +[[test]] +name = "test_fix" +path = "tests/test_fix.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/repo/src/lib.rs new file mode 100644 index 00000000..ebce9b05 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/repo/src/lib.rs @@ -0,0 +1,3 @@ +pub fn wrap_index(index: i32, len: i32) -> i32 { + index % len +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/repo/tests/test_fix.rs new file mode 100644 index 00000000..be0ad8da --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/repo/tests/test_fix.rs @@ -0,0 +1,46 @@ +use wrap_index::wrap_index; + +#[test] +fn test_negative_one_wraps_to_last() { + assert_eq!( + wrap_index(-1, 5), + 4, + "wrap_index(-1, 5) should be 4, not -1" + ); +} + +#[test] +fn test_negative_large_wraps_correctly() { + assert_eq!( + wrap_index(-7, 5), + 3, + "wrap_index(-7, 5) should be 3" + ); +} + +#[test] +fn test_positive_index_unchanged_when_in_range() { + assert_eq!( + wrap_index(3, 5), + 3, + "wrap_index(3, 5) should remain 3" + ); +} + +#[test] +fn test_positive_index_wraps_when_over_len() { + assert_eq!( + wrap_index(7, 5), + 2, + "wrap_index(7, 5) should wrap to 2" + ); +} + +#[test] +fn test_zero_index_is_zero() { + assert_eq!( + wrap_index(0, 5), + 0, + "wrap_index(0, 5) should be 0" + ); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/task.yaml new file mode 100644 index 00000000..74fe533a --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_005_modulo_negative/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_005_modulo_negative +family: bugfix +difficulty: easy +language: rust +max_minutes: 5 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 3 +task_type: single_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/metadata.json new file mode 100644 index 00000000..c7448d29 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 3, + "primary_skill": "rust-debugging", + "secondary_skills": ["pattern-matching", "http-status-codes"], + "name": "rust_bugfix_006_retry_status_match", + "difficulty": "medium", + "runtime_stressors": ["fault_localization", "test_interpretation"], + "task_type": "single_file_bugfix", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/prompt.txt new file mode 100644 index 00000000..a1c18a83 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/prompt.txt @@ -0,0 +1 @@ +The API client retries on client errors like 400 and 404, which should not be retried. Fix the retry policy so only server errors (5xx) and 429 are retried. Verify the fix with the existing tests. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/repo/Cargo.lock new file mode 100644 index 00000000..edccfec0 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "retry-policy" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/repo/Cargo.toml new file mode 100644 index 00000000..9c60673b --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/repo/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "retry-policy" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/repo/src/lib.rs new file mode 100644 index 00000000..ecd56825 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/repo/src/lib.rs @@ -0,0 +1,6 @@ +pub fn should_retry(status: u16) -> bool { + match status { + 400..=499 => true, + _ => false, + } +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/repo/tests/test_fix.rs new file mode 100644 index 00000000..9aef47c0 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/repo/tests/test_fix.rs @@ -0,0 +1,31 @@ +use retry_policy::should_retry; + +#[test] +fn server_error_500_should_be_retried() { + assert!(should_retry(500), "500 Internal Server Error should be retried"); +} + +#[test] +fn server_error_503_should_be_retried() { + assert!(should_retry(503), "503 Service Unavailable should be retried"); +} + +#[test] +fn client_error_400_should_not_be_retried() { + assert!(!should_retry(400), "400 Bad Request should not be retried"); +} + +#[test] +fn client_error_404_should_not_be_retried() { + assert!(!should_retry(404), "404 Not Found should not be retried"); +} + +#[test] +fn rate_limit_429_should_be_retried() { + assert!(should_retry(429), "429 Too Many Requests should be retried"); +} + +#[test] +fn success_200_should_not_be_retried() { + assert!(!should_retry(200), "200 OK should not be retried"); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/task.yaml new file mode 100644 index 00000000..943b7840 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_006_retry_status_match/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_006_retry_status_match +family: bugfix +difficulty: medium +language: rust +max_minutes: 8 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 3 +task_type: single_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/metadata.json new file mode 100644 index 00000000..72c61120 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 2, + "primary_skill": "rust-debugging", + "secondary_skills": ["serde", "json-serialization"], + "name": "rust_bugfix_007_serde_rename_roundtrip", + "difficulty": "medium", + "runtime_stressors": ["fault_localization", "test_interpretation"], + "task_type": "single_file_bugfix", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": true, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/prompt.txt new file mode 100644 index 00000000..aa5959a8 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/prompt.txt @@ -0,0 +1 @@ +Serialized user records cannot be deserialized back. The JSON field names don't match on the way back in, causing deserialization to fail. Fix the serialization roundtrip so records survive a serialize-then-deserialize cycle. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/repo/Cargo.lock new file mode 100644 index 00000000..ae4805c4 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/repo/Cargo.lock @@ -0,0 +1,107 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "user-serde" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/repo/Cargo.toml new file mode 100644 index 00000000..17df8191 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/repo/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "user-serde" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/repo/src/lib.rs new file mode 100644 index 00000000..a6574179 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/repo/src/lib.rs @@ -0,0 +1,18 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct User { + #[serde(rename(serialize = "userName"))] + pub user_name: String, + + #[serde(rename(serialize = "emailAddress"))] + pub email_address: String, +} + +pub fn to_json(user: &User) -> String { + serde_json::to_string(user).expect("serialization failed") +} + +pub fn from_json(s: &str) -> Result { + serde_json::from_str(s) +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/repo/tests/test_fix.rs new file mode 100644 index 00000000..d126a292 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/repo/tests/test_fix.rs @@ -0,0 +1,34 @@ +use user_serde::{from_json, to_json, User}; + +#[test] +fn roundtrip_preserves_user_name() { + let original = User { + user_name: "alice".to_string(), + email_address: "alice@example.com".to_string(), + }; + let json = to_json(&original); + let restored: User = from_json(&json).expect("deserialization should succeed"); + assert_eq!(restored.user_name, original.user_name); +} + +#[test] +fn roundtrip_preserves_email_address() { + let original = User { + user_name: "bob".to_string(), + email_address: "bob@example.com".to_string(), + }; + let json = to_json(&original); + let restored: User = from_json(&json).expect("deserialization should succeed"); + assert_eq!(restored.email_address, original.email_address); +} + +#[test] +fn serialized_json_uses_camel_case_keys() { + let user = User { + user_name: "carol".to_string(), + email_address: "carol@example.com".to_string(), + }; + let json = to_json(&user); + assert!(json.contains("userName"), "JSON should contain 'userName' key"); + assert!(json.contains("emailAddress"), "JSON should contain 'emailAddress' key"); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/task.yaml new file mode 100644 index 00000000..1245acf3 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_007_serde_rename_roundtrip/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_007_serde_rename_roundtrip +family: bugfix +difficulty: medium +language: rust +max_minutes: 8 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 3 +task_type: single_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/metadata.json new file mode 100644 index 00000000..f2523a18 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 4, + "primary_skill": "rust-debugging", + "secondary_skills": ["utf8-handling", "string-manipulation"], + "name": "rust_bugfix_008_string_utf8_slice", + "difficulty": "medium", + "runtime_stressors": ["fault_localization", "test_interpretation"], + "task_type": "single_file_bugfix", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/prompt.txt new file mode 100644 index 00000000..9cffc701 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/prompt.txt @@ -0,0 +1 @@ +The truncate function panics on some input strings containing non-ASCII characters. Fix it so it safely truncates to the given number of characters without panicking on any valid UTF-8 input. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/repo/Cargo.lock new file mode 100644 index 00000000..6cab4ef1 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "string-util" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/repo/Cargo.toml new file mode 100644 index 00000000..47822dbd --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/repo/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "string-util" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/repo/src/lib.rs new file mode 100644 index 00000000..94da28c0 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/repo/src/lib.rs @@ -0,0 +1,6 @@ +pub fn truncate(s: &str, max_chars: usize) -> &str { + if s.len() <= max_chars { + return s; + } + &s[..max_chars] +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/repo/tests/test_fix.rs new file mode 100644 index 00000000..264ac5bf --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/repo/tests/test_fix.rs @@ -0,0 +1,32 @@ +use string_util::truncate; + +#[test] +fn ascii_string_truncates_correctly() { + assert_eq!(truncate("hello world", 5), "hello"); +} + +#[test] +fn ascii_string_shorter_than_limit_is_unchanged() { + assert_eq!(truncate("hi", 10), "hi"); +} + +#[test] +fn emoji_string_does_not_panic() { + let s = "Hello 🌍🌎🌏"; + let result = truncate(s, 7); + assert_eq!(result, "Hello 🌍"); +} + +#[test] +fn multibyte_characters_counted_by_char_not_byte() { + let s = "café!"; + let result = truncate(s, 4); + assert_eq!(result, "café"); +} + +#[test] +fn cjk_characters_do_not_panic() { + let s = "你好世界"; + let result = truncate(s, 2); + assert_eq!(result, "你好"); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/task.yaml new file mode 100644 index 00000000..7e18f761 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_008_string_utf8_slice/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_008_string_utf8_slice +family: bugfix +difficulty: medium +language: rust +max_minutes: 8 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 3 +task_type: single_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/metadata.json new file mode 100644 index 00000000..bb900a3f --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 3, + "primary_skill": "rust-debugging", + "secondary_skills": ["numeric-types", "error-handling"], + "name": "rust_bugfix_009_integer_as_truncation", + "difficulty": "medium", + "runtime_stressors": ["fault_localization", "test_interpretation"], + "task_type": "single_file_bugfix", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/prompt.txt new file mode 100644 index 00000000..5cccb9b0 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/prompt.txt @@ -0,0 +1 @@ +The byte converter silently produces wrong values for large inputs instead of reporting an error. Fix it so it returns an error when the input value is out of the valid byte range. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/repo/Cargo.lock new file mode 100644 index 00000000..0561411e --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "byte-convert" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/repo/Cargo.toml new file mode 100644 index 00000000..de8d6bcc --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/repo/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "byte-convert" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/repo/src/lib.rs new file mode 100644 index 00000000..ee703afd --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/repo/src/lib.rs @@ -0,0 +1,3 @@ +pub fn to_byte(value: u32) -> Result { + Ok(value as u8) +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/repo/tests/test_fix.rs new file mode 100644 index 00000000..4bbfeec9 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/repo/tests/test_fix.rs @@ -0,0 +1,38 @@ +use byte_convert::to_byte; + +#[test] +fn value_100_converts_to_ok() { + assert_eq!(to_byte(100), Ok(100u8)); +} + +#[test] +fn value_0_converts_to_ok() { + assert_eq!(to_byte(0), Ok(0u8)); +} + +#[test] +fn value_255_converts_to_ok() { + assert_eq!(to_byte(255), Ok(255u8)); +} + +#[test] +fn value_256_returns_err() { + assert!( + to_byte(256).is_err(), + "256 is out of u8 range and should return Err" + ); +} + +#[test] +fn value_1000_returns_err() { + assert!( + to_byte(1000).is_err(), + "1000 is out of u8 range and should return Err" + ); +} + +#[test] +fn out_of_range_does_not_silently_truncate() { + let result = to_byte(256); + assert!(result != Ok(0u8), "256 should not silently become 0"); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/task.yaml new file mode 100644 index 00000000..cd3217ae --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_009_integer_as_truncation/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_009_integer_as_truncation +family: bugfix +difficulty: medium +language: rust +max_minutes: 8 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 3 +task_type: single_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/metadata.json new file mode 100644 index 00000000..a320e947 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 2, + "primary_skill": "rust-debugging", + "secondary_skills": ["sorting", "stability"], + "name": "rust_bugfix_010_sort_unstable_order", + "difficulty": "medium", + "runtime_stressors": ["fault_localization", "test_interpretation"], + "task_type": "single_file_bugfix", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/prompt.txt new file mode 100644 index 00000000..22c6c64d --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/prompt.txt @@ -0,0 +1 @@ +Records with equal priority are returned in a different order than they were inserted. Fix the sort so that insertion order is preserved for records with the same priority. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/repo/Cargo.lock new file mode 100644 index 00000000..d420dae1 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "record-sort" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/repo/Cargo.toml new file mode 100644 index 00000000..f659edee --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/repo/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "record-sort" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/repo/src/lib.rs new file mode 100644 index 00000000..ccdb006a --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/repo/src/lib.rs @@ -0,0 +1,22 @@ +#[derive(Debug, Clone, PartialEq)] +pub struct Record { + pub name: String, + pub priority: u32, +} + +impl Record { + pub fn new(name: &str, priority: u32) -> Self { + Record { + name: name.to_string(), + priority, + } + } +} + +pub fn sort_records(records: &mut Vec) { + records.sort_by(|a, b| { + a.priority + .cmp(&b.priority) + .then(b.name.cmp(&a.name)) + }); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/repo/tests/test_fix.rs new file mode 100644 index 00000000..b1b026c0 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/repo/tests/test_fix.rs @@ -0,0 +1,42 @@ +use record_sort::{sort_records, Record}; + +#[test] +fn records_sorted_by_priority_ascending() { + let mut records = vec![ + Record::new("high", 3), + Record::new("low", 1), + Record::new("mid", 2), + ]; + sort_records(&mut records); + assert_eq!(records[0].name, "low"); + assert_eq!(records[1].name, "mid"); + assert_eq!(records[2].name, "high"); +} + +#[test] +fn equal_priority_preserves_insertion_order() { + let mut records = vec![ + Record::new("first", 1), + Record::new("alpha", 2), + Record::new("beta", 2), + Record::new("last", 3), + ]; + sort_records(&mut records); + assert_eq!(records[0].name, "first"); + assert_eq!(records[1].name, "alpha", "alpha should retain insertion order before beta"); + assert_eq!(records[2].name, "beta", "beta should retain insertion order after alpha"); + assert_eq!(records[3].name, "last"); +} + +#[test] +fn all_equal_priority_preserves_insertion_order() { + let mut records = vec![ + Record::new("a", 5), + Record::new("b", 5), + Record::new("c", 5), + Record::new("d", 5), + ]; + sort_records(&mut records); + let names: Vec<&str> = records.iter().map(|r| r.name.as_str()).collect(); + assert_eq!(names, vec!["a", "b", "c", "d"]); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/task.yaml new file mode 100644 index 00000000..1e970583 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_010_sort_unstable_order/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_010_sort_unstable_order +family: bugfix +difficulty: medium +language: rust +max_minutes: 8 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 3 +task_type: single_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/metadata.json new file mode 100644 index 00000000..722bb012 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 4, + "primary_skill": "rust-debugging", + "secondary_skills": ["cargo-test", "hashmap"], + "name": "rust_bugfix_011_hashmap_cache_stale", + "difficulty": "medium", + "runtime_stressors": ["fault_localization", "test_interpretation"], + "task_type": "single_file_bugfix", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/prompt.txt new file mode 100644 index 00000000..605f1338 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/prompt.txt @@ -0,0 +1 @@ +After updating a cached value, reading it back still returns the old value. Fix the cache so reads after writes return the updated value. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/repo/Cargo.lock new file mode 100644 index 00000000..4201008f --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "hashmap-cache" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/repo/Cargo.toml new file mode 100644 index 00000000..1775b055 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/repo/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "hashmap-cache" +version = "0.1.0" +edition = "2021" + +[dependencies] + +[[test]] +name = "test_fix" +path = "tests/test_fix.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/repo/src/lib.rs new file mode 100644 index 00000000..a16dc399 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/repo/src/lib.rs @@ -0,0 +1,31 @@ +use std::collections::HashMap; + +pub struct SimpleCache { + store: HashMap, +} + +impl SimpleCache { + pub fn new() -> Self { + SimpleCache { + store: HashMap::new(), + } + } + + pub fn get(&self, key: &str) -> Option<&str> { + self.store.get(key).map(|s| s.as_str()) + } + + pub fn set(&mut self, key: &str, value: &str) { + let old = self.store.get(key).cloned(); + self.store.insert(key.to_string(), value.to_string()); + if let Some(prev) = old { + self.store.insert(key.to_string(), prev); + } + } +} + +impl Default for SimpleCache { + fn default() -> Self { + Self::new() + } +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/repo/tests/test_fix.rs new file mode 100644 index 00000000..d403c95f --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/repo/tests/test_fix.rs @@ -0,0 +1,48 @@ +use hashmap_cache::SimpleCache; + +#[test] +fn test_set_then_get_returns_new_value() { + let mut cache = SimpleCache::new(); + cache.set("key", "v1"); + assert_eq!( + cache.get("key"), + Some("v1"), + "first set should store 'v1'" + ); + cache.set("key", "v2"); + assert_eq!( + cache.get("key"), + Some("v2"), + "after update, get should return 'v2' not the stale 'v1'" + ); +} + +#[test] +fn test_insert_new_key() { + let mut cache = SimpleCache::new(); + assert_eq!(cache.get("missing"), None, "absent key should return None"); + cache.set("new_key", "hello"); + assert_eq!( + cache.get("new_key"), + Some("hello"), + "newly inserted key should be retrievable" + ); +} + +#[test] +fn test_multiple_keys_independent() { + let mut cache = SimpleCache::new(); + cache.set("a", "alpha"); + cache.set("b", "beta"); + cache.set("a", "ALPHA"); + assert_eq!( + cache.get("a"), + Some("ALPHA"), + "key 'a' should reflect updated value" + ); + assert_eq!( + cache.get("b"), + Some("beta"), + "key 'b' should be unaffected by update to 'a'" + ); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/task.yaml new file mode 100644 index 00000000..c499b36d --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_011_hashmap_cache_stale/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_011_hashmap_cache_stale +family: bugfix +difficulty: medium +language: rust +max_minutes: 5 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 3 +task_type: single_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/metadata.json new file mode 100644 index 00000000..fa5d9ae0 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 3, + "primary_skill": "rust-debugging", + "secondary_skills": ["cargo-test", "rust-std-path"], + "name": "rust_bugfix_012_path_join_absolute", + "difficulty": "medium", + "runtime_stressors": ["fault_localization", "test_interpretation"], + "task_type": "single_file_bugfix", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/prompt.txt new file mode 100644 index 00000000..0d060268 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/prompt.txt @@ -0,0 +1 @@ +The config file resolver ignores the base directory and always resolves paths relative to the filesystem root. When given a base of "/home/app" and a filename of "config/app.toml", the returned path should be "/home/app/config/app.toml", but the function returns a path that discards the base entirely. Fix the path construction so the base directory is respected. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/repo/Cargo.lock new file mode 100644 index 00000000..f36cf823 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "config-loader" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/repo/Cargo.toml new file mode 100644 index 00000000..5bda4bee --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/repo/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "config-loader" +version = "0.1.0" +edition = "2021" + +[lib] +name = "config_loader" +path = "src/lib.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/repo/src/lib.rs new file mode 100644 index 00000000..abc86052 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/repo/src/lib.rs @@ -0,0 +1,5 @@ +use std::path::{Path, PathBuf}; + +pub fn resolve_config(base: &Path, filename: &str) -> PathBuf { + base.join(filename) +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/repo/tests/test_fix.rs new file mode 100644 index 00000000..b2926e39 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/repo/tests/test_fix.rs @@ -0,0 +1,23 @@ +use config_loader::resolve_config; +use std::path::Path; + +#[test] +fn base_dir_is_respected_when_filename_has_leading_slash() { + let base = Path::new("/home/app"); + let result = resolve_config(base, "/config/app.toml"); + assert_eq!(result, Path::new("/home/app/config/app.toml")); +} + +#[test] +fn relative_filename_is_joined_correctly() { + let base = Path::new("/home/app"); + let result = resolve_config(base, "config/app.toml"); + assert_eq!(result, Path::new("/home/app/config/app.toml")); +} + +#[test] +fn nested_base_and_relative_filename() { + let base = Path::new("/srv/myapp/data"); + let result = resolve_config(base, "/settings.json"); + assert_eq!(result, Path::new("/srv/myapp/data/settings.json")); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/task.yaml new file mode 100644 index 00000000..0bba0a09 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_012_path_join_absolute/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_012_path_join_absolute +family: bugfix +difficulty: medium +language: rust +max_minutes: 8 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 3 +task_type: single_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/metadata.json new file mode 100644 index 00000000..86121987 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 2, + "primary_skill": "rust-debugging", + "secondary_skills": ["cargo-test", "rust-generics", "rust-traits"], + "name": "rust_bugfix_013_trait_bound_missing", + "difficulty": "hard", + "runtime_stressors": ["fault_localization", "test_interpretation", "compile_error"], + "task_type": "single_file_bugfix", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/prompt.txt new file mode 100644 index 00000000..96cff648 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/prompt.txt @@ -0,0 +1 @@ +The generic print_items function fails to compile when called with any type. Fix the function signature so it works correctly with any displayable type and the tests pass. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/repo/Cargo.lock new file mode 100644 index 00000000..6704d5d6 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "item-printer" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/repo/Cargo.toml new file mode 100644 index 00000000..93f073e9 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/repo/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "item-printer" +version = "0.1.0" +edition = "2021" + +[lib] +name = "item_printer" +path = "src/lib.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/repo/src/lib.rs new file mode 100644 index 00000000..91dac25e --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/repo/src/lib.rs @@ -0,0 +1,7 @@ +fn print_items(items: &[T]) -> Vec { + items.iter().map(|x| format!("{}", x)).collect() +} + +pub fn collect_display(items: &[T]) -> Vec { + print_items(items) +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/repo/tests/test_fix.rs new file mode 100644 index 00000000..61363670 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/repo/tests/test_fix.rs @@ -0,0 +1,22 @@ +use item_printer::collect_display; + +#[test] +fn test_integers_display() { + let nums = vec![1_i32, 2, 3]; + let result = collect_display(&nums); + assert_eq!(result, vec!["1", "2", "3"]); +} + +#[test] +fn test_strings_display() { + let words = vec!["hello".to_string(), "world".to_string()]; + let result = collect_display(&words); + assert_eq!(result, vec!["hello", "world"]); +} + +#[test] +fn test_empty_slice() { + let empty: Vec = vec![]; + let result = collect_display(&empty); + assert!(result.is_empty()); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/task.yaml new file mode 100644 index 00000000..df5dcfad --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_013_trait_bound_missing/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_013_trait_bound_missing +family: bugfix +difficulty: hard +language: rust +max_minutes: 10 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 3 +task_type: single_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/metadata.json b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/metadata.json new file mode 100644 index 00000000..496ab263 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs", "src/errors.rs"], + "reference_patch_size_lines": 6, + "primary_skill": "rust-debugging", + "secondary_skills": ["cargo-test", "rust-error-handling", "rust-traits"], + "name": "rust_bugfix_014_error_chain_swallowed", + "difficulty": "hard", + "runtime_stressors": ["fault_localization", "test_interpretation", "multi_file_patch"], + "task_type": "multi_file_bugfix", + "requires_repo_navigation": true, + "requires_multi_step_reasoning": true, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests", "error_chain_preservation"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/prompt.txt new file mode 100644 index 00000000..074b69d6 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/prompt.txt @@ -0,0 +1 @@ +Error messages from the data loader always say "generic error" regardless of what actually went wrong, making it impossible to diagnose failures. Fix the error handling so that the original error details are preserved and visible in the error message. diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/Cargo.lock new file mode 100644 index 00000000..dcce1883 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "data-loader" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/Cargo.toml new file mode 100644 index 00000000..4a154009 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "data-loader" +version = "0.1.0" +edition = "2021" + +[lib] +name = "data_loader" +path = "src/lib.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/src/errors.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/src/errors.rs new file mode 100644 index 00000000..08a4aa3b --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/src/errors.rs @@ -0,0 +1,20 @@ +use std::fmt; + +#[derive(Debug)] +pub enum AppError { + Generic, + ParseError(String), + NotFound(String), +} + +impl fmt::Display for AppError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + AppError::Generic => write!(f, "generic error"), + AppError::ParseError(msg) => write!(f, "parse error: {}", msg), + AppError::NotFound(key) => write!(f, "not found: {}", key), + } + } +} + +impl std::error::Error for AppError {} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/src/lib.rs new file mode 100644 index 00000000..b7fb0747 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/src/lib.rs @@ -0,0 +1,21 @@ +pub mod errors; + +use errors::AppError; + +pub fn load_value(line: &str) -> Result { + let parts: Vec<&str> = line.splitn(2, '=').collect(); + if parts.len() != 2 { + return Err(AppError::NotFound(line.to_string())); + } + let value_str = parts[1].trim(); + let value: i64 = value_str.parse().map_err(|_| AppError::Generic)?; + Ok(value) +} + +pub fn load_sum(lines: &[&str]) -> Result { + let mut total = 0i64; + for line in lines { + total += load_value(line)?; + } + Ok(total) +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/tests/test_fix.rs new file mode 100644 index 00000000..9c69bef5 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/repo/tests/test_fix.rs @@ -0,0 +1,40 @@ +use data_loader::{load_value, load_sum}; + +#[test] +fn test_valid_line_parses() { + assert_eq!(load_value("count=42").unwrap(), 42); +} + +#[test] +fn test_parse_error_contains_original_cause() { + let err = load_value("count=not_a_number").unwrap_err(); + let msg = err.to_string(); + assert!( + !msg.contains("generic error"), + "Expected a meaningful parse error, got: {}", + msg + ); + assert!( + msg.contains("parse") || msg.contains("invalid") || msg.contains("not_a_number"), + "Error message should describe the parse failure, got: {}", + msg + ); +} + +#[test] +fn test_sum_valid_lines() { + let lines = vec!["a=10", "b=20", "c=30"]; + assert_eq!(load_sum(&lines).unwrap(), 60); +} + +#[test] +fn test_sum_propagates_meaningful_error() { + let lines = vec!["a=10", "b=bad_value", "c=30"]; + let err = load_sum(&lines).unwrap_err(); + let msg = err.to_string(); + assert!( + !msg.contains("generic error"), + "Sum should propagate meaningful error, got: {}", + msg + ); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/task.yaml b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/task.yaml new file mode 100644 index 00000000..a542b99b --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_bugfix_014_error_chain_swallowed/task.yaml @@ -0,0 +1,28 @@ +id: rust_bugfix_014_error_chain_swallowed +family: bugfix +difficulty: hard +language: rust +max_minutes: 10 +max_files_touched: 4 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 4 +task_type: multi_file_bugfix +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/metadata.json b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/metadata.json new file mode 100644 index 00000000..58909c2e --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/app.rs"], + "reference_patch_size_lines": 2, + "primary_skill": "rust-debugging", + "secondary_skills": ["cargo-test", "rust-modules", "fault_localization"], + "name": "rust_localize_001_feature_flag_propagation", + "difficulty": "hard", + "runtime_stressors": ["fault_localization", "test_interpretation", "multi_file_patch", "search_required"], + "task_type": "localize", + "requires_repo_navigation": true, + "requires_multi_step_reasoning": true, + "has_false_fix_trap": true, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["localization_accuracy", "correctness", "deterministic_tests"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/prompt.txt new file mode 100644 index 00000000..0797d3ba --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/prompt.txt @@ -0,0 +1 @@ +A feature flag is enabled in the configuration but has no effect at runtime — the application always behaves as if the flag is disabled. Find the issue and make the minimal correct fix. diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/Cargo.lock new file mode 100644 index 00000000..b89be6d1 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "feature-flags" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/Cargo.toml new file mode 100644 index 00000000..e91a39fa --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "feature-flags" +version = "0.1.0" +edition = "2021" + +[lib] +name = "feature_flags" +path = "src/lib.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/src/app.rs b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/src/app.rs new file mode 100644 index 00000000..d33e630f --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/src/app.rs @@ -0,0 +1,24 @@ +use crate::config::Config; + +pub struct App { + config: Config, +} + +impl App { + pub fn new(config: Config) -> Self { + App { config } + } + + pub fn greet(&self, name: &str) -> String { + let fancy = false; + if fancy { + format!("Hello, {}!", name) + } else { + format!("Hello, {}", name) + } + } + + pub fn dark_mode_active(&self) -> bool { + false + } +} diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/src/config.rs b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/src/config.rs new file mode 100644 index 00000000..f248ad99 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/src/config.rs @@ -0,0 +1,21 @@ +use std::collections::HashMap; + +pub struct Config { + flags: HashMap, +} + +impl Config { + pub fn new() -> Self { + Config { + flags: HashMap::new(), + } + } + + pub fn set_flag(&mut self, name: &str, value: bool) { + self.flags.insert(name.to_string(), value); + } + + pub fn is_enabled(&self, name: &str) -> bool { + *self.flags.get(name).unwrap_or(&false) + } +} diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/src/lib.rs new file mode 100644 index 00000000..5d692a2c --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/src/lib.rs @@ -0,0 +1,2 @@ +pub mod app; +pub mod config; diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/tests/test_fix.rs new file mode 100644 index 00000000..2199afb0 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/repo/tests/test_fix.rs @@ -0,0 +1,42 @@ +use feature_flags::app::App; +use feature_flags::config::Config; + +fn make_app_with_flags(flags: &[(&str, bool)]) -> App { + let mut config = Config::new(); + for (name, value) in flags { + config.set_flag(name, *value); + } + App::new(config) +} + +#[test] +fn test_fancy_greeting_when_flag_disabled() { + let app = make_app_with_flags(&[("fancy_greeting", false)]); + assert_eq!(app.greet("Alice"), "Hello, Alice"); +} + +#[test] +fn test_fancy_greeting_when_flag_enabled() { + let app = make_app_with_flags(&[("fancy_greeting", true)]); + let result = app.greet("Alice"); + assert_eq!( + result, "Hello, Alice!", + "fancy_greeting flag is enabled but greeting has no exclamation: {:?}", + result + ); +} + +#[test] +fn test_dark_mode_off_by_default() { + let app = make_app_with_flags(&[]); + assert!(!app.dark_mode_active()); +} + +#[test] +fn test_dark_mode_on_when_flag_enabled() { + let app = make_app_with_flags(&[("dark_mode", true)]); + assert!( + app.dark_mode_active(), + "dark_mode flag is enabled in config but dark_mode_active() returned false" + ); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/task.yaml b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/task.yaml new file mode 100644 index 00000000..5b8b9cc2 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_001_feature_flag_propagation/task.yaml @@ -0,0 +1,28 @@ +id: rust_localize_001_feature_flag_propagation +family: localize_patch +difficulty: hard +language: rust +max_minutes: 10 +max_files_touched: 4 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 4 +task_type: localize +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/metadata.json b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/metadata.json new file mode 100644 index 00000000..39a57ed9 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/auth.rs"], + "reference_patch_size_lines": 2, + "primary_skill": "rust-debugging", + "secondary_skills": ["http-middleware", "fault_localization", "case-sensitivity"], + "name": "rust_localize_002_auth_middleware_drop", + "difficulty": "hard", + "runtime_stressors": ["fault_localization", "multi_file_navigation", "subtle_interaction"], + "task_type": "localize_patch", + "requires_repo_navigation": true, + "requires_multi_step_reasoning": true, + "has_false_fix_trap": true, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "trace_data_flow", "edit_files", "run_tests"], + "evaluation_focus": ["minimal_patch", "root_cause_identification"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/prompt.txt new file mode 100644 index 00000000..77f4e209 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/prompt.txt @@ -0,0 +1 @@ +Authenticated API requests are being rejected even when a valid token is provided. The Authorization header is set correctly by the caller but the server always returns 401 Unauthorized. Find the root cause in the middleware pipeline and apply the minimal correct fix. diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/Cargo.lock new file mode 100644 index 00000000..ab26c330 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "auth-middleware" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/Cargo.toml new file mode 100644 index 00000000..2588be4d --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "auth-middleware" +version = "0.1.0" +edition = "2021" + +[[test]] +name = "test_auth" +path = "tests/test_auth.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/src/auth.rs b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/src/auth.rs new file mode 100644 index 00000000..8093c3cf --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/src/auth.rs @@ -0,0 +1,9 @@ +use std::collections::HashMap; + +pub fn check_auth(headers: &HashMap) -> Result<(), String> { + match headers.get("Authorization") { + Some(token) if !token.is_empty() => Ok(()), + Some(_) => Err("Authorization token is empty".to_string()), + None => Err("Missing Authorization header".to_string()), + } +} diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/src/client.rs b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/src/client.rs new file mode 100644 index 00000000..fa40f058 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/src/client.rs @@ -0,0 +1,40 @@ +use std::collections::HashMap; + +use crate::auth::check_auth; +use crate::normalize::normalize_headers; + +#[derive(Debug, Clone)] +pub struct Request { + pub headers: HashMap, + pub body: String, +} + +impl Request { + pub fn new(headers: HashMap, body: impl Into) -> Self { + Request { + headers, + body: body.into(), + } + } +} + +#[derive(Debug, Clone)] +pub struct Response { + pub status: u16, + pub body: String, +} + +pub fn process_request(req: Request) -> Response { + let normalised = normalize_headers(req.headers); + + match check_auth(&normalised) { + Ok(()) => Response { + status: 200, + body: "OK".to_string(), + }, + Err(reason) => Response { + status: 401, + body: format!("Unauthorized: {}", reason), + }, + } +} diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/src/lib.rs new file mode 100644 index 00000000..54d79a67 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/src/lib.rs @@ -0,0 +1,3 @@ +pub mod auth; +pub mod client; +pub mod normalize; diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/src/normalize.rs b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/src/normalize.rs new file mode 100644 index 00000000..74d92c65 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/src/normalize.rs @@ -0,0 +1,8 @@ +use std::collections::HashMap; + +pub fn normalize_headers(headers: HashMap) -> HashMap { + headers + .into_iter() + .map(|(k, v)| (k.to_lowercase(), v)) + .collect() +} diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/tests/test_auth.rs b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/tests/test_auth.rs new file mode 100644 index 00000000..12004678 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/repo/tests/test_auth.rs @@ -0,0 +1,54 @@ +use std::collections::HashMap; + +use auth_middleware::client::{process_request, Request}; + +fn make_request_with_auth(token: &str) -> Request { + let mut headers = HashMap::new(); + headers.insert("Authorization".to_string(), token.to_string()); + headers.insert("Content-Type".to_string(), "application/json".to_string()); + Request::new(headers, r#"{"action":"read"}"#) +} + +#[test] +fn test_authenticated_request_returns_200() { + let req = make_request_with_auth("Bearer valid-token-abc123"); + let resp = process_request(req); + assert_eq!( + resp.status, 200, + "a request with a valid Authorization header should return 200, got {}; body: {}", + resp.status, resp.body + ); +} + +#[test] +fn test_unauthenticated_request_returns_401() { + let headers = HashMap::new(); + let req = Request::new(headers, "{}"); + let resp = process_request(req); + assert_eq!( + resp.status, 401, + "a request with no Authorization header should return 401" + ); +} + +#[test] +fn test_empty_token_returns_401() { + let req = make_request_with_auth(""); + let resp = process_request(req); + assert_eq!( + resp.status, 401, + "a request with an empty Authorization token should return 401" + ); +} + +#[test] +fn test_mixed_case_header_key_is_accepted() { + let mut headers = HashMap::new(); + headers.insert("authorization".to_string(), "Bearer another-token".to_string()); + let req = Request::new(headers, "{}"); + let resp = process_request(req); + assert_eq!( + resp.status, 200, + "a request with lowercase 'authorization' header should return 200" + ); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/task.yaml b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/task.yaml new file mode 100644 index 00000000..71671f18 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_localize_002_auth_middleware_drop/task.yaml @@ -0,0 +1,29 @@ +id: rust_localize_002_auth_middleware_drop +family: localize_patch +difficulty: hard +language: rust +max_minutes: 6 +max_files_touched: 4 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_auth -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +allowed_paths: +- target/ +- src/ +forbidden_paths: +- tests/ +- Cargo.toml +expected_touched_max: 4 +task_type: localize_patch +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/metadata.json b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/metadata.json new file mode 100644 index 00000000..cdc6e878 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["Cargo.toml"], + "reference_patch_size_lines": 2, + "primary_skill": "cargo-configuration", + "secondary_skills": ["cargo-test", "rust-project-structure"], + "name": "rust_terminal_001_cargo_test_path", + "difficulty": "easy", + "runtime_stressors": ["fault_localization", "test_interpretation"], + "task_type": "terminal", + "requires_repo_navigation": false, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests", "cargo_config"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/prompt.txt new file mode 100644 index 00000000..b42428ac --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/prompt.txt @@ -0,0 +1 @@ +Running `cargo test` fails because Cargo cannot find the integration test file. Fix the project configuration so the test command succeeds. diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/repo/Cargo.lock new file mode 100644 index 00000000..4df6f185 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "greeter" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/repo/Cargo.toml new file mode 100644 index 00000000..2d1c44c3 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/repo/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "greeter" +version = "0.1.0" +edition = "2021" + +[lib] +name = "greeter" +path = "src/lib.rs" + +[[test]] +name = "test_integration" +path = "test/test_integration.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/repo/src/lib.rs new file mode 100644 index 00000000..2a5264b2 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/repo/src/lib.rs @@ -0,0 +1,3 @@ +pub fn greet(name: &str) -> String { + format!("Hello, {}!", name) +} diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/repo/tests/test_integration.rs b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/repo/tests/test_integration.rs new file mode 100644 index 00000000..7f590fdd --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/repo/tests/test_integration.rs @@ -0,0 +1,11 @@ +use greeter::greet; + +#[test] +fn test_greet_returns_hello() { + assert_eq!(greet("World"), "Hello, World!"); +} + +#[test] +fn test_greet_with_name() { + assert_eq!(greet("Rust"), "Hello, Rust!"); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/task.yaml b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/task.yaml new file mode 100644 index 00000000..938d0d5e --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_001_cargo_test_path/task.yaml @@ -0,0 +1,30 @@ +id: rust_terminal_001_cargo_test_path +family: terminal_workflow +difficulty: easy +language: rust +max_minutes: 5 +max_files_touched: 2 +expected_artifacts: +- patch +visible_verification: +- cargo test -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +- Cargo.toml +allowed_paths: +- target/ +- src/ +- tests/ +- Cargo.toml +expected_touched_max: 2 +task_type: terminal +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/metadata.json b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/metadata.json new file mode 100644 index 00000000..d794f5af --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 4, + "primary_skill": "rust-module-system", + "secondary_skills": ["cargo-build", "rust-project-structure"], + "name": "rust_terminal_002_module_structure", + "difficulty": "medium", + "runtime_stressors": ["compile_error_diagnosis", "file_navigation"], + "task_type": "terminal", + "requires_repo_navigation": true, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": false, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "list_directory", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "compile_success"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/prompt.txt new file mode 100644 index 00000000..9c842003 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/prompt.txt @@ -0,0 +1 @@ +The project fails to compile because a module cannot be found. Fix the module structure so the project builds and all tests pass. diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/Cargo.lock new file mode 100644 index 00000000..c9d5d92c --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "module-structure" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/Cargo.toml new file mode 100644 index 00000000..cf99cb19 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "module-structure" +version = "0.1.0" +edition = "2021" + +[lib] +name = "module_structure" +path = "src/lib.rs" + +[[bin]] +name = "module-structure" +path = "src/main.rs" + +[[test]] +name = "test_build" +path = "tests/test_build.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/src/helpers/mod.rs b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/src/helpers/mod.rs new file mode 100644 index 00000000..b5614dd8 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/src/helpers/mod.rs @@ -0,0 +1 @@ +pub mod utils; diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/src/helpers/utils.rs b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/src/helpers/utils.rs new file mode 100644 index 00000000..699b6694 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/src/helpers/utils.rs @@ -0,0 +1,7 @@ +pub fn greet(name: &str) -> String { + format!("Hello, {}!", name) +} + +pub fn add(a: i32, b: i32) -> i32 { + a + b +} diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/src/lib.rs new file mode 100644 index 00000000..b3a5dd7a --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/src/lib.rs @@ -0,0 +1,4 @@ +mod utils; + +pub use utils::greet; +pub use utils::add; diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/src/main.rs b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/src/main.rs new file mode 100644 index 00000000..216d73df --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/src/main.rs @@ -0,0 +1,6 @@ +use module_structure::{greet, add}; + +fn main() { + println!("{}", greet("world")); + println!("1 + 2 = {}", add(1, 2)); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/tests/test_build.rs b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/tests/test_build.rs new file mode 100644 index 00000000..c779b00f --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/repo/tests/test_build.rs @@ -0,0 +1,30 @@ +use module_structure::{greet, add}; + +#[test] +fn test_greet_returns_hello_message() { + let result = greet("Alice"); + assert_eq!(result, "Hello, Alice!"); +} + +#[test] +fn test_greet_world() { + let result = greet("world"); + assert_eq!(result, "Hello, world!"); +} + +#[test] +fn test_add_positive_numbers() { + assert_eq!(add(1, 2), 3); +} + +#[test] +fn test_add_with_zero() { + assert_eq!(add(0, 5), 5); + assert_eq!(add(7, 0), 7); +} + +#[test] +fn test_add_negative_numbers() { + assert_eq!(add(-3, 3), 0); + assert_eq!(add(-1, -1), -2); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/task.yaml b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/task.yaml new file mode 100644 index 00000000..569f90b3 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_002_module_structure/task.yaml @@ -0,0 +1,30 @@ +id: rust_terminal_002_module_structure +family: terminal_workflow +difficulty: medium +language: rust +max_minutes: 5 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +- Cargo.toml +allowed_paths: +- target/ +- src/ +- tests/ +- Cargo.toml +expected_touched_max: 3 +task_type: terminal +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/metadata.json b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/metadata.json new file mode 100644 index 00000000..a5e1dc19 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/lib.rs"], + "reference_patch_size_lines": 4, + "primary_skill": "cargo-features", + "secondary_skills": ["rust-cfg-attributes", "cargo-toml"], + "name": "rust_terminal_003_feature_flag_cargo", + "difficulty": "medium", + "runtime_stressors": ["feature_flag_mismatch", "silent_wrong_behavior"], + "task_type": "terminal", + "requires_repo_navigation": true, + "requires_multi_step_reasoning": false, + "has_false_fix_trap": true, + "requires_retry_recovery": false, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "cfg_attribute_usage"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/prompt.txt new file mode 100644 index 00000000..b2fa63cc --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/prompt.txt @@ -0,0 +1 @@ +The JSON output feature is implemented in the codebase and enabled by default in Cargo.toml, but tests expecting JSON output always fail — the function returns plain text instead. Find the misconfiguration and fix it so all tests pass. diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/repo/Cargo.lock new file mode 100644 index 00000000..16edd75a --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/repo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "feature-flag-cargo" +version = "0.1.0" diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/repo/Cargo.toml new file mode 100644 index 00000000..fc07343d --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/repo/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "feature-flag-cargo" +version = "0.1.0" +edition = "2021" + +[features] +default = ["json_output"] +json_output = [] + +[[test]] +name = "test_format" +path = "tests/test_format.rs" diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/repo/src/lib.rs new file mode 100644 index 00000000..d568a86f --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/repo/src/lib.rs @@ -0,0 +1,19 @@ +pub fn format_plain(value: &str) -> String { + value.trim().to_string() +} + +#[cfg(feature = "json_fmt")] +pub fn format_json(value: &str) -> String { + format!("{{\"value\":\"{}\"}}", value.trim()) +} + +pub fn format_output(value: &str) -> String { + #[cfg(feature = "json_fmt")] + { + format_json(value) + } + #[cfg(not(feature = "json_fmt"))] + { + format_plain(value) + } +} diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/repo/tests/test_format.rs b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/repo/tests/test_format.rs new file mode 100644 index 00000000..0cf909fa --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/repo/tests/test_format.rs @@ -0,0 +1,29 @@ +use feature_flag_cargo::format_output; + +#[test] +fn test_format_output_plain_always_works() { + #[cfg(not(feature = "json_output"))] + { + let result = format_output(" hello "); + assert_eq!(result, "hello", "plain output should trim whitespace"); + } + #[cfg(feature = "json_output")] + { + let result = format_output(" hello "); + assert!( + result.contains("hello"), + "output should contain the value" + ); + } +} + +#[test] +fn test_format_output_produces_json_with_default_features() { + let result = format_output(" hello "); + assert_eq!( + result, + "{\"value\":\"hello\"}", + "default features should enable json_output; got plain text instead — \ + check the [features] default list in Cargo.toml" + ); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/task.yaml b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/task.yaml new file mode 100644 index 00000000..59fc43d3 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_terminal_003_feature_flag_cargo/task.yaml @@ -0,0 +1,30 @@ +id: rust_terminal_003_feature_flag_cargo +family: terminal_workflow +difficulty: medium +language: rust +max_minutes: 5 +max_files_touched: 3 +expected_artifacts: +- patch +visible_verification: +- cargo test -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +- Cargo.toml +allowed_paths: +- target/ +- src/ +- tests/ +- Cargo.toml +expected_touched_max: 3 +task_type: terminal +benchmark_track: core diff --git a/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/metadata.json b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/metadata.json new file mode 100644 index 00000000..aa4be501 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/metadata.json @@ -0,0 +1,18 @@ +{ + "seed_type": "bug-seeded", + "expected_files": ["src/models.rs", "src/cache.rs"], + "reference_patch_size_lines": 4, + "primary_skill": "rust-debugging", + "secondary_skills": ["cargo-test", "rust-serde", "rust-hashmap"], + "name": "rust_two_stage_fix", + "difficulty": "hard", + "runtime_stressors": ["fault_localization", "test_interpretation", "multi_file_patch", "two_bugs"], + "task_type": "adjacent_two_file_fix", + "requires_repo_navigation": true, + "requires_multi_step_reasoning": true, + "has_false_fix_trap": true, + "requires_retry_recovery": true, + "likely_tool_sequence": ["read_files", "edit_files", "run_tests"], + "evaluation_focus": ["correctness", "deterministic_tests", "two_bug_repair"], + "benchmark_bucket": "baseline" +} diff --git a/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/prompt.txt b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/prompt.txt new file mode 100644 index 00000000..7f37d256 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/prompt.txt @@ -0,0 +1 @@ +Cached user lookups always miss even for users that were just inserted, and deserialized user records appear to be missing the email field. There may be more than one bug. Fix the issues so that serialization round-trips correctly and cache lookups hit as expected. diff --git a/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/Cargo.lock b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/Cargo.lock new file mode 100644 index 00000000..d0497d06 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/Cargo.lock @@ -0,0 +1,107 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "user-cache" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/Cargo.toml b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/Cargo.toml new file mode 100644 index 00000000..bf69ae84 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "user-cache" +version = "0.1.0" +edition = "2021" + +[lib] +name = "user_cache" +path = "src/lib.rs" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/src/cache.rs b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/src/cache.rs new file mode 100644 index 00000000..7aaf0570 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/src/cache.rs @@ -0,0 +1,26 @@ +use std::collections::HashMap; +use crate::models::User; + +pub struct UserCache { + store: HashMap, +} + +impl UserCache { + pub fn new() -> Self { + UserCache { + store: HashMap::new(), + } + } + + pub fn insert(&mut self, user: User) { + self.store.insert(user.username.clone(), user); + } + + pub fn get_by_username(&self, username: &str) -> Option<&User> { + self.store.get(username) + } + + pub fn get_by_email(&self, email: &str) -> Option<&User> { + self.store.get(email) + } +} diff --git a/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/src/lib.rs b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/src/lib.rs new file mode 100644 index 00000000..0c440f6a --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/src/lib.rs @@ -0,0 +1,2 @@ +pub mod cache; +pub mod models; diff --git a/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/src/models.rs b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/src/models.rs new file mode 100644 index 00000000..11cbe408 --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/src/models.rs @@ -0,0 +1,9 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct User { + pub id: u32, + pub username: String, + #[serde(rename = "email_address", default)] + pub email: String, +} diff --git a/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/tests/test_fix.rs b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/tests/test_fix.rs new file mode 100644 index 00000000..69588c2f --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/repo/tests/test_fix.rs @@ -0,0 +1,57 @@ +use user_cache::cache::UserCache; +use user_cache::models::User; + +fn make_user() -> User { + User { + id: 1, + username: "alice".to_string(), + email: "alice@example.com".to_string(), + } +} + +#[test] +fn test_serialize_deserialize_roundtrip() { + let user = make_user(); + let json = serde_json::to_string(&user).unwrap(); + let restored: User = serde_json::from_str(&json).unwrap(); + assert_eq!(restored, user); +} + +#[test] +fn test_deserialize_from_canonical_json() { + let json = r#"{"id":1,"username":"alice","email":"alice@example.com"}"#; + let user: User = serde_json::from_str(json).unwrap(); + assert_eq!( + user.email, "alice@example.com", + "email field should be populated from JSON key 'email', got: {:?}", + user.email + ); +} + +#[test] +fn test_cache_hit_by_username() { + let mut cache = UserCache::new(); + cache.insert(make_user()); + let found = cache.get_by_username("alice"); + assert!(found.is_some(), "Should find user by username"); + assert_eq!(found.unwrap().email, "alice@example.com"); +} + +#[test] +fn test_cache_hit_by_email() { + let mut cache = UserCache::new(); + cache.insert(make_user()); + let found = cache.get_by_email("alice@example.com"); + assert!( + found.is_some(), + "Should find user by email but got None — cache lookup is broken" + ); + assert_eq!(found.unwrap().username, "alice"); +} + +#[test] +fn test_cache_miss_for_unknown_email() { + let mut cache = UserCache::new(); + cache.insert(make_user()); + assert!(cache.get_by_email("nobody@example.com").is_none()); +} diff --git a/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/task.yaml b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/task.yaml new file mode 100644 index 00000000..9023b1cd --- /dev/null +++ b/benchmark_tasks/villani_rust_v1/rust_two_stage_fix/task.yaml @@ -0,0 +1,30 @@ +id: rust_two_stage_fix +family: bugfix +difficulty: hard +language: rust +max_minutes: 10 +max_files_touched: 4 +expected_artifacts: +- patch +visible_verification: +- cargo test --test test_fix -q +hidden_verification: +- cargo test -q +success_policy: + require_visible_pass: true + require_hidden_pass: true + fail_on_timeout: true + fail_on_repo_dirty_outside_allowlist: true +allowlist_paths: +- target/ +- src/ +- tests/ +allowed_paths: +- target/ +- src/ +- tests/ +expected_touched_max: 4 +adjacency_expected: true +recovery_expected: true +task_type: adjacent_two_file_fix +benchmark_track: core diff --git a/rust_01_success_rate.png b/rust_01_success_rate.png new file mode 100644 index 00000000..085823f4 Binary files /dev/null and b/rust_01_success_rate.png differ diff --git a/rust_02_solved_by_type.png b/rust_02_solved_by_type.png new file mode 100644 index 00000000..c926a3e6 Binary files /dev/null and b/rust_02_solved_by_type.png differ diff --git a/rust_03_execution_time_distribution.png b/rust_03_execution_time_distribution.png new file mode 100644 index 00000000..9154132e Binary files /dev/null and b/rust_03_execution_time_distribution.png differ diff --git a/villani_code/benchmark/agents/claude_code.py b/villani_code/benchmark/agents/claude_code.py index fdd77c38..713feb1d 100644 --- a/villani_code/benchmark/agents/claude_code.py +++ b/villani_code/benchmark/agents/claude_code.py @@ -59,7 +59,6 @@ def build_env(self, *, base_url: str | None, api_key: str | None, provider: str normalized_provider = (provider or "anthropic").strip().lower() if normalized_provider == "openai": env.pop("ANTHROPIC_BASE_URL", None) - env.pop("ANTHROPIC_API_KEY", None) if base_url: env["OPENAI_BASE_URL"] = base_url if api_key: diff --git a/villani_code/benchmark/policy.py b/villani_code/benchmark/policy.py index d3f39cb0..70bd90e0 100644 --- a/villani_code/benchmark/policy.py +++ b/villani_code/benchmark/policy.py @@ -14,6 +14,7 @@ "**/*.egg-info/**", "build/**", "dist/**", + "target/**", ".pytest_cache/**", "**/.mypy_cache/**", "**/.ruff_cache/**",