Extensibility requests - evals

dataset/extensibility_requests/default_setup.json

@@ -0,0 +1,8 @@
+{
+    "repo": "microsoftInternal/NAV",
+    "base_commit": "06e095c485e474431177e2170c0e22182bfdcc28",
+    "environment_setup_version": "27.0",
+    "project_paths": [
+        "App\\Layers"
+    ]
+}

dataset/extensibility_requests/extensibility_dataset.yaml

@@ -0,0 +1,140 @@
+entries:
+  - instance_id: issue-29447
+    input:
+      title: "[Event Request] Codeunit 5880 \"Phys. Invt. Order-Finish\""
+      description: |
+        ### Why do you need this change?
+
+        Hi,
+        i need an event in the Codeunit 5880 "Phys. Invt. Order-Finish" in the procedure "CreateOrderTrackingBufferLines" to modify the TempInvtOrderTrackingBuffer record.
+
+        ### Describe the request
+
+        Add a new event in the procedure "CreateOrderTrackingBufferLines", before modifying the record TempInvtOrderTrackingBuffer.
+
+            procedure CreateOrderTrackingBufferLines(DocNo: Code[20]; LineNo: Integer)
+            var
+                ExpInvtOrderTracking: Record "Exp. Invt. Order Tracking";
+                ItemTrackingSetup: Record "Item Tracking Setup";
+            begin
+                PhysInvtRecordLine.Reset();
+                PhysInvtRecordLine.SetCurrentKey("Order No.", "Order Line No.");
+                PhysInvtRecordLine.SetRange("Order No.", DocNo);
+                PhysInvtRecordLine.SetRange("Order Line No.", LineNo);
+                PhysInvtRecordLine.SetFilter("Quantity (Base)", '<>%1', 0);
+                if PhysInvtRecordLine.Find('-') then
+                    repeat
+                        ItemTrackingSetup."Serial No." := PhysInvtRecordLine."Serial No.";
+                        ItemTrackingSetup."Lot No." := PhysInvtRecordLine."Lot No.";
+                        ItemTrackingSetup."Package No." := PhysInvtRecordLine."Package No.";
+                        UpdateBufferRecordedQty(ItemTrackingSetup, PhysInvtRecordLine."Quantity (Base)", LineNo);
+                        OnCreateOrderTrackingBufferLinesFromPhysInvtRecordLine(TempInvtOrderTrackingBuffer, PhysInvtRecordLine);
+                    until PhysInvtRecordLine.Next() = 0;
+
+                ExpInvtOrderTracking.Reset();
+                ExpInvtOrderTracking.SetRange("Order No", DocNo);
+                ExpInvtOrderTracking.SetRange("Order Line No.", LineNo);
+                if ExpInvtOrderTracking.Find('-') then
+                    repeat
+                        ItemTrackingSetup."Serial No." := ExpInvtOrderTracking."Serial No.";
+                        ItemTrackingSetup."Lot No." := ExpInvtOrderTracking."Lot No.";
+                        ItemTrackingSetup."Package No." := ExpInvtOrderTracking."Package No.";
+                        UpdateBufferExpectedQty(ItemTrackingSetup, ExpInvtOrderTracking."Quantity (Base)", LineNo);
+                        OnCreateOrderTrackingBufferLinesFromExpInvtOrderTracking(TempInvtOrderTrackingBuffer, ExpInvtOrderTracking);
+                    until ExpInvtOrderTracking.Next() = 0;
+
+                TempInvtOrderTrackingBuffer.Reset();
+                if TempInvtOrderTrackingBuffer.Find('-') then
+                    repeat
+                        TempInvtOrderTrackingBuffer."Qty. To Transfer" :=
+                          TempInvtOrderTrackingBuffer."Qty. Recorded (Base)" - TempInvtOrderTrackingBuffer."Qty. Expected (Base)";
+                        TempInvtOrderTrackingBuffer."Outstanding Quantity" := TempInvtOrderTrackingBuffer."Qty. To Transfer";
+                        TempInvtOrderTrackingBuffer.Open := TempInvtOrderTrackingBuffer."Outstanding Quantity" <> 0;
+
+                        // Start Event
+
+                          OnCreateOrderTrackingBufferLinesOnBeforeModifyTempInvtOrderTracking(TempInvtOrderTrackingBuffer, DocNo, LineNo);
+
+                       // End Event
+
+                        TempInvtOrderTrackingBuffer.Modify();
+                    until TempInvtOrderTrackingBuffer.Next() = 0;
+            end;
+
+        **New Event**
+        ```
+        [IntegrationEvent(false, false)]
+            local procedure OnCreateOrderTrackingBufferLinesOnBeforeModifyTempInvtOrderTracking(var TempInvtOrderTracking: Record "Invt. Order Tracking" temporary; DocNo: Code[20]; LineNo: Integer)
+            begin
+            end;
+        ```
+
+      comments: ""
+
+    expected:
+      labels: "SCM, event-request"
+      comments:
+        - author: Agent
+          body: |
+            ✅ Analysis complete - approved for implementation
+
+            **Implementation:**
+
+            Add event publisher in procedure `CreateOrderTrackingBufferLines` before `TempInvtOrderTrackingBuffer.Modify()`:
+
+            ```al
+            TempInvtOrderTrackingBuffer.Reset();
+            if TempInvtOrderTrackingBuffer.Find('-') then
+                repeat
+                    TempInvtOrderTrackingBuffer."Qty. To Transfer" :=
+                      TempInvtOrderTrackingBuffer."Qty. Recorded (Base)" - TempInvtOrderTrackingBuffer."Qty. Expected (Base)";
+                    TempInvtOrderTrackingBuffer."Outstanding Quantity" := TempInvtOrderTrackingBuffer."Qty. To Transfer";
+                    TempInvtOrderTrackingBuffer.Open := TempInvtOrderTrackingBuffer."Outstanding Quantity" <> 0;
+                    OnCreateOrderTrackingBufferLinesOnBeforeModifyTempInvtOrderTracking(TempInvtOrderTrackingBuffer, DocNo, LineNo);
+                    TempInvtOrderTrackingBuffer.Modify();
+                until TempInvtOrderTrackingBuffer.Next() = 0;
+            ```
+
+            Add event publisher procedure at the end of the codeunit (before the closing brace):
+
+            ```al
+            [IntegrationEvent(false, false)]
+            local procedure OnCreateOrderTrackingBufferLinesOnBeforeModifyTempInvtOrderTracking(var TempInvtOrderTracking: Record "Invt. Order Tracking" temporary; DocNo: Code[20]; LineNo: Integer)
+            begin
+            end;
+            ```
+
+  - instance_id: issue-29442
+    input:
+      title: "[W1][Page][176][Standard Purchase Code Subform] Make procedure UpdateTypeText() not local"
+      description: |
+        ### Why do you need this change?
+
+        Procedure UpdateTypeText() is in local scope and it updates field variable TypeAsText
+
+        ### Describe the request
+
+        Remove "local" scope from procedure UpdateTypeText() in page 176 "Standard Purchase Code Subform";
+
+      comments: ""
+
+    expected:
+      labels: "SCM, request-for-external"
+      comments:
+        - author: Agent
+          body: |
+            ✅ Analysis complete - approved for implementation
+
+            **Implementation:**
+
+            Remove `local` scope from procedure `UpdateTypeText()` in Page 176 "Standard Purchase Code Subform":
+
+            ```al
+            procedure UpdateTypeText()
+            var
+                RecRef: RecordRef;
+            begin
+                RecRef.GetTable(Rec);
+                TypeAsText := TempOptionLookupBuffer.FormatOption(RecRef.Field(Rec.FieldNo(Type)));
+            end;
+            ```

docs/_data/extensibility-request.json

@@ -0,0 +1,4 @@
+{
+    "runs": [],
+    "aggregate": []
+}

src/bcbench/agent/copilot/agent.py

@@ -8,9 +8,9 @@
 import yaml
 
 from bcbench.agent.copilot.metrics import parse_metrics
-from bcbench.agent.shared import build_mcp_config, build_prompt
+from bcbench.agent.shared import build_mcp_config, build_prompt, build_prompt_ext
 from bcbench.config import get_config
-from bcbench.dataset import DatasetEntry
+from bcbench.dataset import DatasetEntry, ExtensibilityDatasetEntry
 from bcbench.exceptions import AgentError, AgentTimeoutError
 from bcbench.logger import get_logger
 from bcbench.operations import setup_custom_agent, setup_instructions_from_config
@@ -54,7 +54,6 @@
             "--log-level=debug",
             "--disable-parallel-tools-execution",
             f"--log-dir={output_dir.resolve()}",
-            f"-p={prompt.replace('\r', '').replace('\n', ' ')}",
         ]
         if not instructions_enabled:
             cmd_args.append("--no-custom-instructions")
@@ -69,6 +68,101 @@
             cmd_args,
             cwd=str(repo_path),
             stderr=subprocess.PIPE,  # only capture stderr where metrics are printed
+            input=prompt.encode("utf-8"),
+            timeout=_config.timeout.agent_execution,
+            check=True,
+        )
+
+        if result.stderr:
+            sys.stdout.buffer.write(result.stderr)
+            sys.stdout.buffer.flush()
+        logger.info(f"Copilot CLI run complete for: {entry.instance_id}")
+
+        stderr = result.stderr.decode("utf-8", errors="replace") if result.stderr else ""
+        stderr_lines = stderr.splitlines()
+
+        # Find the most recent session log for tool usage parsing
+        session_logs = list(output_dir.glob("process-*.log"))
+        session_log_path = max(session_logs, key=lambda p: p.stat().st_mtime) if session_logs else None
+
+        metrics = parse_metrics(stderr_lines, session_log_path=session_log_path)
+
+        return metrics, config
+    except subprocess.TimeoutExpired:
+        logger.error(f"Copilot CLI timed out after {_config.timeout.agent_execution} seconds")
+        metrics = AgentMetrics(execution_time=_config.timeout.agent_execution)
+        raise AgentTimeoutError("Copilot CLI timed out", metrics=metrics, config=config) from None
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Copilot CLI execution failed with error {e.stderr}")
+        raise AgentError(f"Copilot CLI execution failed: {e}") from None
+    except Exception as e:
+        logger.exception(f"Unexpected error running Copilot CLI: {e}")
+        raise
+
+
+def run_copilot_agent_ext(
+    entry: ExtensibilityDatasetEntry, model: str, category: EvaluationCategory, repo_path: Path, output_dir: Path, al_mcp: bool = False
+) -> tuple[AgentMetrics | None, ExperimentConfiguration]:
+    """Run GitHub Copilot CLI agent on a single dataset entry.
+
+    Returns:
+        Tuple of (AgentMetrics, ExperimentConfiguration) with metrics and configuration used during the experiment
+    """
+    config_file = Path(__file__).parent.parent / "shared" / "config.yaml"
+    copilot_config = yaml.safe_load(config_file.read_text())
+
+    logger.info(f"Running GitHub Copilot CLI on: {entry.instance_id}")
+
+    prompt: str = build_prompt_ext(entry, repo_path, copilot_config, category, al_mcp=al_mcp)
+    mcp_config_json, mcp_server_names = build_mcp_config(copilot_config, entry, repo_path, al_mcp=al_mcp)
+    instructions_enabled: bool = setup_instructions_from_config(copilot_config, entry, repo_path)
+    custom_agent: str | None = setup_custom_agent(copilot_config, entry, repo_path)
+    config = ExperimentConfiguration(mcp_servers=mcp_server_names, custom_instructions=instructions_enabled, custom_agent=custom_agent)
+
+    logger.info(f"Executing Copilot CLI in directory: {repo_path}")
+    logger.debug(f"Using prompt:\n{prompt}")
+
+    copilot_cmd = shutil.which("copilot.cmd") or shutil.which("copilot")
+    if not copilot_cmd:
+        raise AgentError("Copilot CLI not found in PATH. Please ensure it is installed and available.")
+
+    try:
+        cmd_args = [
+            copilot_cmd,
+            "--allow-all-tools",  # required for non-interactive mode
+            "--allow-all-paths",  # might be required for non-interactive mode, seems to hang when trying to access files outside allowed dirs
+            "--disable-builtin-mcps",
+            f"--model={model}",
+            "--log-level=debug",
+            "--disable-parallel-tools-execution",
+            f"--log-dir={output_dir.resolve()}",
+        ]
+        if not instructions_enabled:
+            cmd_args.append("--no-custom-instructions")
+        if mcp_config_json:
+            cmd_args.append(f"--additional-mcp-config={mcp_config_json}")
+        if custom_agent:
+            cmd_args.append(f"--agent={custom_agent}")
+
+        # Add prompt as argument
+        # We write it to a temporary file to avoid issues with quoting on Windows
+        import tempfile
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, encoding="utf-8") as tmp:
+            tmp.write(prompt)
+            tmp_path = tmp.name
+
+        # Copilot CLI often accepts a file via redirection but here we use the cat strategy or just pass regular args if we can,
+        # but since we had issues, let's try the pipe approach if supported, OR just standard input.
+        # But actually, looking at similar tools, they often just take the prompt as the last argument or via a file.
+        # Let's try passing it via stdin first as it assumes less about file flags.
+
+        logger.debug(f"Copilot command args: {cmd_args}")
+
+        result = subprocess.run(
+            cmd_args,
+            cwd=str(repo_path),
+            stderr=subprocess.PIPE,  # only capture stderr where metrics are printed
+            input=prompt.encode("utf-8"),
             timeout=_config.timeout.agent_execution,
             check=True,
         )

src/bcbench/agent/copilot/instructions/microsoftInternal-NAV/agents/Argus.agent.md

@@ -0,0 +1,77 @@
+---
+name: Argus
+description: 'Extensibility Analysis Agent specialized in analyzing GitHub extensibility issues.'
+tools: ['read/readFile', 'search/fileSearch', 'github/add_issue_comment', 'agent', 'todo']
+---
+
+This agent acts as an Extensibility Analysis Agent. Its purpose is to analyze GitHub extensibility issues by collecting data, checking eligibility, determining request types, verifying requirements, analyzing the codebase, and finally assigning teams and applying labels/comments.
+
+0. Initialize the agent based on the instructions from #file:../instructions/Argus/step0-getting-started.md . Determine the issues to process (single or multiple) and validate the environment.
+
+For each issue identified in step 0, execute ALL the following steps (1-7) sequentially.
+
+**Logging:**
+Create a new markdown file named `issue_<issue_number>_log.md` to record the execution flow.
+After each step (1-6), append the step name, the input variables sent, and the full output received from the subagent to this log file.
+
+**Progress Tracking:**
+Use the `todo` tool to track the execution of the workflow for the current issue.
+- **Initialize:** Before starting Step 1, create a todo list with items for Steps 1 through 7.
+- **Update:** Before executing a step, mark it as `in-progress`. Upon completion, mark it as `completed`.
+- **Flow Control:** If a step fails or logic dictates skipping to Step 7, leave skipped steps as `not-started` and proceed to update Step 7.
+
+1. Step 1: Collect
+- Call  #tool:agent/runSubagent with:
+  - name: "step1-collector-subagent"
+  - instructions: "Collect all necessary data with using your own file access to open the instructions file. If this step fails (returned output is not success), stop processing the current issue."
+  - input_vars: {issue_number: "${state.issue_number}"}
+  - context.resources: ["file:../instructions/Argus/step1-collect-data.md"]
+  - context.tools: ["github/issue_read"]
+- Expect output: {"Success": boolean, "GH_REQUEST": object, "FailureReason": string}
+
+2. Step 2: Eligibility Check
+- Call  #tool:agent/runSubagent with:
+  - name: "step2-eligibility-check-subagent"
+  - instructions: "Analyze the issue eligibility using your own file access to open the instructions file. If this step fails (returned output is not eligible), stop processing the current issue or if stale, proceed directly to step 7."
+  - input_vars: {GH_REQUEST: "${state.GH_REQUEST}"}
+  - context.resources: ["file:../instructions/Argus/step2-eligibility-check.md"]
+  - context.tools: []
+- Expect output: {"IsEligible": boolean, "IsStale": boolean, "FailureReason": string}
+
+3. Step 3: Request Types
+- Call  #tool:agent/runSubagent with:
+  - name: "step3-request-types-subagent"
+  - instructions: "Determine request types using your own file access to open the instructions file. If this step fails (returned output is not success), proceed directly to step 7."
+  - input_vars: {GH_REQUEST: "${state.GH_REQUEST}"}
+  - context.resources: ["file:../instructions/Argus/step3-request-types.md"]
+  - context.tools: []
+- Expect output: {"Success": boolean, "TYPE": string, "SUBTYPE": string, "FailureLabel": string, "FailureReason": string}
+
+4. Step 4: Requirements Check
+- Call  #tool:agent/runSubagent with:
+  - name: "step4-requirements-check-subagent"
+  - instructions: "Verify requirements using your own file access to open the instructions file. If this step fails (returned output is not success), proceed directly to step 7."
+  - input_vars: {GH_REQUEST: "${state.GH_REQUEST}", TYPE: "${state.TYPE}", SUBTYPE: "${state.SUBTYPE}"}
+  - context.resources: ["file:../instructions/Argus/step4-requirements-check.md"]
+  - context.tools: ['search/fileSearch', 'read/readFile']
+- Expect output: {"Success": boolean, "FailureLabel": string, "FailureReason": string}
+
+5. Step 5: Codebase Analysis
+- Call  #tool:agent/runSubagent with:
+  - name: "step5-codebase-analysis-subagent"
+  - instructions: "Analyze the codebase using your own file access to open the instructions file. If this step fails (returned output is not success), proceed directly to step 7."
+  - input_vars: {GH_REQUEST: "${state.GH_REQUEST}", TYPE: "${state.TYPE}", SUBTYPE: "${state.SUBTYPE}"}
+  - context.resources: ["file:../instructions/Argus/step5-codebase-analysis.md"]
+  - context.tools: ['search/fileSearch', 'read/readFile']
+- Expect output: {"Success": boolean, "OBJECT_LIST": array, "SUGGESTED_IMPLEMENTATION": string, "FailureLabel": string, "FailureReason": string}
+
+6. Step 6: Team Assignment
+- Call  #tool:agent/runSubagent with:
+  - name: "step6-team-assignment-subagent"
+  - instructions: "Assign teams based on namespaces using your own file access to open the instructions file. If this step fails (returned output is not success), proceed directly to step 7."
+  - input_vars: {OBJECT_LIST: "${state.OBJECT_LIST}"}
+  - context.resources: ["file:../instructions/Argus/step6-team-assignment.md"]
+  - context.tools: ['read/readFile', 'write/writeFile']
+- Expect output: {"Success": boolean, "TEAM_LABEL": string, "FailureLabel": string, "FailureReason": string}
+
+7. Step 7: Finalize the process based on the instructions from #file:../instructions/Argus/step7-labels-comments.md . Use all collected data (including any failure reasons if applicable) to avoid refetching.