Fix nightly CI: workflow bugs, test hangs, diagnosability, milestone automation (Fixes #3149)

.github/workflows/nightly.yml

@@ -7,7 +7,11 @@ on:
 
 concurrency:
   group: 'nightly-${{ github.ref }}'
-  cancel-in-progress: false
+  # cancel-in-progress ensures that a superseded run (manual dispatch on the
+  # same ref, or a new nightly cron trigger) does not hold runner capacity.
+  # With it off those runs serialize in the group and contend for the runner
+  # pool, which produced multi-hour queue-wait outliers (issue #3149).
+  cancel-in-progress: true
 
 permissions:
   checks: 'write'
@@ -22,6 +26,7 @@ jobs:
   windows_ci:
     name: 'Windows CI (Nightly)'
     runs-on: '${{ matrix.os }}'
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
@@ -114,12 +119,6 @@ jobs:
           LLXPRT_COVERAGE: 'false'
         run: 'npm run test'
 
-      - name: 'Run script harness tests (macOS)'
-        if: matrix.os == 'macos-latest'
-        env:
-          CI: true
-        run: npm run test:scripts
-
       - name: 'Smoke test CLI entry (launcher -> Bun, no Node in chain)'
         run: './packages/cli/bin/llxprt --version'
 
@@ -210,6 +209,7 @@ jobs:
           echo "VITEST_POOL_TIMEOUT=60000" >> "$GITHUB_ENV"
 
       - name: 'Run tests and generate reports'
+        id: macos_main_tests
         env:
           # Provider configuration from repository secrets/variables
           OPENAI_API_KEY: ${{ secrets[vars.KEY_VAR_NAME] }}
@@ -231,11 +231,17 @@ jobs:
         run: 'npm run test'
 
       - name: 'Run script harness tests'
+        # Run when the main test step completed (success or failure) but NOT
+        # when the workflow was cancelled — so macOS platform coverage of the
+        # script harness still runs after a test failure, without burning
+        # runner minutes after cancellation (issue #3149).
+        if: ${{ !cancelled() && (steps.macos_main_tests.outcome == 'success' || steps.macos_main_tests.outcome == 'failure') }}
         env:
           CI: true
         run: 'npm run test:scripts'
 
       - name: 'Run shell-script behavioral tests (#2606)'
+        if: ${{ !cancelled() && (steps.macos_main_tests.outcome == 'success' || steps.macos_main_tests.outcome == 'failure') }}
         env:
           CI: true
         run: 'npm run test:shell'
@@ -316,6 +322,7 @@ jobs:
   e2e_full:
     name: 'E2E Full - ${{ matrix.os }} - ${{ matrix.sandbox }}'
     runs-on: '${{ matrix.os }}'
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
@@ -570,6 +577,7 @@ jobs:
     if: ${{ always() && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
     permissions:
       issues: 'write'
+      contents: 'read'
     steps:
       - name: 'Create Issue on Failure'
         env:
@@ -639,13 +647,50 @@ jobs:
             fi
           }
 
+          # Resolve the milestone for the version currently being worked on in
+          # main, so the auto-created nightly failure issue surfaces in the
+          # release view (issue #3149). Source of truth is the root package.json
+          # `version` on main — not the dispatched ref, which may have bumped
+          # it. Resolved by exact title match against open milestones (not
+          # nearest due date). Fails soft: an unresolvable milestone logs a
+          # warning and returns empty, so the notification still goes out.
+          resolve_milestone() {
+            local package_json
+            if ! package_json="$(gh api "repos/${GH_REPO}/contents/package.json?ref=main" -H 'Accept: application/vnd.github.raw' 2>/dev/null)"; then
+              echo "Warning: could not read package.json from main; skipping milestone resolution" >&2
+              return
+            fi
+            local current_version
+            current_version="$(printf '%s' "${package_json}" | jq -r '.version // empty' 2>/dev/null)"
+            if [[ -z "${current_version}" ]]; then
+              echo "Warning: package.json on main has no version field; skipping milestone resolution" >&2
+              return
+            fi
+            # gh issue --milestone accepts a milestone TITLE, not a number.
+            # Resolve the title by exact match across ALL open milestone
+            # pages (issue #3149), then return it for assignment.
+            local milestone_title
+            milestone_title="$(gh api --paginate --slurp "repos/${GH_REPO}/milestones?state=open&per_page=100" --jq "[.[][] | select(.title == \"${current_version}\")] | .[0].title // empty" 2>/dev/null)"
+            if [[ -z "${milestone_title}" ]]; then
+              echo "Warning: no open milestone titled '${current_version}'; skipping milestone assignment" >&2
+              return
+            fi
+            printf '%s' "${milestone_title}"
+          }
+
           LABEL_ARGS=()
           if ensure_label "ci/cd" "e33539" "Issues with github and the workflow scripts and CI CD environment."; then
             LABEL_ARGS+=(--label "ci/cd")
           else
             echo "Warning: continuing without ci/cd label" >&2
           fi
 
+          MILESTONE_TITLE="$(resolve_milestone)"
+          MILESTONE_ARGS=()
+          if [[ -n "${MILESTONE_TITLE}" ]]; then
+            MILESTONE_ARGS+=(--milestone "${MILESTONE_TITLE}")
+          fi
+
           retry_gh() {
             local attempt
             for attempt in 1 2 3 4; do
@@ -701,11 +746,20 @@ jobs:
               echo "ERROR: Failed to comment on existing issue #${EXISTING_ISSUE}" >&2
               exit 1
             }
+            # Keep a long-lived failure issue on the current release milestone
+            # too, not just on first creation (issue #3149).
+            if [[ -n "${MILESTONE_TITLE}" ]]; then
+              retry_gh gh issue edit "${EXISTING_ISSUE}" \
+                --repo "${GH_REPO}" \
+                --milestone "${MILESTONE_TITLE}" || \
+                echo "Warning: failed to set milestone '${MILESTONE_TITLE}' on existing issue #${EXISTING_ISSUE}" >&2
+            fi
           else
             INTRODUCTION="The nightly test workflow failed. Affected jobs: ${FAILED_JOBS_TEXT}."
             build_body_file "${BODY_FILE}" "${INTRODUCTION}"
             CREATE_ARGS=(--repo "${GH_REPO}" --title "${ISSUE_TITLE}" --body-file "${BODY_FILE}")
             CREATE_ARGS+=("${LABEL_ARGS[@]}")
+            CREATE_ARGS+=("${MILESTONE_ARGS[@]}")
             retry_gh gh issue create "${CREATE_ARGS[@]}" || {
               echo "ERROR: Failed to create nightly failure issue" >&2
               exit 1

packages/cli/run-bun-tests.ts

@@ -334,6 +334,94 @@ export function parseCaseCounts(output: string): {
   };
 }
 
+/**
+ * React's "not wrapped in act(...)" warning is a fixed ~10-line block that can
+ * repeat dozens of times in a single React/Ink test file. A naive tail of the
+ * output is then entirely warning text, and the assertion failures that
+ * actually failed the file are unrecoverable from the log. This collapses each
+ * warning block to a one-line count before taking the excerpt, so the failures
+ * stay visible within the budget. (issue #3149)
+ */
+const ACT_WARNING_START =
+  /^An update to .* inside a test was not wrapped in act\(\.\.\.\)/;
+const ACT_WARNING_END =
+  /Learn more at https:\/\/react\.dev\/link\/wrap-tests-with-act/;
+const NEWLINE = String.fromCharCode(10);
+
+function collapseActWarnings(output: string): {
+  body: string;
+  elidedWarnings: number;
+} {
+  const kept: string[] = [];
+  let elided = 0;
+  let skipping = false;
+  // Cap lines consumed after a warning start whose end marker never arrives
+  // (truncated at the source). The standard block is ~10 lines.
+  let skipLineCount = 0;
+  const MAX_WARNING_BODY_LINES = 15;
+  for (const line of output.split(NEWLINE)) {
+    if (!skipping) {
+      if (ACT_WARNING_START.test(line)) {
+        elided += 1;
+        skipLineCount = 1;
+        skipping = !ACT_WARNING_END.test(line);
+      } else {
+        kept.push(line);
+      }
+    } else {
+      skipLineCount++;
+      if (ACT_WARNING_END.test(line)) {
+        skipping = false;
+      } else if (ACT_WARNING_START.test(line)) {
+        elided += 1;
+        skipLineCount = 1;
+        skipping = !ACT_WARNING_END.test(line);
+      } else if (skipLineCount > MAX_WARNING_BODY_LINES) {
+        skipping = false;
+        kept.push(line);
+      }
+    }
+  }
+  return { body: kept.join(NEWLINE), elidedWarnings: elided };
+}
+
+/**
+ * Keeps an initial run and a final run of `text` so both the first failures
+ * and the trailing summary stay visible when the content exceeds the budget.
+ */
+function headTail(text: string, maxChars: number): string {
+  if (maxChars <= 0) return '';
+  if (text.length <= maxChars) return text;
+  const marker = `
+[... output elided ...]
+`;
+  const budget = Math.max(0, maxChars - marker.length);
+  const head = Math.ceil(budget / 2);
+  const tail = budget - head;
+  // Guard tail === 0: String.prototype.slice(-0) returns the whole string.
+  const tailSlice = tail > 0 ? text.slice(-tail) : '';
+  return `${text.slice(0, head)}${marker}${tailSlice}`;
+}
+
+/**
+ * Returns a bounded excerpt of a failing file's output that keeps assertion
+ * failures visible even when repetitive diagnostic noise would otherwise crowd
+ * them out of a fixed-size slice.
+ */
+export function failureExcerpt(output: string, maxChars: number): string {
+  if (output.length <= maxChars) {
+    return output;
+  }
+  const { body, elidedWarnings } = collapseActWarnings(output);
+  const banner =
+    elidedWarnings > 0
+      ? `[${elidedWarnings} React "not wrapped in act(...)" warning block(s) elided]
+`
+      : '';
+  const excerpt = headTail(body, maxChars - banner.length);
+  return (banner + excerpt).slice(0, maxChars);
+}
+
 export function generateJUnit(results: readonly TestResult[]): string {
   const failedCount = results.filter((result) => !result.passed).length;
   const testCases = results
@@ -349,7 +437,7 @@ export function generateJUnit(results: readonly TestResult[]): string {
             }s">TIMEOUT</failure>`
           : `<failure message="Exit code ${
               result.exitCode ?? -1
-            }">${escapeXml(result.output.slice(-4000))}</failure>`;
+            }">${escapeXml(failureExcerpt(stripAnsi(result.output), 4000))}</failure>`;
       return `    <testcase classname="${className}" name="${className}">${failure}</testcase>`;
     })
     .join('\n');
@@ -414,7 +502,7 @@ async function main(): Promise<void> {
 
   for (const result of failed) {
     console.error(`\n----- ${result.file} -----`);
-    console.error(result.output.slice(-6000));
+    console.error(failureExcerpt(stripAnsi(result.output), 6000));
   }
 
   const cases = results.reduce(

packages/cli/src/test-utils/render.tsx

@@ -30,8 +30,23 @@ export const render = (
   const originalUnmount = renderResult.unmount;
   const originalRerender = renderResult.rerender;
 
+  const actWrappedStdin = new Proxy(renderResult.stdin, {
+    get(target, prop, receiver) {
+      if (prop === 'write') {
+        return (...args: Parameters<typeof renderResult.stdin.write>) => {
+          act(() => {
+            target.write(...args);
+          });
+        };
+      }
+      const value = Reflect.get(target, prop, receiver);
+      return typeof value === 'function' ? value.bind(target) : value;
+    },
+  });
+
   return {
     ...renderResult,
+    stdin: actWrappedStdin,
     unmount: () => {
       act(() => {
         originalUnmount();